From 1a9da7c70db73aad4597eed7cae598e7f0130ece Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 06:19:48 +0000 Subject: [PATCH 01/13] Initial plan From faa4dd674682f4cd207a882bd93c2e2e8b48ca72 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 06:25:28 +0000 Subject: [PATCH 02/13] Add Thai profanity detection feature Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/corpus/__init__.py | 2 + pythainlp/corpus/common.py | 20 ++++++ pythainlp/corpus/profanity_th.txt | 29 ++++++++ pythainlp/util/__init__.py | 8 +++ pythainlp/util/profanity.py | 115 ++++++++++++++++++++++++++++++ tests/core/test_profanity.py | 80 +++++++++++++++++++++ 6 files changed, 254 insertions(+) create mode 100644 pythainlp/corpus/profanity_th.txt create mode 100644 pythainlp/util/profanity.py create mode 100644 tests/core/test_profanity.py diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 424a25976..9c73a801f 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -34,6 +34,7 @@ "thai_male_names", "thai_negations", "thai_orst_words", + "thai_profanity_words", "thai_stopwords", "thai_syllables", "thai_synonym", @@ -115,6 +116,7 @@ def corpus_db_path() -> str: thai_male_names, thai_negations, thai_orst_words, + thai_profanity_words, thai_stopwords, thai_syllables, thai_synonym, diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 6e23a7da4..7d0ee2398 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -18,6 +18,7 @@ "thai_male_names", "thai_negations", "thai_dict", + "thai_profanity_words", "thai_stopwords", "thai_syllables", "thai_synonym", @@ -50,6 +51,9 @@ _THAI_NEGATIONS: FrozenSet[str] = frozenset() _THAI_NEGATIONS_FILENAME = "negations_th.txt" +_THAI_PROFANITY_WORDS: FrozenSet[str] = frozenset() +_THAI_PROFANITY_WORDS_FILENAME = "profanity_th.txt" + _THAI_FAMLIY_NAMES: FrozenSet[str] = frozenset() _THAI_FAMLIY_NAMES_FILENAME = "family_names_th.txt" _THAI_FEMALE_NAMES: FrozenSet[str] = frozenset() @@ -213,6 +217,22 @@ def thai_negations() -> FrozenSet[str]: return _THAI_NEGATIONS +def thai_profanity_words() -> FrozenSet[str]: + """ + Return a frozenset of Thai profanity words for content filtering. + \n(See: `dev/pythainlp/corpus/profanity_th.txt\ + `_) + + :return: :class:`frozenset` containing profanity words in the Thai language. + :rtype: :class:`frozenset` + """ + global _THAI_PROFANITY_WORDS + if not _THAI_PROFANITY_WORDS: + _THAI_PROFANITY_WORDS = get_corpus(_THAI_PROFANITY_WORDS_FILENAME, comments=False) + + return _THAI_PROFANITY_WORDS + + def thai_family_names() -> FrozenSet[str]: """ Return a frozenset of Thai family names diff --git a/pythainlp/corpus/profanity_th.txt b/pythainlp/corpus/profanity_th.txt new file mode 100644 index 000000000..797e86efe --- /dev/null +++ b/pythainlp/corpus/profanity_th.txt @@ -0,0 +1,29 @@ +# Thai profanity words +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 +# +# This list contains common Thai profanity words for filtering/detection purposes +# Words are stored without tone marks or variations for broader matching +ควย +สัส +เหี้ย +ไอ้เหี้ย +ไอ้สัส +ระยำ +เย็ด +มึง +ไอ้มึง +กู +ไอ้กู +เชี่ย +ดอกทอง +สัตว์ +ไอ้สัตว์ +ชาติหมา +ตาย +ไอ้ตาย +บ้า +ไอ้บ้า +เวร +ไอ้เวร +เหี้ยน diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index a36aa5cdd..dca216969 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -11,7 +11,9 @@ "abbreviation_to_full_text", "arabic_digit_to_thai_digit", "bahttext", + "censor_profanity", "collate", + "contains_profanity", "convert_years", "count_thai_chars", "countthai", @@ -22,6 +24,7 @@ "eng_to_thai", "expand_maiyamok", "find_keyword", + "find_profanity", "ipa_to_rtgs", "is_native_thai", "isthai", @@ -111,6 +114,11 @@ ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa +from pythainlp.util.profanity import ( + censor_profanity, + contains_profanity, + find_profanity, +) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py new file mode 100644 index 000000000..cf1676672 --- /dev/null +++ b/pythainlp/util/profanity.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Profanity detection for Thai language +""" +from typing import List, Union + +from pythainlp.corpus.common import thai_profanity_words +from pythainlp.tokenize import word_tokenize + + +def contains_profanity(text: str, engine: str = "newmm") -> bool: + """ + Check if the given text contains profanity words. + + :param str text: Thai text to check + :param str engine: tokenization engine (default: "newmm") + :return: True if text contains profanity, False otherwise + :rtype: bool + + :Example: + :: + + from pythainlp.util import contains_profanity + + print(contains_profanity("สวัสดีครับ")) + # output: False + + print(contains_profanity("คำหยาบคาย")) + # output: True if the word is in the profanity list + """ + if not text: + return False + + profanity_set = thai_profanity_words() + tokens = word_tokenize(text, engine=engine) + + for token in tokens: + if token in profanity_set: + return True + + return False + + +def find_profanity(text: str, engine: str = "newmm") -> List[str]: + """ + Find all profanity words in the given text. + + :param str text: Thai text to check + :param str engine: tokenization engine (default: "newmm") + :return: List of profanity words found in the text + :rtype: List[str] + + :Example: + :: + + from pythainlp.util import find_profanity + + print(find_profanity("สวัสดีครับ")) + # output: [] + + print(find_profanity("text with profanity words")) + # output: ['profanity_word1', 'profanity_word2'] + """ + if not text: + return [] + + profanity_set = thai_profanity_words() + tokens = word_tokenize(text, engine=engine) + + found_profanity = [] + for token in tokens: + if token in profanity_set: + found_profanity.append(token) + + return found_profanity + + +def censor_profanity(text: str, replacement: str = "*", engine: str = "newmm") -> str: + """ + Replace profanity words in the text with a replacement character. + + :param str text: Thai text to censor + :param str replacement: character to replace profanity with (default: "*") + :param str engine: tokenization engine (default: "newmm") + :return: Text with profanity words censored + :rtype: str + + :Example: + :: + + from pythainlp.util import censor_profanity + + print(censor_profanity("สวัสดีครับ")) + # output: สวัสดีครับ + + print(censor_profanity("text with profanity word")) + # output: text with *** word + """ + if not text: + return text + + profanity_set = thai_profanity_words() + tokens = word_tokenize(text, engine=engine, keep_whitespace=True) + + censored_tokens = [] + for token in tokens: + if token in profanity_set: + censored_tokens.append(replacement * len(token)) + else: + censored_tokens.append(token) + + return "".join(censored_tokens) diff --git a/tests/core/test_profanity.py b/tests/core/test_profanity.py new file mode 100644 index 000000000..ecf8ed798 --- /dev/null +++ b/tests/core/test_profanity.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for profanity detection functions +""" + +import unittest + +from pythainlp.corpus import thai_profanity_words +from pythainlp.util import ( + censor_profanity, + contains_profanity, + find_profanity, +) + + +class TestProfanity(unittest.TestCase): + def test_thai_profanity_words(self): + """Test loading profanity word list""" + words = thai_profanity_words() + self.assertIsInstance(words, frozenset) + self.assertGreater(len(words), 0) + + def test_contains_profanity_clean_text(self): + """Test clean text without profanity""" + self.assertFalse(contains_profanity("สวัสดีครับ")) + self.assertFalse(contains_profanity("วันนี้อากาศดีมาก")) + self.assertFalse(contains_profanity("")) + + def test_contains_profanity_with_profanity(self): + """Test text containing profanity""" + # Test with actual profanity words from the list + self.assertTrue(contains_profanity("ควย")) + self.assertTrue(contains_profanity("สัส")) + self.assertTrue(contains_profanity("สวัสดี ควย ครับ")) + + def test_find_profanity_clean_text(self): + """Test finding profanity in clean text""" + self.assertEqual(find_profanity("สวัสดีครับ"), []) + self.assertEqual(find_profanity(""), []) + + def test_find_profanity_with_profanity(self): + """Test finding profanity words""" + result = find_profanity("ควย") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = find_profanity("สัส") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_censor_profanity_clean_text(self): + """Test censoring clean text""" + text = "สวัสดีครับ" + self.assertEqual(censor_profanity(text), text) + self.assertEqual(censor_profanity(""), "") + + def test_censor_profanity_with_profanity(self): + """Test censoring profanity words""" + # Test that profanity is replaced with stars + result = censor_profanity("ควย") + self.assertNotEqual(result, "ควย") + self.assertIn("*", result) + + result = censor_profanity("สัส") + self.assertNotEqual(result, "สัส") + self.assertIn("*", result) + + def test_censor_profanity_custom_replacement(self): + """Test censoring with custom replacement character""" + result = censor_profanity("ควย", replacement="#") + self.assertIn("#", result) + self.assertNotIn("*", result) + + +if __name__ == "__main__": + unittest.main() From 9bce90264bdae29697bd0d63d3a2554fc446f274 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 06:27:32 +0000 Subject: [PATCH 03/13] Remove trailing empty line from profanity_th.txt Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/corpus/profanity_th.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/corpus/profanity_th.txt b/pythainlp/corpus/profanity_th.txt index 797e86efe..d812820e3 100644 --- a/pythainlp/corpus/profanity_th.txt +++ b/pythainlp/corpus/profanity_th.txt @@ -26,4 +26,4 @@ ไอ้บ้า เวร ไอ้เวร -เหี้ยน +เหี้ยน \ No newline at end of file From 10c1413613a337884f003f1dcab84e0b771b862a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 14:59:52 +0000 Subject: [PATCH 04/13] Add custom_words parameter and improve tokenization with merged dict - All three functions (contains_profanity, find_profanity, censor_profanity) now accept custom_words parameter - Implemented custom dictionary that merges thai_words and profanity_set for better tokenization - Added comprehensive tests for custom words functionality - All 11 tests pass Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/util/profanity.py | 78 +++++++++++++++++++++++++++++++----- tests/core/test_profanity.py | 34 ++++++++++++++++ 2 files changed, 101 insertions(+), 11 deletions(-) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index cf1676672..e677bab76 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -5,17 +5,23 @@ """ Profanity detection for Thai language """ -from typing import List, Union +from typing import List, Set, Union -from pythainlp.corpus.common import thai_profanity_words +from pythainlp.corpus.common import thai_profanity_words, thai_words from pythainlp.tokenize import word_tokenize +from pythainlp.util.trie import Trie, dict_trie -def contains_profanity(text: str, engine: str = "newmm") -> bool: +def contains_profanity( + text: str, + custom_words: Set[str] = None, + engine: str = "newmm" +) -> bool: """ Check if the given text contains profanity words. :param str text: Thai text to check + :param set custom_words: additional profanity words to check (default: None) :param str engine: tokenization engine (default: "newmm") :return: True if text contains profanity, False otherwise :rtype: bool @@ -30,12 +36,25 @@ def contains_profanity(text: str, engine: str = "newmm") -> bool: print(contains_profanity("คำหยาบคาย")) # output: True if the word is in the profanity list + + # Add custom profanity words + print(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: True """ if not text: return False - profanity_set = thai_profanity_words() - tokens = word_tokenize(text, engine=engine) + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) for token in tokens: if token in profanity_set: @@ -44,11 +63,16 @@ def contains_profanity(text: str, engine: str = "newmm") -> bool: return False -def find_profanity(text: str, engine: str = "newmm") -> List[str]: +def find_profanity( + text: str, + custom_words: Set[str] = None, + engine: str = "newmm" +) -> List[str]: """ Find all profanity words in the given text. :param str text: Thai text to check + :param set custom_words: additional profanity words to check (default: None) :param str engine: tokenization engine (default: "newmm") :return: List of profanity words found in the text :rtype: List[str] @@ -63,12 +87,25 @@ def find_profanity(text: str, engine: str = "newmm") -> List[str]: print(find_profanity("text with profanity words")) # output: ['profanity_word1', 'profanity_word2'] + + # Add custom profanity words + print(find_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: ['คำใหม่'] """ if not text: return [] - profanity_set = thai_profanity_words() - tokens = word_tokenize(text, engine=engine) + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) found_profanity = [] for token in tokens: @@ -78,12 +115,18 @@ def find_profanity(text: str, engine: str = "newmm") -> List[str]: return found_profanity -def censor_profanity(text: str, replacement: str = "*", engine: str = "newmm") -> str: +def censor_profanity( + text: str, + replacement: str = "*", + custom_words: Set[str] = None, + engine: str = "newmm" +) -> str: """ Replace profanity words in the text with a replacement character. :param str text: Thai text to censor :param str replacement: character to replace profanity with (default: "*") + :param set custom_words: additional profanity words to censor (default: None) :param str engine: tokenization engine (default: "newmm") :return: Text with profanity words censored :rtype: str @@ -98,12 +141,25 @@ def censor_profanity(text: str, replacement: str = "*", engine: str = "newmm") - print(censor_profanity("text with profanity word")) # output: text with *** word + + # Add custom profanity words + print(censor_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: ****** """ if not text: return text - profanity_set = thai_profanity_words() - tokens = word_tokenize(text, engine=engine, keep_whitespace=True) + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine, keep_whitespace=True) censored_tokens = [] for token in tokens: diff --git a/tests/core/test_profanity.py b/tests/core/test_profanity.py index ecf8ed798..e013a6c5d 100644 --- a/tests/core/test_profanity.py +++ b/tests/core/test_profanity.py @@ -75,6 +75,40 @@ def test_censor_profanity_custom_replacement(self): self.assertIn("#", result) self.assertNotIn("*", result) + def test_contains_profanity_with_custom_words(self): + """Test detection with custom profanity words""" + # Clean text shouldn't be detected + self.assertFalse(contains_profanity("สวัสดีครับ", custom_words={"คำใหม่"})) + + # Custom word should be detected + self.assertTrue(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) + + # Mix of default and custom words + self.assertTrue(contains_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"})) + + def test_find_profanity_with_custom_words(self): + """Test finding profanity with custom words""" + # Should find custom words + result = find_profanity("คำใหม่", custom_words={"คำใหม่"}) + self.assertIn("คำใหม่", result) + + # Should find both default and custom words + result = find_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) + self.assertGreater(len(result), 1) + + def test_censor_profanity_with_custom_words(self): + """Test censoring with custom profanity words""" + # Should censor custom words + result = censor_profanity("คำใหม่", custom_words={"คำใหม่"}) + self.assertNotEqual(result, "คำใหม่") + self.assertIn("*", result) + + # Should censor both default and custom words + result = censor_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) + self.assertNotIn("ควย", result) + self.assertNotIn("คำใหม่", result) + self.assertIn("*", result) + if __name__ == "__main__": unittest.main() From 1af6b4875504a12cc6f8585599c2a9d2df35ae94 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 16:40:26 +0000 Subject: [PATCH 05/13] Update type hints to use lowercase built-in types (Python 3.9+) - Changed FrozenSet[str] to frozenset[str] - Changed List[str] to list[str] - Changed Set[str] to set[str] - Removed unnecessary typing imports - Aligns with dev branch type hint style Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/corpus/common.py | 52 ++++++++++++++++++------------------- pythainlp/util/profanity.py | 14 +++++----- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 7d0ee2398..09d4ee9b1 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -27,48 +27,48 @@ "thai_wsd_dict", ] -from typing import FrozenSet, List, Union +from typing import Union from pythainlp.corpus import get_corpus, get_corpus_as_is, get_corpus_path from pythainlp.tools import warn_deprecation -_THAI_COUNTRIES: FrozenSet[str] = frozenset() +_THAI_COUNTRIES: frozenset[str] = frozenset() _THAI_COUNTRIES_FILENAME = "countries_th.txt" -_THAI_THAILAND_PROVINCES: FrozenSet[str] = frozenset() -_THAI_THAILAND_PROVINCES_DETAILS: List[dict] = [] +_THAI_THAILAND_PROVINCES: frozenset[str] = frozenset() +_THAI_THAILAND_PROVINCES_DETAILS: list[dict] = [] _THAI_THAILAND_PROVINCES_FILENAME = "thailand_provinces_th.csv" -_THAI_SYLLABLES: FrozenSet[str] = frozenset() +_THAI_SYLLABLES: frozenset[str] = frozenset() _THAI_SYLLABLES_FILENAME = "syllables_th.txt" -_THAI_WORDS: FrozenSet[str] = frozenset() +_THAI_WORDS: frozenset[str] = frozenset() _THAI_WORDS_FILENAME = "words_th.txt" -_THAI_STOPWORDS: FrozenSet[str] = frozenset() +_THAI_STOPWORDS: frozenset[str] = frozenset() _THAI_STOPWORDS_FILENAME = "stopwords_th.txt" -_THAI_NEGATIONS: FrozenSet[str] = frozenset() +_THAI_NEGATIONS: frozenset[str] = frozenset() _THAI_NEGATIONS_FILENAME = "negations_th.txt" -_THAI_PROFANITY_WORDS: FrozenSet[str] = frozenset() +_THAI_PROFANITY_WORDS: frozenset[str] = frozenset() _THAI_PROFANITY_WORDS_FILENAME = "profanity_th.txt" -_THAI_FAMLIY_NAMES: FrozenSet[str] = frozenset() +_THAI_FAMLIY_NAMES: frozenset[str] = frozenset() _THAI_FAMLIY_NAMES_FILENAME = "family_names_th.txt" -_THAI_FEMALE_NAMES: FrozenSet[str] = frozenset() +_THAI_FEMALE_NAMES: frozenset[str] = frozenset() _THAI_FEMALE_NAMES_FILENAME = "person_names_female_th.txt" -_THAI_MALE_NAMES: FrozenSet[str] = frozenset() +_THAI_MALE_NAMES: frozenset[str] = frozenset() _THAI_MALE_NAMES_FILENAME = "person_names_male_th.txt" -_THAI_ORST_WORDS: FrozenSet[str] = frozenset() +_THAI_ORST_WORDS: frozenset[str] = frozenset() _THAI_DICT: dict[str, list] = {} _THAI_WSD_DICT: dict[str, list] = {} _THAI_SYNONYMS: dict[str, list] = {} -def countries() -> FrozenSet[str]: +def countries() -> frozenset[str]: """ Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", "แอลจีเรีย", and "ลาว". @@ -85,7 +85,7 @@ def countries() -> FrozenSet[str]: return _THAI_COUNTRIES -def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]: +def provinces(details: bool = False) -> Union[frozenset[str], list[dict]]: """ Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". @@ -128,7 +128,7 @@ def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]: return _THAI_THAILAND_PROVINCES -def thai_syllables() -> FrozenSet[str]: +def thai_syllables() -> frozenset[str]: """ Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", "โมน", "โม่ง", "กา", "ก่า", and, "ก้า". @@ -146,7 +146,7 @@ def thai_syllables() -> FrozenSet[str]: return _THAI_SYLLABLES -def thai_words() -> FrozenSet[str]: +def thai_words() -> frozenset[str]: """ Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", and "พิษภัย". \n(See: `dev/pythainlp/corpus/words_th.txt\ @@ -162,7 +162,7 @@ def thai_words() -> FrozenSet[str]: return _THAI_WORDS -def thai_orst_words() -> FrozenSet[str]: +def thai_orst_words() -> frozenset[str]: """ Return a frozenset of Thai words from Royal Society of Thailand \n(See: `dev/pythainlp/corpus/thai_orst_words.txt\ @@ -178,7 +178,7 @@ def thai_orst_words() -> FrozenSet[str]: return _THAI_ORST_WORDS -def thai_stopwords() -> FrozenSet[str]: +def thai_stopwords() -> frozenset[str]: """ Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", "การ", and "ประการหนึ่ง". \n(See: `dev/pythainlp/corpus/stopwords_th.txt\ @@ -201,7 +201,7 @@ def thai_stopwords() -> FrozenSet[str]: return _THAI_STOPWORDS -def thai_negations() -> FrozenSet[str]: +def thai_negations() -> frozenset[str]: """ Return a frozenset of Thai negation words including "ไม่" and "แต่". \n(See: `dev/pythainlp/corpus/negations_th.txt\ @@ -217,7 +217,7 @@ def thai_negations() -> FrozenSet[str]: return _THAI_NEGATIONS -def thai_profanity_words() -> FrozenSet[str]: +def thai_profanity_words() -> frozenset[str]: """ Return a frozenset of Thai profanity words for content filtering. \n(See: `dev/pythainlp/corpus/profanity_th.txt\ @@ -233,7 +233,7 @@ def thai_profanity_words() -> FrozenSet[str]: return _THAI_PROFANITY_WORDS -def thai_family_names() -> FrozenSet[str]: +def thai_family_names() -> frozenset[str]: """ Return a frozenset of Thai family names \n(See: `dev/pythainlp/corpus/family_names_th.txt\ @@ -249,7 +249,7 @@ def thai_family_names() -> FrozenSet[str]: return _THAI_FAMLIY_NAMES -def thai_female_names() -> FrozenSet[str]: +def thai_female_names() -> frozenset[str]: """ Return a frozenset of Thai female names \n(See: `dev/pythainlp/corpus/person_names_female_th.txt\ @@ -265,7 +265,7 @@ def thai_female_names() -> FrozenSet[str]: return _THAI_FEMALE_NAMES -def thai_male_names() -> FrozenSet[str]: +def thai_male_names() -> frozenset[str]: """ Return a frozenset of Thai male names \n(See: `dev/pythainlp/corpus/person_names_male_th.txt\ @@ -380,13 +380,13 @@ def thai_synonym() -> dict: return thai_synonyms() -def find_synonyms(word: str) -> List[str]: +def find_synonyms(word: str) -> list[str]: """ Find synonyms :param str word: Thai word :return: List of synonyms of the input word or an empty list if it isn't exist. - :rtype: List[str] + :rtype: list[str] :Example: :: diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index e677bab76..58e6e77b1 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -5,7 +5,7 @@ """ Profanity detection for Thai language """ -from typing import List, Set, Union +from typing import Union from pythainlp.corpus.common import thai_profanity_words, thai_words from pythainlp.tokenize import word_tokenize @@ -14,7 +14,7 @@ def contains_profanity( text: str, - custom_words: Set[str] = None, + custom_words: set[str] = None, engine: str = "newmm" ) -> bool: """ @@ -65,17 +65,17 @@ def contains_profanity( def find_profanity( text: str, - custom_words: Set[str] = None, + custom_words: set[str] = None, engine: str = "newmm" -) -> List[str]: +) -> list[str]: """ Find all profanity words in the given text. :param str text: Thai text to check :param set custom_words: additional profanity words to check (default: None) :param str engine: tokenization engine (default: "newmm") - :return: List of profanity words found in the text - :rtype: List[str] + :return: list of profanity words found in the text + :rtype: list[str] :Example: :: @@ -118,7 +118,7 @@ def find_profanity( def censor_profanity( text: str, replacement: str = "*", - custom_words: Set[str] = None, + custom_words: set[str] = None, engine: str = "newmm" ) -> str: """ From a8b17f7743a2798898783430db23a0c081d99660 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 16:42:58 +0000 Subject: [PATCH 06/13] Merge dev branch - Resolve conflicts and align with latest codebase - Resolved merge conflicts in pythainlp/corpus/common.py - Added profanity detection feature to merged codebase - Updated to use Python 3.9+ type hints (lowercase built-in types) - All tests pass (11 profanity tests) Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi-test.yml | 3 +- .github/workflows/unittest.yml | 6 +- CONTRIBUTING.md | 61 +- Dockerfile | 4 +- Makefile | 15 +- README.md | 32 +- README_TH.md | 30 +- codemeta.json | 16 +- docker_requirements.txt | 38 - docs/api/spell.rst | 4 +- docs/conf.py | 8 +- examples/khavee.py | 6 +- notebooks/convert_thai2rom_to_onnx.ipynb | 2 +- notebooks/create_words.ipynb | 4 +- notebooks/test_chat.ipynb | 5 +- notebooks/test_el.ipynb | 1 + notebooks/test_wangchanglm.ipynb | 5 +- notebooks/test_wsd.ipynb | 2 +- pyproject.toml | 288 +++- pythainlp/__init__.py | 142 +- pythainlp/__main__.py | 1 - pythainlp/ancient/__init__.py | 4 +- pythainlp/ancient/aksonhan.py | 5 +- pythainlp/ancient/currency.py | 22 +- pythainlp/augment/__init__.py | 4 +- pythainlp/augment/lm/__init__.py | 4 +- pythainlp/augment/lm/fasttext.py | 25 +- pythainlp/augment/lm/phayathaibert.py | 8 +- pythainlp/augment/lm/wangchanberta.py | 9 +- pythainlp/augment/word2vec/__init__.py | 4 +- pythainlp/augment/word2vec/bpemb_wv.py | 19 +- pythainlp/augment/word2vec/core.py | 17 +- pythainlp/augment/word2vec/ltw2v.py | 19 +- pythainlp/augment/word2vec/thai2fit.py | 24 +- pythainlp/augment/wordnet.py | 28 +- pythainlp/benchmarks/__init__.py | 4 +- pythainlp/benchmarks/word_tokenization.py | 43 +- pythainlp/chat/__init__.py | 4 +- pythainlp/chat/core.py | 17 +- pythainlp/classify/__init__.py | 4 +- pythainlp/classify/param_free.py | 52 +- pythainlp/cli/__init__.py | 18 +- pythainlp/cli/benchmark.py | 21 +- pythainlp/cli/data.py | 4 +- pythainlp/cli/misspell.py | 3 +- pythainlp/cli/soundex.py | 4 +- pythainlp/cli/tag.py | 4 +- pythainlp/cli/tokenize.py | 4 +- pythainlp/coref/__init__.py | 5 +- pythainlp/coref/_fastcoref.py | 5 +- pythainlp/coref/core.py | 8 +- pythainlp/coref/han_coref.py | 3 +- pythainlp/corpus/__init__.py | 22 +- pythainlp/corpus/common.py | 57 +- pythainlp/corpus/core.py | 120 +- pythainlp/corpus/corpus_license.md | 54 +- pythainlp/corpus/icu.py | 13 +- pythainlp/corpus/oscar.py | 19 +- pythainlp/corpus/th_en_translit.py | 11 +- pythainlp/corpus/tnc.py | 29 +- pythainlp/corpus/ttc.py | 15 +- pythainlp/corpus/util.py | 34 +- pythainlp/corpus/volubilis.py | 14 +- pythainlp/corpus/wikipedia.py | 16 +- pythainlp/corpus/wordnet.py | 99 +- pythainlp/el/__init__.py | 4 +- pythainlp/el/_multiel.py | 1 - pythainlp/el/core.py | 34 +- pythainlp/generate/__init__.py | 4 +- pythainlp/generate/core.py | 37 +- pythainlp/generate/thai2fit.py | 12 +- pythainlp/generate/wangchanglm.py | 126 +- pythainlp/khavee/__init__.py | 15 +- pythainlp/khavee/core.py | 1373 ++++++++--------- pythainlp/lm/__init__.py | 11 +- pythainlp/lm/text_util.py | 28 +- pythainlp/morpheme/__init__.py | 10 +- pythainlp/morpheme/thaiwordcheck.py | 12 +- pythainlp/morpheme/word_formation.py | 10 +- pythainlp/parse/__init__.py | 5 +- pythainlp/parse/core.py | 11 +- pythainlp/parse/esupar_engine.py | 11 +- pythainlp/parse/spacy_thai_engine.py | 11 +- pythainlp/parse/transformers_ud.py | 12 +- pythainlp/parse/ud_goeswith.py | 11 +- pythainlp/phayathaibert/__init__.py | 4 +- pythainlp/phayathaibert/core.py | 55 +- pythainlp/soundex/__init__.py | 4 +- pythainlp/soundex/core.py | 10 +- pythainlp/soundex/lk82.py | 10 +- pythainlp/soundex/metasound.py | 10 +- pythainlp/soundex/prayut_and_somchaip.py | 16 +- pythainlp/soundex/sound.py | 45 +- pythainlp/soundex/udom83.py | 11 +- pythainlp/spell/__init__.py | 10 +- pythainlp/spell/core.py | 48 +- pythainlp/spell/phunspell.py | 13 +- pythainlp/spell/pn.py | 82 +- pythainlp/spell/symspellpy.py | 18 +- pythainlp/spell/tltk.py | 14 +- .../spell/wanchanberta_thai_grammarly.py | 90 +- pythainlp/spell/words_spelling_correction.py | 106 +- pythainlp/summarize/__init__.py | 4 +- pythainlp/summarize/core.py | 24 +- pythainlp/summarize/freq.py | 12 +- pythainlp/summarize/keybert.py | 43 +- pythainlp/summarize/mt5.py | 29 +- pythainlp/tag/__init__.py | 4 +- pythainlp/tag/_tag_perceptron.py | 57 +- pythainlp/tag/blackboard.py | 15 +- pythainlp/tag/chunk.py | 10 +- pythainlp/tag/crfchunk.py | 10 +- pythainlp/tag/locations.py | 13 +- pythainlp/tag/named_entity.py | 42 +- pythainlp/tag/orchid.py | 19 +- pythainlp/tag/perceptron.py | 12 +- pythainlp/tag/pos_tag.py | 56 +- pythainlp/tag/thai_nner.py | 5 +- pythainlp/tag/thainer.py | 20 +- pythainlp/tag/tltk.py | 16 +- pythainlp/tag/unigram.py | 17 +- pythainlp/tag/wangchanberta_onnx.py | 6 +- pythainlp/tokenize/__init__.py | 32 +- pythainlp/tokenize/_utils.py | 25 +- pythainlp/tokenize/attacut.py | 16 +- pythainlp/tokenize/budoux.py | 9 +- pythainlp/tokenize/core.py | 83 +- pythainlp/tokenize/crfcut.py | 29 +- pythainlp/tokenize/deepcut.py | 10 +- pythainlp/tokenize/etcc.py | 27 +- pythainlp/tokenize/han_solo.py | 9 +- pythainlp/tokenize/longest.py | 29 +- pythainlp/tokenize/multi_cut.py | 37 +- pythainlp/tokenize/nercut.py | 14 +- pythainlp/tokenize/newmm.py | 21 +- pythainlp/tokenize/nlpo3.py | 6 +- pythainlp/tokenize/oskut.py | 9 +- pythainlp/tokenize/pyicu.py | 16 +- pythainlp/tokenize/sefr_cut.py | 9 +- pythainlp/tokenize/ssg.py | 8 +- pythainlp/tokenize/tcc.py | 26 +- pythainlp/tokenize/tcc_p.py | 26 +- pythainlp/tokenize/thai2fit.py | 16 + pythainlp/tokenize/thaisumcut.py | 13 +- pythainlp/tokenize/tltk.py | 13 +- pythainlp/tokenize/wtsplit.py | 11 +- pythainlp/tools/__init__.py | 1 - pythainlp/tools/core.py | 7 +- pythainlp/tools/misspell.py | 19 +- pythainlp/tools/path.py | 18 +- pythainlp/translate/__init__.py | 4 +- pythainlp/translate/core.py | 31 +- pythainlp/translate/en_th.py | 30 +- pythainlp/translate/small100.py | 25 +- pythainlp/translate/th_fr.py | 13 +- pythainlp/translate/tokenization_small100.py | 140 +- pythainlp/translate/word2word_translate.py | 141 +- pythainlp/translate/zh_th.py | 19 +- pythainlp/transliterate/__init__.py | 4 +- pythainlp/transliterate/core.py | 24 +- pythainlp/transliterate/ipa.py | 11 +- pythainlp/transliterate/iso_11940.py | 10 +- pythainlp/transliterate/lookup.py | 19 +- pythainlp/transliterate/pyicu.py | 10 +- pythainlp/transliterate/royin.py | 87 +- pythainlp/transliterate/spoonerism.py | 6 +- pythainlp/transliterate/thai2rom.py | 20 +- pythainlp/transliterate/thai2rom_onnx.py | 16 +- pythainlp/transliterate/thaig2p.py | 22 +- pythainlp/transliterate/thaig2p_v2.py | 15 +- pythainlp/transliterate/tltk.py | 10 +- pythainlp/transliterate/umt5_thaig2p.py | 15 +- pythainlp/transliterate/w2p.py | 19 +- pythainlp/transliterate/wunsen.py | 33 +- pythainlp/ulmfit/__init__.py | 4 +- pythainlp/ulmfit/core.py | 30 +- pythainlp/ulmfit/preprocess.py | 53 +- pythainlp/ulmfit/tokenizer.py | 22 +- pythainlp/util/__init__.py | 12 +- pythainlp/util/abbreviation.py | 18 +- pythainlp/util/collate.py | 14 +- pythainlp/util/date.py | 77 +- pythainlp/util/digitconv.py | 25 +- pythainlp/util/emojiconv.py | 9 +- pythainlp/util/encoding.py | 11 +- pythainlp/util/keyboard.py | 15 +- pythainlp/util/keywords.py | 14 +- pythainlp/util/lcs.py | 7 +- pythainlp/util/morse.py | 9 +- pythainlp/util/normalize.py | 38 +- pythainlp/util/numtoword.py | 15 +- pythainlp/util/phoneme.py | 29 +- pythainlp/util/pronounce.py | 45 +- .../util/remove_trailing_repeat_consonants.py | 34 +- pythainlp/util/spell_words.py | 25 +- pythainlp/util/strftime.py | 29 +- pythainlp/util/syllable.py | 18 +- pythainlp/util/thai.py | 24 +- pythainlp/util/thai_lunar_date.py | 24 +- pythainlp/util/thaiwordcheck.py | 3 +- pythainlp/util/time.py | 36 +- pythainlp/util/trie.py | 27 +- pythainlp/util/wordtonum.py | 49 +- pythainlp/wangchanberta/__init__.py | 1 - pythainlp/wangchanberta/core.py | 25 +- pythainlp/word_vector/__init__.py | 5 +- pythainlp/word_vector/core.py | 59 +- pythainlp/wsd/__init__.py | 5 +- pythainlp/wsd/core.py | 16 +- requirements.txt | 5 - setup.cfg | 32 - setup.py | 221 --- tests/__init__.py | 3 +- tests/compact/__init__.py | 3 +- tests/compact/test_cli.py | 5 +- tests/compact/testc_util.py | 3 +- tests/core/__init__.py | 3 +- tests/core/test_corpus.py | 2 - tests/core/test_tokenize.py | 7 +- tests/core/test_util.py | 7 +- tests/extra/__init__.py | 3 +- tests/extra/testx_cli.py | 2 +- tests/extra/testx_coref.py | 2 - tests/extra/testx_spell.py | 2 +- tests/extra/testx_tokenize.py | 6 +- tests/extra/testx_util.py | 3 +- tox.ini | 6 +- 229 files changed, 3285 insertions(+), 3393 deletions(-) delete mode 100644 docker_requirements.txt create mode 100644 pythainlp/tokenize/thai2fit.py delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 6c390f24d..897d58552 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -21,7 +21,7 @@ jobs: python-version: "3.10" - name: Install build tools and doc build tools run: | - pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1" + pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" pip install boto smart_open sphinx sphinx-rtd-theme # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 22c99450a..92944d046 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,4 +26,4 @@ jobs: uses: astral-sh/ruff-action@v3 with: src: "./pythainlp" - args: check --verbose --line-length 79 --select C901 + args: check --fix --verbose --line-length 79 --select I,W,C901,W291,W293 diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index c5c3cdebe..5c73a28a4 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -26,8 +26,7 @@ jobs: SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True run: | python -m pip install --upgrade "pip<24.1" setuptools - python -m pip install -r https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/docker_requirements.txt - python -m pip install pythainlp[full] + python -m pip install pythainlp[testing,full] python -m nltk.downloader omw-1.4 - name: Test run: | diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 9c79a0a08..59683d0d6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -41,7 +41,7 @@ jobs: cache: "pip" - name: Install build tools run: | - pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1" + pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" pip install coverage coveralls # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 @@ -70,11 +70,11 @@ jobs: # If torch for the platform is not available in PyPI, use this command: # pip install "" # Get wheel URL from http://download.pytorch.org/whl/torch/ - - name: Install dependencies from docker_requirements.txt + - name: Install testing dependencies if: env.INSTALL_FULL_DEPS == 'true' env: SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True - run: pip install -r docker_requirements.txt + run: pip install ".[testing]" - name: Install PyThaiNLP + dependencies (minimum) if: matrix.python-version != env.PYTHON_VERSION_LATEST && matrix.python-version != env.PYTHON_VERSION_LATEST_2 run: pip install . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87d06b780..667401363 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ --- -SPDX-FileCopyrightText: 2025 PyThaiNLP Project +SPDX-FileCopyrightText: 2025-2026 PyThaiNLP Project SPDX-FileType: DOCUMENTATION SPDX-License-Identifier: CC0-1.0 --- @@ -108,44 +108,79 @@ Make sure the tests pass on GitHub Actions. See more in [tests/README.md](./tests/README.md) +## Installing and Building + +### Installing for Development + +Install PyThaiNLP in editable mode with core dependencies: + +```sh +pip install -e . +``` + +Install with optional dependency groups: + +```sh +# Install with compact set of dependencies (recommended for development) +pip install -e ".[compact]" + +# Install with full dependencies +pip install -e ".[full]" + +# Install with testing dependencies (pinned versions for reproducibility) +pip install -e ".[testing]" +``` + +See all available optional dependency groups in `pyproject.toml` under `[project.optional-dependencies]`. + +### Building Distribution Packages + +To build source distribution and wheel: + +```sh +python -m build +``` + +This will create distribution packages in the `dist/` directory. + ## Releasing - We use [semantic versioning](https://semver.org/): MAJOR.MINOR.PATCH, with development build suffix: MAJOR.MINOR.PATCH-devBUILD -- We use [`bumpversion`](https://github.com/c4urself/bump2version/#installation) to manage versioning. - - `bumpversion [major|minor|patch|release|build]` +- We use [`bump-my-version`](https://github.com/callowayproject/bump-my-version) to manage versioning. The configuration is in `pyproject.toml` under `[tool.bumpversion]`. + - `bump-my-version bump [major|minor|patch|release|build]` - Example: ```sh #current_version = 2.3.3-dev0 - bumpversion build + bump-my-version bump build #current_version = 2.3.3-dev1 - bumpversion build + bump-my-version bump build #current_version = 2.3.3-dev2 - bumpversion release + bump-my-version bump release #current_version = 2.3.3-beta0 - bumpversion release + bump-my-version bump release #current_version = 2.3.3 - bumpversion patch + bump-my-version bump patch #current_version = 2.3.6-dev0 - bumpversion minor + bump-my-version bump minor #current_version = 2.3.1-dev0 - bumpversion build + bump-my-version bump build #current_version = 2.3.1-dev1 - bumpversion major + bump-my-version bump major #current_version = 3.0.0-dev0 - bumpversion release + bump-my-version bump release #current_version = 3.0.0-beta0 - bumpversion release + bump-my-version bump release #current_version = 3.0.0 ``` diff --git a/Dockerfile b/Dockerfile index b26019182..bb8e74ff7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN if [ -f docker_requirements.txt ]; then pip install -r docker_requirements.txt; fi -RUN pip install -e .[full] && pip cache purge +# Install PyThaiNLP with testing dependencies (replaces docker_requirements.txt) +RUN pip install -e ".[testing]" && pip cache purge diff --git a/Makefile b/Makefile index 0f103632c..d0874e03f 100644 --- a/Makefile +++ b/Makefile @@ -50,25 +50,24 @@ lint: ## check style with flake8 flake8 pythainlp tests test: ## run tests quickly with the default Python - python setup.py test + python -m unittest discover test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source pythainlp setup.py test + coverage run --source pythainlp -m unittest discover coverage report -m coverage html $(BROWSER) htmlcov/index.html -release: clean ## package and upload a release - python setup.py sdist upload - python setup.py bdist_wheel upload +release: clean ## package and upload a release (deprecated - use GitHub Actions) + python -m build + python -m twine upload dist/* dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel + python -m build ls -l dist install: clean ## install the package to the active Python's site-packages - python setup.py install + python -m pip install . diff --git a/README.md b/README.md index 14d0ff4e7..f23a05c2f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install pythainlp ## Getting Started -- PyThaiNLP requires Python 3.7+. +- PyThaiNLP requires Python 3.9+. - Python 2.7 users can use PyThaiNLP 1.6. See [2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118) | [Upgrading from 1.7](https://pythainlp.org/docs/2.0/notes/pythainlp-1_7-2_0.html) | [Upgrading ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0) - [PyThaiNLP Get Started notebook](https://pythainlp.org/tutorials/notebooks/pythainlp_get_started.html) | [API document](https://pythainlp.org/docs) | [Tutorials](https://pythainlp.org/tutorials) - [Official website](https://pythainlp.org/) | [PyPI](https://pypi.org/project/pythainlp/) | [Facebook page](https://www.facebook.com/pythainlp/) @@ -81,17 +81,43 @@ Possible `extras`: - `full` (install everything) - `compact` (install a stable and small subset of dependencies) +- `abbreviation` (for Thai abbreviation support) - `attacut` (to support attacut, a fast and accurate tokenizer) - `benchmarks` (for [word tokenization benchmarking](tokenization-benchmark.md)) +- `budoux` (for BudouX text segmentation) +- `coreference_resolution` (for coreference resolution) +- `dependency_parsing` (for dependency parsing) +- `el` (for entity linking) +- `esupar` (for esupar parser support) +- `generate` (for text generation) - `icu` (for ICU, International Components for Unicode, support in transliteration and tokenization) - `ipa` (for IPA, International Phonetic Alphabet, support in transliteration) - `ml` (to support ULMFiT models for classification) +- `mt5` (for mT5 model support) +- `nlpo3` (for nlpo3 Thai word tokenizer) +- `onnx` (for ONNX model support) +- `oskut` (for OSKut Thai word tokenizer) +- `sefr_cut` (for SEFR CUT Thai word tokenizer) +- `spacy_thai` (for spaCy Thai language support) +- `spell` (for spelling correction) +- `ssg` (for sentence segmentation) +- `testing` (pinned versions for CI/CD reproducibility) +- `textaugment` (for text augmentation) +- `thai_nner` (for Thai named entity recognition) - `thai2fit` (for Thai word vector) - `thai2rom` (for machine-learnt romanization) +- `transformers_ud` (for Universal Dependencies with transformers) +- `translate` (for machine translation) +- `wangchanberta` (for WangchanBERTa model) +- `wangchanglm` (for WangchanGLM model) +- `word_approximation` (for word approximation) - `wordnet` (for Thai WordNet API) +- `wsd` (for word sense disambiguation) +- `wtp` (for Where's the Point text segmentation) +- `wunsen` (for Wunsen spell checker) -For dependency details, look at the `extras` variable in -[`setup.py`](https://github.com/PyThaiNLP/pythainlp/blob/dev/setup.py). +For dependency details, look at the `[project.optional-dependencies]` section in +[`pyproject.toml`](https://github.com/PyThaiNLP/pythainlp/blob/dev/pyproject.toml). ## Data Directory diff --git a/README_TH.md b/README_TH.md index 12a060221..452edf008 100644 --- a/README_TH.md +++ b/README_TH.md @@ -86,18 +86,44 @@ pip install pythainlp[extra1,extra2,...] - `full` (ติดตั้งทุกอย่าง) - `compact` (ติดตั้งไลบารีชุดเล็กที่ทดสอบแล้วว่าไม่ตีกันเองและติดตั้งได้ในทุกระบบปฏิบัติการ) +- `abbreviation` (สำหรับการย่อคำภาษาไทย) - `attacut` (เพื่อสนับสนุน attacut ซึ่งเป็นตัวตัดคำที่ทำงานได้รวดเร็วและมีประสิทธิภาพ) - `benchmarks` (สำหรับ [word tokenization benchmarking](tokenization-benchmark.md)) +- `budoux` (สำหรับการแบ่งข้อความด้วย BudouX) +- `coreference_resolution` (สำหรับการหาคำที่อ้างอิงถึงกัน) +- `dependency_parsing` (สำหรับการวิเคราะห์โครงสร้างประโยค) +- `el` (สำหรับการเชื่อมโยงเอนทิตี) +- `esupar` (สำหรับการรองรับ esupar parser) +- `generate` (สำหรับการสร้างข้อความ) - `icu` (สำหรับการรองรับ ICU หรือ International Components for Unicode ในการถอดเสียงเป็นอักษรและการตัดแบ่งคำ) - `ipa` (สำหรับการรองรับ IPA หรือ International Phonetic Alphabet ในการถอดเสียงเป็นอักษร) - `ml` (เพื่อให้สนับสนุนตัวแบบภาษา ULMFiT สำหรับการจำแนกข้อความ) +- `mt5` (สำหรับรองรับโมเดล mT5) +- `nlpo3` (สำหรับตัวตัดคำภาษาไทย nlpo3) +- `onnx` (สำหรับรองรับโมเดล ONNX) +- `oskut` (สำหรับตัวตัดคำภาษาไทย OSKut) +- `sefr_cut` (สำหรับตัวตัดคำภาษาไทย SEFR CUT) +- `spacy_thai` (สำหรับรองรับภาษาไทยใน spaCy) +- `spell` (สำหรับการแก้ไขคำสะกดผิด) +- `ssg` (สำหรับการแบ่งประโยค) +- `testing` (เวอร์ชันที่ปักหมุดสำหรับ CI/CD) +- `textaugment` (สำหรับการเพิ่มข้อมูลข้อความ) +- `thai_nner` (สำหรับการจดจำชื่อเฉพาะภาษาไทย) - `thai2fit` (สำหรับ Thai word vector) - `thai2rom` (สำหรับการถอดอักษรไทยเป็นอักษรโรมัน) +- `transformers_ud` (สำหรับ Universal Dependencies ด้วย transformers) +- `translate` (สำหรับการแปลภาษา) +- `wangchanberta` (สำหรับโมเดล WangchanBERTa) +- `wangchanglm` (สำหรับโมเดล WangchanGLM) +- `word_approximation` (สำหรับการประมาณคำ) - `wordnet` (สำหรับ Thai WordNet API) +- `wsd` (สำหรับการแก้ความกำกวมของความหมายคำ) +- `wtp` (สำหรับการแบ่งข้อความด้วย Where's the Point) +- `wunsen` (สำหรับตัวตรวจการสะกดคำ Wunsen) -สำหรับโมดูลที่ต้องการ สามารถดูรายละเอียดได้ที่ตัวแปร `extras` ใน [`setup.py`](https://github.com/PyThaiNLP/pythainlp/blob/dev/setup.py). +สำหรับโมดูลที่ต้องการ สามารถดูรายละเอียดได้ที่ส่วน `[project.optional-dependencies]` ใน [`pyproject.toml`](https://github.com/PyThaiNLP/pythainlp/blob/dev/pyproject.toml). ## Command-line @@ -117,7 +143,7 @@ thainlp help ## ผู้ใช้งาน Python 2 -- PyThaiNLP สนับสนุน Python 3.7 ขึ้นไป บางความสามารถ สามารถใช้งานกับ Python 3 รุ่นก่อนหน้าได้ แต่ไม่ได้มีการทดสอบว่าใช้งานได้หรือไม่ อ่านเพิ่มเติม [1.7 -> 2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118). +- PyThaiNLP สนับสนุน Python 3.9 ขึ้นไป บางความสามารถ สามารถใช้งานกับ Python 3 รุ่นก่อนหน้าได้ แต่ไม่ได้มีการทดสอบว่าใช้งานได้หรือไม่ อ่านเพิ่มเติม [1.7 -> 2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118). - [Upgrading from 1.7](https://pythainlp.org/docs/2.0/notes/pythainlp-1_7-2_0.html) - [Upgrade ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0) - ผู้ใช้งาน Python 2.7 สามารถใช้งาน PyThaiNLP 1.6 diff --git a/codemeta.json b/codemeta.json index 7df85191a..d7f0babc7 100644 --- a/codemeta.json +++ b/codemeta.json @@ -3,7 +3,7 @@ "@type": "SoftwareSourceCode", "name": "PyThaiNLP", "description": "Thai Natural Language Processing in Python", - "version": "5.1.0", + "version": "5.2.0", "author": [ { "@type": "Person", @@ -61,15 +61,15 @@ "issueTracker": "https://github.com/PyThaiNLP/pythainlp/issues", "url": "https://pythainlp.org/", "keywords": [ + "NLP", "natural language processing", - "Thai", - "Python", + "tokenization", "text processing", + "linguistics", + "localization", "computational linguistics", - "tokenization", - "word segmentation", - "NLP", - "Thai language", - "Thai NLP" + "ThaiNLP", + "Thai NLP", + "Thai language" ] } diff --git a/docker_requirements.txt b/docker_requirements.txt deleted file mode 100644 index 1b80ccfc0..000000000 --- a/docker_requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -PyYAML>=5.4.1,<6.0.2 -attacut==1.0.6 -bpemb>=0.3.6,<0.4 -deepcut==0.7.0.0 -emoji>=0.6.0,<1 -epitran==1.26.0 -esupar>=1.3.9,<2 -fairseq>=0.10.0,<0.13;python_version<"3.11" -fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11" -fastai>=1.0.61,<2 -fastcoref==2.1.6 -gensim>=4.3.3,<5 -khanaa>=0.1.1,<1 -nlpo3>=1.3.1 -nltk>=3.6.6,<4 -numpy>=1.26.0,<3 -pandas>=2.2.0,<3 -panphon==0.22.2 -phunspell==0.1.6 -pyicu>=2.15.2,<3 -python-crfsuite==0.9.12 -requests>=2.32.0,<2.33 -sacremoses==0.1.1 -sentence-transformers>=2.7.0,<3 -sentencepiece==0.2.1 -spacy_thai==0.7.8 -spacy==3.8.7,<4 -ssg==0.0.8 -symspellpy==6.9.0 -thai-nner==0.3 -tltk>=1.6.8,<2 -torch>=1.13.1,<3 -transformers==4.57.3 -ufal.chu-liu-edmonds==1.0.3 -wtpsplit==1.3.0 -wunsen==0.0.3 -word2word>=1.0.0,<2 -budoux==0.7.0 diff --git a/docs/api/spell.rst b/docs/api/spell.rst index ce2dd035d..b0345219d 100644 --- a/docs/api/spell.rst +++ b/docs/api/spell.rst @@ -48,9 +48,9 @@ The `NorvigSpellChecker` class is a fundamental component of the `pythainlp.spel DEFAULT_SPELL_CHECKER ~~~~~~~~~~~~~~~~~~~~~ .. autodata:: DEFAULT_SPELL_CHECKER - :annotation: = Default instance of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ + :annotation: = Default reference of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ -The `DEFAULT_SPELL_CHECKER` is an instance of the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. +The `DEFAULT_SPELL_CHECKER` is an reference to the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. References ---------- diff --git a/docs/conf.py b/docs/conf.py index cd5bf5be1..250bf5b86 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,8 +9,8 @@ import os import sys import traceback -from datetime import datetime -from datetime import date +from datetime import date, datetime + import pythainlp # -- Path setup -------------------------------------------------------------- @@ -34,7 +34,7 @@ # -- Get version information and date from Git ---------------------------- try: - from subprocess import check_output, STDOUT + from subprocess import STDOUT, check_output current_branch = ( os.environ["CURRENT_BRANCH"] @@ -69,7 +69,7 @@ # .decode() # .strip() # ) -except Exception as e: +except Exception: traceback.print_exc() release = pythainlp.__version__ # today = "" diff --git a/examples/khavee.py b/examples/khavee.py index 79e1a72be..1a1c0d766 100644 --- a/examples/khavee.py +++ b/examples/khavee.py @@ -2,13 +2,11 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Example of using KhaveeVerifier from pythainlp.khavee +"""Example of using KhaveeVerifier from pythainlp.khavee """ from pythainlp.khavee import KhaveeVerifier - kv = KhaveeVerifier() # การเช็คสระ @@ -64,7 +62,7 @@ เรื่องวิศวะเก่งกาจประหลาดใจ เรื่องฟิสิกส์ไร้ผู้ใดมาต่อไป นริศราอีฟเก่งกว่าใครเพื่อน คอยช่วยเตือนเรื่องงานคอยสั่งสอน อ่านตำราหาความรู้ไม่ละทอน เป็นคนดีศรีนครของจิตรลดา -ภัสนันท์นาคลออหรือมีมี่ เรื่องเกมเอ่อเก่งกาจไม่กังขา +ภัสนันท์นาคลออหรือมีมี่ เรื่องเกมเอ่อเก่งกาจไม่กังขา เกมอะไรก็เล่นได้ไม่ลดวา สุดฉลาดมากปัญญามาครบครัน""", k_type=8, ) diff --git a/notebooks/convert_thai2rom_to_onnx.ipynb b/notebooks/convert_thai2rom_to_onnx.ipynb index a6383c81a..7233b7799 100644 --- a/notebooks/convert_thai2rom_to_onnx.ipynb +++ b/notebooks/convert_thai2rom_to_onnx.ipynb @@ -75,6 +75,7 @@ "outputs": [], "source": [ "import torch\n", + "\n", "from pythainlp.corpus import get_corpus_path\n", "from pythainlp.transliterate.thai2rom import _MODEL_NAME\n", "\n", @@ -115,7 +116,6 @@ "outputs": [], "source": [ "import torch\n", - "import numpy as np\n", "\n", "input_tensor = torch.Tensor([[30, 19, 8, 30, 38, 37, 10, 3]]).long()\n", "\n", diff --git a/notebooks/create_words.ipynb b/notebooks/create_words.ipynb index d8d3ced83..14e848c65 100644 --- a/notebooks/create_words.ipynb +++ b/notebooks/create_words.ipynb @@ -6,8 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from pythainlp.transliterate import pronunciate\n", - "from pythainlp import thai_consonants" + "from pythainlp import thai_consonants\n", + "from pythainlp.transliterate import pronunciate" ] }, { diff --git a/notebooks/test_chat.ipynb b/notebooks/test_chat.ipynb index d3c64f3c1..d0d7a7d6b 100644 --- a/notebooks/test_chat.ipynb +++ b/notebooks/test_chat.ipynb @@ -9,8 +9,9 @@ }, "outputs": [], "source": [ - "from pythainlp.chat.core import ChatBotModel\n", - "import torch" + "import torch\n", + "\n", + "from pythainlp.chat.core import ChatBotModel" ] }, { diff --git a/notebooks/test_el.ipynb b/notebooks/test_el.ipynb index da6060583..3ca1cf8af 100644 --- a/notebooks/test_el.ipynb +++ b/notebooks/test_el.ipynb @@ -8,6 +8,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"" ] }, diff --git a/notebooks/test_wangchanglm.ipynb b/notebooks/test_wangchanglm.ipynb index 2235ad0a0..bb7f886e1 100644 --- a/notebooks/test_wangchanglm.ipynb +++ b/notebooks/test_wangchanglm.ipynb @@ -9,8 +9,9 @@ }, "outputs": [], "source": [ - "from pythainlp.generate.wangchanglm import WangChanGLM\n", - "import torch" + "import torch\n", + "\n", + "from pythainlp.generate.wangchanglm import WangChanGLM" ] }, { diff --git a/notebooks/test_wsd.ipynb b/notebooks/test_wsd.ipynb index 07ffbf589..5b283c929 100644 --- a/notebooks/test_wsd.ipynb +++ b/notebooks/test_wsd.ipynb @@ -80,7 +80,7 @@ }, "outputs": [], "source": [ - "from pythainlp.corpus import get_corpus_path, thai_wsd_dict" + "from pythainlp.corpus import thai_wsd_dict" ] }, { diff --git a/pyproject.toml b/pyproject.toml index d84632ba3..6a6f6f6da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,10 +2,294 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +[build-system] +requires = ["setuptools>=69.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pythainlp" +version = "5.2.0" +description = "Thai Natural Language Processing library" +readme = "README.md" +requires-python = ">=3.9" +license = "Apache-2.0" +license-files = ["LICENSE", "pythainlp/corpus/corpus_license.md"] +authors = [ + { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@pythainlp.org" }, + { name = "Korakot Chaovavanich" }, + { name = "Charin Polpanumas" }, + { name = "Arthit Suriyawongkul", email = "suriyawa@tcd.ie" }, + { name = "Lalita Lowphansirikul" }, + { name = "Pattarawat Chormai" }, + { name = "Peerat Limkonchotiwat" }, + { name = "Thanathip Suntorntip" }, + { name = "Can Udomcharoenchaikit" }, +] +maintainers = [ + { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@pythainlp.org" }, + { name = "Arthit Suriyawongkul", email = "suriyawa@tcd.ie" }, +] +keywords = [ + "pythainlp", + "NLP", + "natural language processing", + "tokenization", + "text processing", + "linguistics", + "localization", + "computational linguistics", + "ThaiNLP", + "Thai NLP", + "Thai language", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Intended Audience :: Developers", + "Natural Language :: Thai", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Localization", + "Topic :: Text Processing", + "Topic :: Text Processing :: General", + "Topic :: Text Processing :: Linguistic", +] + +# Core dependencies +dependencies = ["requests>=2.31", "tzdata; sys_platform == 'win32'"] + +[project.optional-dependencies] + +abbreviation = ["khamyo>=0.2.0"] + +attacut = ["attacut>=1.0.6"] + +benchmarks = ["numpy>=1.22", "pandas>=0.24", "PyYAML>=5.4.1"] + +budoux = ["budoux>=0.7.0"] + +coreference_resolution = ["fastcoref>=2.1.5", "spacy>=3.0"] + +dependency_parsing = [ + "spacy_thai>=0.7.1", + "transformers>=4.22.1", + "ufal.chu-liu-edmonds>=1.0.2", +] + +el = ["multiel>=0.5"] + +esupar = ["esupar>=1.3.8", "numpy>=1.22", "transformers>=4.22.1"] + +generate = ["fastai<2.0"] + +icu = ["pyicu>=2.3"] + +ipa = ["epitran>=1.1"] + +ml = ["numpy>=1.22", "torch>=1.0.0"] + +mt5 = ["sentencepiece>=0.1.91", "transformers>=4.22.1"] + +nlpo3 = ["nlpo3>=1.3.1"] + +onnx = ["numpy>=1.22", "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"] + +oskut = ["oskut>=1.3"] + +sefr_cut = ["sefr_cut>=1.1"] + +spacy_thai = ["spacy_thai>=0.7.1"] + +spell = ["phunspell>=0.1.6", "symspellpy>=6.7.6"] + +ssg = ["ssg>=0.0.8"] + +textaugment = ["bpemb>=0.3.2", "gensim>=4.0.0"] + +thai_nner = ["thai_nner"] + +thai2fit = ["emoji>=0.5.1", "gensim>=4.0.0", "numpy>=1.22"] + +thai2rom = ["numpy>=1.22", "torch>=1.0.0"] + +translate = [ + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "sacremoses>=0.0.41", + "sentencepiece>=0.1.91", + "torch>=1.0.0", + "transformers>=4.22.1", + "word2word>=1.0.0", +] + +transformers_ud = ["transformers>=4.22.1", "ufal.chu-liu-edmonds>=1.0.2"] + +wangchanberta = ["sentencepiece>=0.1.91", "transformers>=4.22.1"] + +wangchanglm = ["pandas>=0.24", "sentencepiece>=0.1.91", "transformers>=4.22.1"] + +word_approximation = ["panphon>=0.20.0"] + +wordnet = ["nltk>=3.3"] + +wsd = ["sentence-transformers>=2.2.2"] + +wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"] + +wunsen = ["wunsen>=0.0.3"] + +# Compact dependencies - safe small set of optional dependencies +compact = [ + "nlpo3>=1.3.1", + "numpy>=1.22", + "pyicu>=2.3", + "python-crfsuite>=0.9.7", + "PyYAML>=5.4.1", +] + +# Full dependencies - comprehensive set of all optional features +full = [ + "attacut>=1.0.6", + "bpemb>=0.3.2", + "budoux>=0.7.0", + "emoji>=0.5.1", + "epitran>=1.1", + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "fastai<2.0", + "fastcoref>=2.1.5", + "gensim>=4.0.0", + "khamyo>=0.2.0", + "nlpo3>=1.3.1", + "nltk>=3.3", + "numpy>=1.22", + "onnxruntime>=1.10.0", + "oskut>=1.3", + "pandas>=0.24", + "panphon>=0.20.0", + "phunspell>=0.1.6", + "pyicu>=2.3", + "sacremoses>=0.0.41", + "sefr_cut>=1.1", + "sentencepiece>=0.1.91", + "sentence-transformers>=2.2.2", + "spacy>=3.0", + "spacy_thai>=0.7.1", + "ssg>=0.0.8", + "symspellpy>=6.7.6", + "thai_nner", + "torch>=1.0.0", + "transformers>=4.22.1", + "ufal.chu-liu-edmonds>=1.0.2", + "word2word>=1.0.0", + "wtpsplit>=1.0.1", + "wunsen>=0.0.3", +] + +# Testing dependencies - pinned versions for CI/CD reproducibility +testing = [ + "attacut==1.0.6", + "bpemb>=0.3.6,<0.4", + "budoux==0.7.0", + "deepcut==0.7.0.0", + "emoji>=0.6.0,<1", + "epitran==1.26.0", + "esupar>=1.3.9,<2", + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "fastai>=1.0.61,<2", + "fastcoref==2.1.6", + "gensim>=4.3.3,<5", + "khanaa>=0.1.1,<1", + "nlpo3>=1.3.1", + "nltk>=3.6.6,<4", + "numpy>=1.26.0,<3", + "pandas>=2.2.0,<3", + "panphon==0.22.2", + "phunspell==0.1.6", + "pyicu>=2.15.2,<3", + "python-crfsuite==0.9.12", + "PyYAML>=5.4.1,<6.0.2", + "sacremoses==0.1.1", + "sentence-transformers>=2.7.0,<3", + "sentencepiece==0.2.1", + "spacy==3.8.7,<4", + "spacy_thai==0.7.8", + "ssg==0.0.8", + "symspellpy==6.9.0", + "thai-nner==0.3", + "tltk>=1.6.8,<2", + "torch>=1.13.1,<3", + "transformers==4.57.3", + "ufal.chu-liu-edmonds==1.0.3", + "word2word>=1.0.0,<2", + "wtpsplit==1.3.0", + "wunsen==0.0.3", +] + +[project.urls] +homepage = "https://pythainlp.org/" +source = "https://github.com/PyThaiNLP/pythainlp.git" +download = "https://pypi.org/project/pythainlp/#files" +changelog = "https://github.com/PyThaiNLP/pythainlp/blob/dev/CHANGELOG.md" +releasenotes = "https://github.com/PyThaiNLP/pythainlp/releases" +documentation = "https://pythainlp.org/docs/" +issues = "https://github.com/PyThaiNLP/pythainlp/issues" +"Tutorials" = "https://pythainlp.org/tutorials/" + +[project.scripts] +thainlp = "pythainlp.__main__:main" + +[tool.setuptools] +zip-safe = false +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["tests", "tests.*"] + +[tool.setuptools.package-data] +pythainlp = ["corpus/*"] + +# Bumpversion configuration +[tool.bumpversion] +current_version = "5.2.0" +commit = true +tag = true +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\-(?P[a-z]+)(?P\\d+))?" +serialize = [ + "{major}.{minor}.{patch}-{release}{build}", + "{major}.{minor}.{patch}", +] + +[[tool.bumpversion.files]] +filename = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "pythainlp/__init__.py" +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' + +[tool.bumpversion.parts.release] +optional_value = "prod" +first_value = "dev" +values = ["dev", "beta", "prod"] + +# Coverage configuration +[tool.coverage.run] +source = ["pythainlp"] + +# Ruff configuration [tool.ruff] line-length = 79 indent-width = 4 -target-version = "py310" +target-version = "py39" [tool.ruff.format] quote-style = "double" @@ -17,4 +301,4 @@ docstring-code-format = true [tool.ruff.lint.mccabe] # Flag errors (`C901`) whenever the complexity level exceeds 5. Default is 10. # We should aim to gradually reduce this to 10. -max-complexity = 40 +max-complexity = 38 diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index 7e0a61544..a41f1c374 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -1,64 +1,78 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -__version__ = "5.2.0" - -thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars - -thai_vowels = ( - "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37" - + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47" -) # 20 -thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 -thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4 -thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 -thai_below_vowels = "\u0e38\u0e39" # 2 - -thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4 - -# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: -# These signs can be part of a word -thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars - -# Any Thai character that can be part of a word -thai_letters = "".join( - [thai_consonants, thai_vowels, thai_tonemarks, thai_signs] -) # 74 - -# Fongman, Angkhankhu, Khomut: -# These characters are section markers -thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars - -thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10 -thai_symbols = "\u0e3f" # Thai Bath ฿ - -# All Thai characters that are presented in Unicode -thai_characters = "".join( - [thai_letters, thai_punctuations, thai_digits, thai_symbols] -) -# Thai pangram by Sungsit Sawaiwan -# CC BY-SA License -# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html -thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ -๏ จับฅอคนบั่นต้อง อาญา -ขุดฆ่าโคตรฃัตติยา ซ่านม้วย -ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ -สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ -๏ กตัญญูไป่พร้อม ปฐมฌาน -เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ -ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ -โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ -๑๒ กรกฎาคม ๒๕๕๘""" - -from pythainlp.soundex import soundex -from pythainlp.spell import correct, spell -from pythainlp.tag import pos_tag -from pythainlp.tokenize import ( - Tokenizer, - sent_tokenize, - subword_tokenize, - word_tokenize, -) -from pythainlp.transliterate import romanize, transliterate -from pythainlp.util import collate, thai_strftime +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +__version__ = "5.2.0" + +thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars + +thai_vowels = ( + "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37" + + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47" +) # 20 +thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 +thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4 +thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 +thai_below_vowels = "\u0e38\u0e39" # 2 + +thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4 + +# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: +# These signs can be part of a word +thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars + +# Any Thai character that can be part of a word +thai_letters = "".join( + [thai_consonants, thai_vowels, thai_tonemarks, thai_signs] +) # 74 + +# Fongman, Angkhankhu, Khomut: +# These characters are section markers +thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars + +thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10 +thai_symbols = "\u0e3f" # Thai Bath ฿ + +# All Thai characters that are presented in Unicode +thai_characters = "".join( + [thai_letters, thai_punctuations, thai_digits, thai_symbols] +) +# Thai pangram by Sungsit Sawaiwan +# CC BY-SA License +# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html +thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ +๏ จับฅอคนบั่นต้อง อาญา +ขุดฆ่าโคตรฃัตติยา ซ่านม้วย +ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ +สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ +๏ กตัญญูไป่พร้อม ปฐมฌาน +เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ +ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ +โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ +๑๒ กรกฎาคม ๒๕๕๘""" + +__all__ = [ + "collate", + "correct", + "pos_tag", + "romanize", + "spell", + "sent_tokenize", + "subword_tokenize", + "soundex", + "thai_strftime", + "transliterate", + "Tokenizer", + "word_tokenize", +] + +from pythainlp.soundex import soundex +from pythainlp.spell import correct, spell +from pythainlp.tag import pos_tag +from pythainlp.tokenize import ( + Tokenizer, + sent_tokenize, + subword_tokenize, + word_tokenize, +) +from pythainlp.transliterate import romanize, transliterate +from pythainlp.util import collate, thai_strftime diff --git a/pythainlp/__main__.py b/pythainlp/__main__.py index cb9d53a45..91a280366 100644 --- a/pythainlp/__main__.py +++ b/pythainlp/__main__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py index 4a30d29c6..23a0f50ab 100644 --- a/pythainlp/ancient/__init__.py +++ b/pythainlp/ancient/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Ancient versions of the Thai language +"""Ancient versions of the Thai language """ __all__ = ["aksonhan_to_current", "convert_currency"] diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 8769993fe..e9fef0cec 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_orst_words @@ -23,8 +23,7 @@ def aksonhan_to_current(word: str) -> str: - """ - Convert AksonHan words to current Thai words + """Convert AksonHan words to current Thai words AksonHan (อักษรหัน) writes down two consonants for the \ spelling of the /a/ vowels. (สระ อะ). diff --git a/pythainlp/ancient/currency.py b/pythainlp/ancient/currency.py index bdf685e96..24aa41ec9 100644 --- a/pythainlp/ancient/currency.py +++ b/pythainlp/ancient/currency.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + def convert_currency(value: float, from_unit: str) -> dict: - """ - Convert ancient Thai currency to other units + """Convert ancient Thai currency to other units * เบี้ย (Bia) * อัฐ (At) @@ -44,14 +44,14 @@ def convert_currency(value: float, from_unit: str) -> dict: # } """ conversion_factors_to_att = { - 'เบี้ย': 1, - 'อัฐ': 100, # 1 อัฐ = 100 เบี้ย - 'ไพ': 2 * 100, # 1 ไพ = 2 อัฐ - 'เฟื้อง': 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ - 'สลึง': 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง - 'บาท': 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง - 'ตำลึง': 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท - 'ชั่ง': 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง + "เบี้ย": 1, + "อัฐ": 100, # 1 อัฐ = 100 เบี้ย + "ไพ": 2 * 100, # 1 ไพ = 2 อัฐ + "เฟื้อง": 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ + "สลึง": 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง + "บาท": 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง + "ตำลึง": 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท + "ชั่ง": 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง } if from_unit not in conversion_factors_to_att: diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py index c82c17995..79c7de2b3 100644 --- a/pythainlp/augment/__init__.py +++ b/pythainlp/augment/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai text augment +"""Thai text augment """ __all__ = ["WordNetAug"] diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py index 1707936a4..8c4e0cea2 100644 --- a/pythainlp/augment/lm/__init__.py +++ b/pythainlp/augment/lm/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Language Models +"""Language Models """ __all__ = [ diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index a91af4b52..3621a7896 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import itertools -from typing import List, Tuple from gensim.models.fasttext import FastText as FastText_gensim from gensim.models.keyedvectors import KeyedVectors @@ -12,15 +12,13 @@ class FastTextAug: - """ - Text Augment from fastText + """Text Augment from fastText :param str model_path: path of model file """ def __init__(self, model_path: str): - """ - :param str model_path: path of model file + """:param str model_path: path of model file """ if model_path.endswith(".bin"): self.model = FastText_gensim.load_facebook_vectors(model_path) @@ -30,9 +28,8 @@ def __init__(self, model_path: str): self.model = FastText_gensim.load(model_path) self.dict_wv = list(self.model.key_to_index.keys()) - def tokenize(self, text: str) -> List[str]: - """ - Thai text tokenization for fastText + def tokenize(self, text: str) -> list[str]: + """Thai text tokenization for fastText :param str text: Thai text @@ -41,9 +38,8 @@ def tokenize(self, text: str) -> List[str]: """ return word_tokenize(text, engine="icu") - def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: - """ - :param str sent: text of sentence + def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: + """:param str sent: text of sentence :param float p: probability :rtype: List[List[str]] """ @@ -61,9 +57,8 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: - """ - Text Augment from fastText + ) -> list[tuple[str]]: + """Text Augment from fastText You may want to download the Thai model from https://fasttext.cc/docs/en/crawl-vectors.html. diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index fa7e9a8d3..a8cf8b4a0 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import random import re -from typing import List from pythainlp.phayathaibert.core import ThaiTextProcessor @@ -57,9 +56,8 @@ def generate( def augment( self, text: str, num_augs: int = 3, sample: bool = False - ) -> List[str]: - """ - Text augmentation from PhayaThaiBERT + ) -> list[str]: + """Text augmentation from PhayaThaiBERT :param str text: Thai text :param int num_augs: an amount of augmentation text needed as an output diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index 962e5f7cf..b6173bbf4 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - -from typing import List +from __future__ import annotations from transformers import ( CamembertTokenizer, @@ -52,9 +50,8 @@ def generate(self, sentence: str, num_replace_tokens: int = 3): masked_text = self.input_text return self.sent2 - def augment(self, sentence: str, num_replace_tokens: int = 3) -> List[str]: - """ - Text augmentation from WangchanBERTa + def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]: + """Text augmentation from WangchanBERTa :param str sentence: Thai sentence :param int num_replace_tokens: number replace tokens diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py index 08786229f..ff04e2cfc 100644 --- a/pythainlp/augment/word2vec/__init__.py +++ b/pythainlp/augment/word2vec/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Word2Vec +"""Word2Vec """ __all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"] diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index 635553f40..0f39f3597 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug class BPEmbAug: - """ - Thai Text Augment using word2vec from BPEmb + """Thai Text Augment using word2vec from BPEmb BPEmb: `github.com/bheinzerling/bpemb `_ @@ -22,16 +20,14 @@ def __init__(self, lang: str = "th", vs: int = 100000, dim: int = 300): self.model = self.bpemb_temp.emb self.load_w2v() - def tokenizer(self, text: str) -> List[str]: - """ - :param str text: Thai text + def tokenizer(self, text: str) -> list[str]: + """:param str text: Thai text :rtype: List[str] """ return self.bpemb_temp.encode(text) def load_w2v(self): - """ - Load BPEmb model + """Load BPEmb model """ self.aug = Word2VecAug( self.model, tokenize=self.tokenizer, type="model" @@ -39,9 +35,8 @@ def load_w2v(self): def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: - """ - Text Augment using word2vec from BPEmb + ) -> list[tuple[str]]: + """Text Augment using word2vec from BPEmb :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py index 345bf9097..8a6033f45 100644 --- a/pythainlp/augment/word2vec/core.py +++ b/pythainlp/augment/word2vec/core.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import itertools -from typing import List, Tuple class Word2VecAug: def __init__( self, model: str, tokenize: object, type: str = "file" ) -> None: - """ - :param str model: path of model + """:param str model: path of model :param object tokenize: tokenize function :param str type: model type (file, binary) """ @@ -28,9 +27,8 @@ def __init__( self.model = model self.dict_wv = list(self.model.key_to_index.keys()) - def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: - """ - :param str sent: text of sentence + def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: + """:param str sent: text of sentence :param float p: probability :rtype: List[List[str]] """ @@ -48,9 +46,8 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: - """ - :param str sentence: text of sentence + ) -> list[tuple[str]]: + """:param str sentence: text of sentence :param int n_sent: maximum number of synonymous sentences :param int p: probability diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 3ae90e24b..679a44600 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path @@ -10,8 +9,7 @@ class LTW2VAug: - """ - Text Augment using word2vec from LTW2V + """Text Augment using word2vec from LTW2V LTW2V: `github.com/PyThaiNLP/large-thaiword2vec `_ @@ -21,24 +19,21 @@ def __init__(self): self.ltw2v_wv = get_corpus_path("ltw2v") self.load_w2v() - def tokenizer(self, text: str) -> List[str]: - """ - :param str text: Thai text + def tokenizer(self, text: str) -> list[str]: + """:param str text: Thai text :rtype: List[str] """ return word_tokenize(text, engine="newmm") def load_w2v(self): # insert substitute - """ - Load LTW2V's word2vec model + """Load LTW2V's word2vec model """ self.aug = Word2VecAug(self.ltw2v_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: - """ - Text Augment using word2vec from Thai2Fit + ) -> list[tuple[str]]: + """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 6c7e3b4c0..6f0bd6af2 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -1,17 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class Thai2fitAug: - """ - Text Augment using word2vec from Thai2Fit + """Text Augment using word2vec from Thai2Fit Thai2Fit: `github.com/cstorm125/thai2fit `_ @@ -21,24 +19,22 @@ def __init__(self): self.thai2fit_wv = get_corpus_path("thai2fit_wv") self.load_w2v() - def tokenizer(self, text: str) -> List[str]: - """ - :param str text: Thai text + def tokenizer(self, text: str) -> list[str]: + """:param str text: Thai text :rtype: List[str] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + tok = thai2fit_tokenizer() + return tok.word_tokenize(text) def load_w2v(self): - """ - Load Thai2Fit's word2vec model + """Load Thai2Fit's word2vec model """ self.aug = Word2VecAug(self.thai2fit_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: - """ - Text Augment using word2vec from Thai2Fit + ) -> list[tuple[str]]: + """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index ea2ab3d70..1a45b80a3 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l """ -Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l -""" + +from __future__ import annotations + __all__ = [ "WordNetAug", "postype2wordnet", @@ -12,7 +13,6 @@ import itertools from collections import OrderedDict -from typing import List from nltk.corpus import wordnet as wn @@ -102,8 +102,7 @@ def postype2wordnet(pos: str, corpus: str): - """ - Convert part-of-speech type to wordnet type + """Convert part-of-speech type to wordnet type :param str pos: POS type :param str corpus: part-of-speech corpus @@ -117,21 +116,19 @@ def postype2wordnet(pos: str, corpus: str): class WordNetAug: - """ - Text Augment using wordnet + """Text Augment using wordnet """ def __init__(self): pass def find_synonyms( - self, word: str, pos: str = None, postag_corpus: str = "orchid" - ) -> List[str]: - """ - Find synonyms using wordnet + self, word: str, pos: str | None = None, postag_corpus: str = "orchid" + ) -> list[str]: + """Find synonyms using wordnet :param str word: word - :param str pos: part-of-speech type + :param str | None pos: part-of-speech type. Default is None. :param str postag_corpus: name of POS tag corpus :return: list of synonyms :rtype: List[str] @@ -162,9 +159,8 @@ def augment( max_syn_sent: int = 6, postag: bool = True, postag_corpus: str = "orchid", - ) -> List[List[str]]: - """ - Text Augment using wordnet + ) -> list[list[str]]: + """Text Augment using wordnet :param str sentence: Thai sentence :param object tokenize: function for tokenizing words diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index 66a012ffe..c63536681 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Performance benchmarking. +"""Performance benchmarking. """ __all__ = ["benchmark"] diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 35bd0ae6c..94e7598f3 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import re import sys -from typing import List, Tuple import numpy as np import pandas as pd @@ -18,18 +17,17 @@ ) # regex for removing repeated separators, i.e. |||| -MULTIPLE_SEPS_RX = re.compile("{sep}+".format(sep=re.escape(SEPARATOR))) +MULTIPLE_SEPS_RX = re.compile(f"{re.escape(SEPARATOR)}+") # regex for removing tags, i.e. , TAG_RX = re.compile(r"<\/?[A-Z]+>") # regex for removing trailing separators, i.e. a|dog| -> a|dog -TAILING_SEP_RX = re.compile("{sep}$".format(sep=re.escape(SEPARATOR))) +TAILING_SEP_RX = re.compile(f"{re.escape(SEPARATOR)}$") def _f1(precision: float, recall: float) -> float: - """ - Compute f1. + """Compute f1. :param float precision :param float recall @@ -43,8 +41,7 @@ def _f1(precision: float, recall: float) -> float: def _flatten_result(my_dict: dict, sep: str = ":") -> dict: - """ - Flatten two-dimension dictionary. + """Flatten two-dimension dictionary. Use keys in the first dimension as a prefix for keys in the second dimension. For example, @@ -68,9 +65,8 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: return dict(items) -def benchmark(ref_samples: List[str], samples: List[str]) -> pd.DataFrame: - """ - Performance benchmarking for samples. +def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: + """Performance benchmarking for samples. Please see :meth:`pythainlp.benchmarks.word_tokenization.compute_stats` for the computed metrics. @@ -113,8 +109,7 @@ def benchmark(ref_samples: List[str], samples: List[str]) -> pd.DataFrame: def preprocessing(txt: str, remove_space: bool = True) -> str: - """ - Clean up text before performing evaluation. + """Clean up text before performing evaluation. :param str text: text to be preprocessed :param bool remove_space: whether to remove white space @@ -137,8 +132,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: def compute_stats(ref_sample: str, raw_sample: str) -> dict: - """ - Compute statistics for tokenization quality + """Compute statistics for tokenization quality These statistics include: @@ -184,9 +178,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: correctly_tokenised_words = np.sum(tokenization_indicators) - tokenization_indicators = list( - map(str, tokenization_indicators) - ) + tokenization_indicators = list(map(str, tokenization_indicators)) return { "char_level": { @@ -207,8 +199,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: def _binary_representation(txt: str, verbose: bool = False): - """ - Transform text into {0, 1} sequence. + """Transform text into {0, 1} sequence. where (1) indicates that the corresponding character is the beginning of a word. For example, ผม|ไม่|ชอบ|กิน|ผัก -> 10100... @@ -240,8 +231,7 @@ def _binary_representation(txt: str, verbose: bool = False): def _find_word_boundaries(bin_reps) -> list: - """ - Find the starting and ending location of each word. + """Find the starting and ending location of each word. :param str bin_reps: binary representation of a text @@ -256,11 +246,10 @@ def _find_word_boundaries(bin_reps) -> list: def _find_words_correctly_tokenised( - ref_boundaries: List[Tuple[int, int]], - predicted_boundaries: List[Tuple[int, int]], -) -> Tuple[int]: - """ - Find whether each word is correctly tokenized. + ref_boundaries: list[tuple[int, int]], + predicted_boundaries: list[tuple[int, int]], +) -> tuple[int]: + """Find whether each word is correctly tokenized. :param list[tuple(int, int)] ref_boundaries: word boundaries of reference tokenization :param list[tuple(int, int)] predicted_boundaries: word boundareies of predicted tokenization diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py index 28698b252..8becbc93d 100644 --- a/pythainlp/chat/__init__.py +++ b/pythainlp/chat/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.chat +"""pythainlp.chat """ __all__ = ["ChatBotModel"] diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 5f198ec84..18e2735a4 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -1,20 +1,19 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import torch class ChatBotModel: def __init__(self): - """ - Chat using AI generation + """Chat using AI generation """ self.history = [] def reset_chat(self): - """ - Reset chat by cleaning history + """Reset chat by cleaning history """ self.history = [] @@ -28,8 +27,7 @@ def load_model( offload_folder: str = "./", low_cpu_mem_usage: bool = True, ): - """ - Load model + """Load model :param str model_name: Model name (Now, we support wangchanglm only) :param bool return_dict: return_dict @@ -56,8 +54,7 @@ def load_model( raise NotImplementedError(f"We doesn't support {model_name}.") def chat(self, text: str) -> str: - """ - Chatbot + """Chatbot :param str text: text for asking chatbot with. :return: answer from chatbot. @@ -69,7 +66,7 @@ def chat(self, text: str) -> str: import torch chatbot = ChatBotModel() - chatbot.load_model(device="cpu",torch_dtype=torch.bfloat16) + chatbot.load_model(device="cpu", torch_dtype=torch.bfloat16) print(chatbot.chat("สวัสดี")) # output: ยินดีที่ได้รู้จัก diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py index af282e47f..16c3f3151 100644 --- a/pythainlp/classify/__init__.py +++ b/pythainlp/classify/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.classify +"""pythainlp.classify """ __all__ = ["GzipModel"] diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index e2a7fbe57..7b21bea3b 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -1,27 +1,31 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import gzip import json -from typing import List, Tuple import numpy as np class GzipModel: - """ - This class is a re-implementation of - “Low-Resource” Text Classification: A Parameter-Free Classification Method with Compressors - (Jiang et al., Findings 2023) + """This class is a re-implementation of + “Low-Resource” Text Classification: A Parameter-Free Classification Method + with Compressors (Jiang et al., Findings 2023) - :param list training_data: list [(text_sample,label)] - :param str model_path: Path for loading model (if you saved the model) + :param list | None training_data: list [(text_sample,label)]. + Default is None. + :param str model_path: Path for loading model (if you saved the model). + Default is empty string. """ - def __init__(self, training_data: List[Tuple[str, str]] = None, model_path: str = None): - if model_path is not None: + def __init__( + self, + training_data: list[tuple[str, str]] | None = None, + model_path: str = "", + ): + if model_path: self.load(model_path) else: self.training_data = np.array(training_data) @@ -36,8 +40,7 @@ def train(self): return Cx2_list def predict(self, x1: str, k: int = 1) -> str: - """ - :param str x1: the text that we want to predict label for. + """:param str x1: the text that we want to predict label for. :param str k: k :return: label :rtype: str @@ -47,7 +50,7 @@ def predict(self, x1: str, k: int = 1) -> str: from pythainlp.classify import GzipModel - training_data = [ + training_data = [ ("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"), ("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"), ("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"), @@ -56,7 +59,7 @@ def predict(self, x1: str, k: int = 1) -> str: ("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"), ("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"), ("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"), - ("นี่เป็นบทความหนึ่ง", "Neutral") + ("นี่เป็นบทความหนึ่ง", "Neutral"), ] model = GzipModel(training_data) print(model.predict("ฉันดีใจ", k=1)) @@ -81,17 +84,22 @@ def predict(self, x1: str, k: int = 1) -> str: return predict_class def save(self, path: str): + """:param str path: path to save model """ - :param str path: path for save model - """ - with open(path, "w") as f: - json.dump({ - "training_data": self.training_data.tolist(), - "Cx2_list": self.Cx2_list - }, f, ensure_ascii=False) + with open(path, "w", encoding="utf-8") as f: + json.dump( + { + "training_data": self.training_data.tolist(), + "Cx2_list": self.Cx2_list, + }, + f, + ensure_ascii=False, + ) def load(self, path: str): - with open(path, "r") as f: + """:param str path: path to load model + """ + with open(path, "r", encoding="utf-8") as f: data = json.load(f) self.Cx2_list = data["Cx2_list"] self.training_data = np.array(data["training_data"]) diff --git a/pythainlp/cli/__init__.py b/pythainlp/cli/__init__.py index 8d6649e6a..2bf309c52 100644 --- a/pythainlp/cli/__init__.py +++ b/pythainlp/cli/__init__.py @@ -1,19 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """Command line helpers.""" +from __future__ import annotations + import io import sys from argparse import ArgumentError, ArgumentParser -from pythainlp.cli import data, tokenize, soundex, tag, benchmark, misspell + +from pythainlp.cli import benchmark, data, misspell, soundex, tag, tokenize sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8") # a command should start with a verb when possible -COMMANDS = sorted(["data", "soundex", "tag", "tokenize", "benchmark", "misspell"]) +COMMANDS = sorted( + ["data", "soundex", "tag", "tokenize", "benchmark", "misspell"] +) CLI_NAME = "thainlp" @@ -37,6 +41,7 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None: parser.print_help() raise ArgumentError(None, "No command provided.") + if __name__ == "__main__": # Create a simple mapping from command name to the imported module COMMAND_MAP = { @@ -54,6 +59,9 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None: COMMAND_MAP[command].run() else: if len(sys.argv) < 2: - print(f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}", file=sys.stderr) + print( + f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}", + file=sys.stderr, + ) else: - print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr) \ No newline at end of file + print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr) diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index b1b2cb7c4..e342ff9fd 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -8,15 +7,12 @@ import json import os -import yaml - from pythainlp import cli -from pythainlp.benchmarks import word_tokenization from pythainlp.tools import safe_print def _read_file(path): - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: lines = map(lambda r: r.strip(), f.readlines()) return list(lines) @@ -79,15 +75,24 @@ def __init__(self, name, argv): actual = _read_file(args.input_file) expected = _read_file(args.test_file) - assert len(actual) == len( - expected - ), "Input and test files do not have the same number of samples" + assert len(actual) == len(expected), ( + "Input and test files do not have the same number of samples" + ) safe_print( "Benchmarking %s against %s with %d samples in total" % (args.input_file, args.test_file, len(actual)) ) + try: + import yaml + + from pythainlp.benchmarks import word_tokenization + except ImportError: + raise ImportError( + "Please install the extra dependencies `benchmarks` to use this command by running `pip install pythainlp[benchmarks]`" + ) + df_raw = word_tokenization.benchmark(expected, actual) columns = [ diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index 0c5c704ed..3004f4852 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's dataset/corpus management. +"""Command line for PyThaiNLP's dataset/corpus management. """ import argparse diff --git a/pythainlp/cli/misspell.py b/pythainlp/cli/misspell.py index 11077282a..66b4601e4 100644 --- a/pythainlp/cli/misspell.py +++ b/pythainlp/cli/misspell.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -54,7 +53,7 @@ def __init__(self, argv): if args.seed is not None: random.seed(args.seed) - with open(args.file, "r", encoding="utf-8") as f: + with open(args.file, encoding="utf-8") as f: lines = f.readlines() misspelled_lines = [ diff --git a/pythainlp/cli/soundex.py b/pythainlp/cli/soundex.py index 60b0a1808..fc1ee9fd6 100644 --- a/pythainlp/cli/soundex.py +++ b/pythainlp/cli/soundex.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's soundex. +"""Command line for PyThaiNLP's soundex. It takes input text from the command line. """ diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index 6d4632cd0..f6e9c8df3 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's taggers. +"""Command line for PyThaiNLP's taggers. """ import argparse diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 7a12914d2..066414f53 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's tokenizers. +"""Command line for PyThaiNLP's tokenizers. """ import argparse diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py index 24c11cdcd..9aa47e950 100644 --- a/pythainlp/coref/__init__.py +++ b/pythainlp/coref/__init__.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""PyThaiNLP Coreference Resolution """ -PyThaiNLP Coreference Resolution -""" + __all__ = ["coreference_resolution"] from pythainlp.coref.core import coreference_resolution diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index 2b996d42a..32c537b3b 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations import spacy @@ -30,7 +29,7 @@ def _to_json(self, _predict): "clusters": _predict.get_clusters(as_strings=False), } - def predict(self, texts: List[str]) -> List[dict]: + def predict(self, texts: list[str]) -> list[dict]: return [ self._to_json(pred) for pred in self.model.predict(texts=texts) ] diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 6dcb3ec75..be6764966 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -1,17 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations _MODEL = None def coreference_resolution( - texts: List[str], model_name: str = "han-coref-v1.0", device: str = "cpu" + texts: list[str], model_name: str = "han-coref-v1.0", device: str = "cpu" ): - """ - Coreference Resolution + """Coreference Resolution :param List[str] texts: list of texts to apply coreference resolution to :param str model_name: coreference resolution model diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py index 45f0e3c62..4fd6d2f6e 100644 --- a/pythainlp/coref/han_coref.py +++ b/pythainlp/coref/han_coref.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import spacy from pythainlp.coref._fastcoref import FastCoref diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 9c73a801f..db112cc4f 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Corpus related functions. +"""Corpus related functions. Access to dictionaries, word lists, and language models. Including download manager. """ +from __future__ import annotations + __all__ = [ "corpus_db_path", "corpus_db_url", @@ -72,26 +72,24 @@ def corpus_path() -> str: - """ - Get path where corpus files are kept locally. + """Get path where corpus files are kept locally. """ return _CORPUS_PATH def corpus_db_url() -> str: - """ - Get remote URL of corpus catalog. + """Get remote URL of corpus catalog. """ return _CORPUS_DB_URL def corpus_db_path() -> str: - """ - Get local path of corpus catalog. + """Get local path of corpus catalog. """ return _CORPUS_DB_PATH - +# DO NOT REORDER these pythainlp.corpus imports. +# These imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.core import ( download, get_corpus, @@ -100,12 +98,12 @@ def corpus_db_path() -> str: get_corpus_db_detail, get_corpus_default_db, get_corpus_path, + get_hf_hub, get_path_folder_corpus, make_safe_directory_name, - get_hf_hub, path_pythainlp_corpus, remove, -) # these imports must come before other pythainlp.corpus.* imports +) from pythainlp.corpus.common import ( countries, find_synonyms, diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 09d4ee9b1..fe63c43a8 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Common lists of words. """ -Common lists of words. -""" + +from __future__ import annotations import ast @@ -18,7 +18,6 @@ "thai_male_names", "thai_negations", "thai_dict", - "thai_profanity_words", "thai_stopwords", "thai_syllables", "thai_synonym", @@ -27,7 +26,6 @@ "thai_wsd_dict", ] -from typing import Union from pythainlp.corpus import get_corpus, get_corpus_as_is, get_corpus_path from pythainlp.tools import warn_deprecation @@ -69,8 +67,7 @@ def countries() -> frozenset[str]: - """ - Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", + """Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", "แอลจีเรีย", and "ลาว". \n(See: `dev/pythainlp/corpus/countries_th.txt\ `_) @@ -85,9 +82,8 @@ def countries() -> frozenset[str]: return _THAI_COUNTRIES -def provinces(details: bool = False) -> Union[frozenset[str], list[dict]]: - """ - Return a frozenset of Thailand province names in Thai such as "กระบี่", +def provinces(details: bool = False) -> frozenset[str] | list[dict]: + """Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". \n(See: `dev/pythainlp/corpus/thailand_provinces_th.txt\ `_) @@ -129,8 +125,7 @@ def provinces(details: bool = False) -> Union[frozenset[str], list[dict]]: def thai_syllables() -> frozenset[str]: - """ - Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", + """Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", "โมน", "โม่ง", "กา", "ก่า", and, "ก้า". \n(See: `dev/pythainlp/corpus/syllables_th.txt\ `_) @@ -147,8 +142,7 @@ def thai_syllables() -> frozenset[str]: def thai_words() -> frozenset[str]: - """ - Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", + """Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", and "พิษภัย". \n(See: `dev/pythainlp/corpus/words_th.txt\ `_) @@ -163,8 +157,7 @@ def thai_words() -> frozenset[str]: def thai_orst_words() -> frozenset[str]: - """ - Return a frozenset of Thai words from Royal Society of Thailand + """Return a frozenset of Thai words from Royal Society of Thailand \n(See: `dev/pythainlp/corpus/thai_orst_words.txt\ `_) @@ -179,8 +172,7 @@ def thai_orst_words() -> frozenset[str]: def thai_stopwords() -> frozenset[str]: - """ - Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", + """Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", "การ", and "ประการหนึ่ง". \n(See: `dev/pythainlp/corpus/stopwords_th.txt\ `_) We use stopword lists by thesis's เพ็ญศิริ ลี้ตระกูล. @@ -202,8 +194,7 @@ def thai_stopwords() -> frozenset[str]: def thai_negations() -> frozenset[str]: - """ - Return a frozenset of Thai negation words including "ไม่" and "แต่". + """Return a frozenset of Thai negation words including "ไม่" and "แต่". \n(See: `dev/pythainlp/corpus/negations_th.txt\ `_) @@ -218,8 +209,7 @@ def thai_negations() -> frozenset[str]: def thai_profanity_words() -> frozenset[str]: - """ - Return a frozenset of Thai profanity words for content filtering. + """Return a frozenset of Thai profanity words for content filtering. \n(See: `dev/pythainlp/corpus/profanity_th.txt\ `_) @@ -234,8 +224,7 @@ def thai_profanity_words() -> frozenset[str]: def thai_family_names() -> frozenset[str]: - """ - Return a frozenset of Thai family names + """Return a frozenset of Thai family names \n(See: `dev/pythainlp/corpus/family_names_th.txt\ `_) @@ -250,8 +239,7 @@ def thai_family_names() -> frozenset[str]: def thai_female_names() -> frozenset[str]: - """ - Return a frozenset of Thai female names + """Return a frozenset of Thai female names \n(See: `dev/pythainlp/corpus/person_names_female_th.txt\ `_) @@ -266,8 +254,7 @@ def thai_female_names() -> frozenset[str]: def thai_male_names() -> frozenset[str]: - """ - Return a frozenset of Thai male names + """Return a frozenset of Thai male names \n(See: `dev/pythainlp/corpus/person_names_male_th.txt\ `_) @@ -282,8 +269,7 @@ def thai_male_names() -> frozenset[str]: def thai_dict() -> dict: - """ - Return Thai dictionary with definition from wiktionary. + """Return Thai dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -312,8 +298,7 @@ def thai_dict() -> dict: def thai_wsd_dict() -> dict: - """ - Return Thai Word Sense Disambiguation dictionary with definition from wiktionary. + """Return Thai Word Sense Disambiguation dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -340,8 +325,7 @@ def thai_wsd_dict() -> dict: def thai_synonyms() -> dict: - """ - Return Thai synonyms. + """Return Thai synonyms. \n(See: `thai_synonym\ `_) @@ -381,12 +365,11 @@ def thai_synonym() -> dict: def find_synonyms(word: str) -> list[str]: - """ - Find synonyms + """Find synonyms :param str word: Thai word :return: List of synonyms of the input word or an empty list if it isn't exist. - :rtype: list[str] + :rtype: List[str] :Example: :: diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index d3372bfe2..f509a0f02 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Corpus related functions. """ -Corpus related functions. -""" + +from __future__ import annotations import json import os import re -from typing import Union from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path @@ -19,8 +18,7 @@ def get_corpus_db(url: str): - """ - Get corpus catalog from server. + """Get corpus catalog from server. :param str url: URL corpus catalog """ @@ -38,14 +36,13 @@ def get_corpus_db(url: str): def get_corpus_db_detail(name: str, version: str = "") -> dict: - """ - Get details about a corpus, using information from local catalog. + """Get details about a corpus, using information from local catalog. :param str name: name of corpus :return: details about corpus :rtype: dict """ - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: local_db = json.load(f) if not version: @@ -61,8 +58,7 @@ def get_corpus_db_detail(name: str, version: str = "") -> dict: def path_pythainlp_corpus(filename: str) -> str: - """ - Get path pythainlp.corpus data + """Get path pythainlp.corpus data :param str filename: filename of the corpus to be read @@ -73,8 +69,7 @@ def path_pythainlp_corpus(filename: str) -> str: def get_corpus(filename: str, comments: bool = True) -> frozenset: - """ - Read corpus data from file and return a frozenset. + """Read corpus data from file and return a frozenset. Each line in the file will be a member of the set. @@ -133,7 +128,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: """ path = path_pythainlp_corpus(filename) lines = [] - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = fh.read().splitlines() if not comments: @@ -144,8 +139,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: def get_corpus_as_is(filename: str) -> list: - """ - Read corpus data from file, as it is, and return a list. + """Read corpus data from file, as it is, and return a list. Each line in the file will be a member of the list. @@ -173,15 +167,14 @@ def get_corpus_as_is(filename: str) -> list: """ path = path_pythainlp_corpus(filename) lines = [] - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = fh.read().splitlines() return lines -def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]: - """ - Get model path from default_db.json +def get_corpus_default_db(name: str, version: str = "") -> str | None: + """Get model path from default_db.json :param str name: corpus name :return: path to the corpus or **None** if the corpus doesn't \ @@ -211,9 +204,8 @@ def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]: def get_corpus_path( name: str, version: str = "", force: bool = False -) -> Union[str, None]: - """ - Get corpus path. +) -> str | None: + """Get corpus path. :param str name: corpus name :param str version: version @@ -252,9 +244,7 @@ def get_corpus_path( print(get_corpus_path('wiki_lm_lstm')) # output: /root/pythainlp-data/thwiki_model_lstm.pth """ - from typing import Dict - - CUSTOMIZE: Dict[str, str] = { + CUSTOMIZE: dict[str, str] = { # "the corpus name":"path" } if name in list(CUSTOMIZE): @@ -287,8 +277,7 @@ def get_corpus_path( def _download(url: str, dst: str) -> int: - """ - Download helper. + """Download helper. @param: URL for downloading file @param: dst place to put the file into @@ -323,8 +312,7 @@ def _download(url: str, dst: str) -> int: def _check_hash(dst: str, md5: str) -> None: - """ - Check hash helper. + """Check hash helper. @param: dst place to put the file into @param: md5 place to file hash (MD5) @@ -341,8 +329,7 @@ def _check_hash(dst: str, md5: str) -> None: def _version2int(v: str) -> int: - """ - X.X.X => X0X0X + """X.X.X => X0X0X """ if "-" in v: v = v.split("-")[0] @@ -406,8 +393,7 @@ def _check_version(cause: str) -> bool: def download( name: str, force: bool = False, url: str = "", version: str = "" ) -> bool: - """ - Download corpus. + """Download corpus. The available corpus names can be seen in this file: https://pythainlp.org/pythainlp-corpus/db.json @@ -450,7 +436,7 @@ def download( # check if corpus is available if name in corpus_db: - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: local_db = json.load(f) corpus = corpus_db[name] @@ -525,9 +511,7 @@ def download( # This awkward behavior is for backward-compatibility with # database files generated previously using TinyDB if local_db["_default"]: - corpus_no = ( - max((int(no) for no in local_db["_default"])) + 1 - ) + corpus_no = max(int(no) for no in local_db["_default"]) + 1 else: corpus_no = 1 local_db["_default"][str(corpus_no)] = { @@ -561,8 +545,7 @@ def download( def remove(name: str) -> bool: - """ - Remove corpus + """Remove corpus :param str name: corpus name :return: **True** if the corpus is found and successfully removed. @@ -588,7 +571,7 @@ def remove(name: str) -> bool: if _CHECK_MODE == "1": print("PyThaiNLP is read-only mode. It can't download.") return False - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: db = json.load(f) data = [ corpus for corpus in db["_default"].values() if corpus["name"] == name @@ -617,31 +600,53 @@ def get_path_folder_corpus(name, version, *path): return os.path.join(get_corpus_path(name, version), *path) -def make_safe_directory_name(name:str) -> str: - """ - Make safe directory name +def make_safe_directory_name(name: str) -> str: + """Make safe directory name :param str name: directory name :return: safe directory name :rtype: str """ # Replace invalid characters with an underscore - safe_name = re.sub(r'[<>:"/\\|?*]', '_', name) + safe_name = re.sub(r'[<>:"/\\|?*]', "_", name) # Remove leading/trailing spaces or periods (especially important for Windows) - safe_name = safe_name.strip(' .') + safe_name = safe_name.strip(" .") # Prevent names that are reserved on Windows - reserved_names = ['CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'] + reserved_names = [ + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", + ] if safe_name.upper() in reserved_names: - safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict + safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict return safe_name -def get_hf_hub(repo_id:str, filename: str=None) -> str: - """ - HuggingFace Hub in :mod:`pythainlp` data directory. +def get_hf_hub(repo_id: str, filename: str = "") -> str: + """HuggingFace Hub in :mod:`pythainlp` data directory. :param str repo_id: repo_id - :param str filename: filename + :param str filename: filename (optional, default is empty string). + If empty, downloads entire snapshot. :return: path :rtype: str """ @@ -653,19 +658,16 @@ def get_hf_hub(repo_id:str, filename: str=None) -> str: Please installing the package via 'pip install huggingface-hub'. """) except Exception as e: - raise Exception(f"An unexpected error occurred: {e}") + raise RuntimeError(f"An unexpected error occurred: {e}") from e hf_root = get_full_data_path("hf_models") name_dir = make_safe_directory_name(repo_id) root_project = os.path.join(hf_root, name_dir) - if filename!=None: + if filename: output_path = hf_hub_download( - repo_id=repo_id, - filename=filename, - local_dir=root_project + repo_id=repo_id, filename=filename, local_dir=root_project ) else: output_path = snapshot_download( - repo_id=repo_id, - local_dir=root_project + repo_id=repo_id, local_dir=root_project ) return output_path diff --git a/pythainlp/corpus/corpus_license.md b/pythainlp/corpus/corpus_license.md index 88c8f15f6..70b72c335 100644 --- a/pythainlp/corpus/corpus_license.md +++ b/pythainlp/corpus/corpus_license.md @@ -4,12 +4,11 @@ - Language models created by PyThaiNLP project are released under [Creative Commons Attribution 4.0 International Public License](https://creativecommons.org/licenses/by/4.0/) (CC-by). - For more information about corpora that PyThaiNLP use, see [https://github.com/PyThaiNLP/pythainlp-corpus/](https://github.com/PyThaiNLP/pythainlp-corpus/). - ## Dictionaries and Word Lists The following word lists are created by the PyThaiNLP project and released under **Creative Commons Zero 1.0 Universal Public Domain Dedication License** -https://creativecommons.org/publicdomain/zero/1.0/ + | Filename | Description | | ---------------------------- | ------------------------------------------------------ | @@ -25,10 +24,10 @@ https://creativecommons.org/publicdomain/zero/1.0/ | words_th_thai2fit_201810.txt | List of Thai words (frozen for thai2fit) | The following word lists are from **Thai Male and Female Names Corpus** -https://github.com/korkeatw/thai-names-corpus/ by Korkeat Wannapat + by Korkeat Wannapat and released under their original licenses which are **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + | Filename | Description | | -------------------------- | -------------------------------- | @@ -36,37 +35,34 @@ https://creativecommons.org/licenses/by-sa/4.0/ | person_names_female_th.txt | List of female names in Thailand | | person_names_male_th.txt | List of male names in Thailand | - ## Models The following language models are created by the PyThaiNLP project and released under **Creative Commons Attribution 4.0 International Public License** -https://creativecommons.org/licenses/by/4.0/ + -| Filename | Description | -| ------------------------- | ----------------------------------------------------------------------------------------------------- | -| pos_orchid_perceptron.json | Part-of-speech tagging model, trained from ORCHID data, using perceptron | -| pos_orchid_unigram.json | Part-of-speech tagging model, trained from ORCHID data, using unigram | +| Filename | Description | +| ------------------------------- | ----------------------------------------------------------------------------------------------------- | +| pos_orchid_perceptron.json | Part-of-speech tagging model, trained from ORCHID data, using perceptron | +| pos_orchid_unigram.json | Part-of-speech tagging model, trained from ORCHID data, using unigram | | pos_ud_perceptron-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using perceptron | -| pos_ud_unigram-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram | -| sentenceseg_crfcut.model | Sentence segmentation model, trained from TED subtitles, using CRF | -| tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron | -| tdtb-unigram_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram | -| pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron | -| pos_tud_unigram.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram | - +| pos_ud_unigram-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram | +| sentenceseg_crfcut.model | Sentence segmentation model, trained from TED subtitles, using CRF | +| tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron | +| tdtb-unigram_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram | +| pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron | +| pos_tud_unigram.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram | ## Thai Dictionary for ICU BreakIterator A Thai word list from ICU (International Components for Unicode) project (icubrk_th.txt) is copyrighted by Unicode, Inc. and others., released under **Unicode License Agreement - Data Files and Software (2016)** -http://www.unicode.org/copyright.html + Original data: -https://github.com/unicode-org/icu/blob/main/icu4c/source/data/brkitr/dictionaries/thaidict.txt - + ## Thai WordNet @@ -74,7 +70,7 @@ Thai WordNet (wordnet_th.db) is created by Thai Computational Linguistic Laboratory at National Institute of Information and Communications Technology (NICT), Japan, and released under the following license: -``` +```text Copyright: 2011 NICT Thai WordNet @@ -116,8 +112,7 @@ For more information about Thai WordNet, see S. Thoongsup et al., ‘Thai WordNet construction’, in Proceedings of the 7th Workshop on Asian Language Resources, Suntec, Singapore, Aug. 2009, pp. 139–144. -https://www.aclweb.org/anthology/W09-3420.pdf - + ## Thai Wikipedia Titles @@ -125,14 +120,13 @@ Thai Wikipedia titles corpus (wikipedia_titles.txt), prepared by konbraphat51, using a Thai Wikipedia dump from 21 November 2023, and released under their original license which is **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + Original data: -https://dumps.wikimedia.org/thwiki/latest/thwiki-latest-all-titles.gz + Preparation code: -https://github.com/konbraphat51/Thai_Dictionary_Cleaner/ - + ## Volubilis @@ -141,10 +135,10 @@ A corpus of Thai words registered in Volubilis dictionary using data from Volubilis 23.1 (Mar. 2023) by Francis Bastien, and released under their original license which is **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + Original data: -https://belisan-volubilis.blogspot.com/ + Preparation code: -https://github.com/konbraphat51/Thai_Dictionary_Cleaner/ + diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py index 3854c598f..400be2863 100644 --- a/pythainlp/corpus/icu.py +++ b/pythainlp/corpus/icu.py @@ -1,26 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Provides an optional word list from International Components for Unicode (ICU) dictionary. """ -Provides an optional word list from International Components for Unicode (ICU) dictionary. -""" -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus _THAI_ICU_FILENAME = "icubrk_th.txt" -def thai_icu_words() -> FrozenSet[str]: - """ - Return a frozenset of words from the Thai dictionary for BreakIterator of the +def thai_icu_words() -> frozenset[str]: + """Return a frozenset of words from the Thai dictionary for BreakIterator of the International Components for Unicode (ICU). :return: :class:`frozenset` containing Thai words. :rtype: :class:`frozenset` """ - _WORDS = get_corpus(_THAI_ICU_FILENAME, comments=False) return _WORDS diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index 6e5b5ba1d..193af0e2c 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -1,27 +1,25 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai unigram word frequency from OSCAR Corpus (words tokenized using ICU) +"""Thai unigram word frequency from OSCAR Corpus (words tokenized using ICU) Credit: Korakot Chaovavanich https://web.facebook.com/groups/colab.thailand/permalink/1524070061101680/ """ +from __future__ import annotations + __all__ = ["word_freqs", "unigram_word_freqs"] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus_path _OSCAR_FILENAME = "oscar_icu" -def word_freqs() -> List[Tuple[str, int]]: - """ - Get word frequency from OSCAR Corpus (words tokenized using ICU) +def word_freqs() -> list[tuple[str, int]]: + """Get word frequency from OSCAR Corpus (words tokenized using ICU) """ freqs: list[tuple[str, int]] = [] path = get_corpus_path(_OSCAR_FILENAME) @@ -29,7 +27,7 @@ def word_freqs() -> List[Tuple[str, int]]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as f: + with open(path, encoding="utf-8-sig") as f: lines = list(f.readlines()) del lines[0] for line in lines: @@ -44,8 +42,7 @@ def word_freqs() -> List[Tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from OSCAR Corpus (words tokenized using ICU) + """Get unigram word frequency from OSCAR Corpus (words tokenized using ICU) """ freqs: dict[str, int] = defaultdict(int) path = get_corpus_path(_OSCAR_FILENAME) @@ -53,7 +50,7 @@ def unigram_word_freqs() -> dict[str, int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = list(fh.readlines()) del lines[0] for i in lines: diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 8f8a61948..94a73f922 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-English Transliteration Dictionary v1.4 +"""Thai-English Transliteration Dictionary v1.4 Wannaphong Phatthiyaphaibun. (2022). wannaphong/thai-english-transliteration-dictionary: v1.4 (v1.4). Zenodo. https://doi.org/10.5281/zenodo.6716672 """ +from __future__ import annotations + __all__ = [ "get_transliteration_dict", "TRANSLITERATE_EN", @@ -26,8 +26,7 @@ def get_transliteration_dict() -> defaultdict: - """ - Get Thai to English transliteration dictionary. + """Get Thai to English transliteration dictionary. The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. """ @@ -43,7 +42,7 @@ def get_transliteration_dict() -> defaultdict: lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} ) try: - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: # assume that the first row contains column names, so skip it. for line in f.readlines()[1:]: stripped = line.strip() diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 36a3e0311..af9513c6d 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project. # SPDX-License-Identifier: Apache-2.0 +"""Thai National Corpus word frequency """ -Thai National Corpus word frequency -""" + +from __future__ import annotations __all__ = [ "bigram_word_freqs", @@ -13,7 +13,6 @@ ] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus, get_corpus_path @@ -22,9 +21,8 @@ _TRIGRAM_CORPUS_NAME = "tnc_trigram_word_freqs" -def word_freqs() -> List[Tuple[str, int]]: - """ - Get word frequency from Thai National Corpus (TNC) +def word_freqs() -> list[tuple[str, int]]: + """Get word frequency from Thai National Corpus (TNC) \n(See: `dev/pythainlp/corpus/tnc_freq.txt\ `_) @@ -41,8 +39,7 @@ def word_freqs() -> List[Tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from Thai National Corpus (TNC) + """Get unigram word frequency from Thai National Corpus (TNC) """ freqs: dict[str, int] = defaultdict(int) lines = list(get_corpus(_UNIGRAM_FILENAME)) @@ -54,9 +51,8 @@ def unigram_word_freqs() -> dict[str, int]: return freqs -def bigram_word_freqs() -> dict[Tuple[str, str], int]: - """ - Get bigram word frequency from Thai National Corpus (TNC) +def bigram_word_freqs() -> dict[tuple[str, str], int]: + """Get bigram word frequency from Thai National Corpus (TNC) """ freqs: dict[tuple[str, str], int] = defaultdict(int) path = get_corpus_path(_BIGRAM_CORPUS_NAME) @@ -64,7 +60,7 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: for i in fh.readlines(): temp = i.strip().split(" ") freqs[(temp[0], temp[1])] = int(temp[-1]) @@ -72,9 +68,8 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]: return freqs -def trigram_word_freqs() -> dict[Tuple[str, str, str], int]: - """ - Get trigram word frequency from Thai National Corpus (TNC) +def trigram_word_freqs() -> dict[tuple[str, str, str], int]: + """Get trigram word frequency from Thai National Corpus (TNC) """ freqs: dict[tuple[str, str, str], int] = defaultdict(int) path = get_corpus_path(_TRIGRAM_CORPUS_NAME) @@ -82,7 +77,7 @@ def trigram_word_freqs() -> dict[Tuple[str, str, str], int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: for i in fh.readlines(): temp = i.strip().split(" ") freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index c6e470f74..99134ca4b 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -1,27 +1,25 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Textbook Corpus (TTC) word frequency +"""Thai Textbook Corpus (TTC) word frequency Credit: Korakot Chaovavanich https://www.facebook.com/photo.php?fbid=363640477387469&set=gm.434330506948445&type=3&permPage=1 """ +from __future__ import annotations + __all__ = ["word_freqs", "unigram_word_freqs"] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus _UNIGRAM_FILENAME = "ttc_freq.txt" -def word_freqs() -> List[Tuple[str, int]]: - """ - Get word frequency from Thai Textbook Corpus (TTC) +def word_freqs() -> list[tuple[str, int]]: + """Get word frequency from Thai Textbook Corpus (TTC) \n(See: `dev/pythainlp/corpus/ttc_freq.txt\ `_) """ @@ -36,8 +34,7 @@ def word_freqs() -> List[Tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from Thai Textbook Corpus (TTC) + """Get unigram word frequency from Thai Textbook Corpus (TTC) """ freqs: dict[str, int] = defaultdict(int) diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index b33b3ef11..80701fd26 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Tool for creating word lists +"""Tool for creating word lists codes are from Korakot Chaovavanich. :See also: @@ -13,17 +11,18 @@ `_ """ +from __future__ import annotations + from collections import Counter -from typing import Callable, Iterable, Iterator, List, Set, Tuple +from collections.abc import Callable, Iterable, Iterator from pythainlp.corpus import thai_words from pythainlp.tokenize import newmm from pythainlp.util import Trie -def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]: - """ - Return beginning and ending indexes of word pairs +def index_pairs(words: list[str]) -> Iterator[tuple[int, int]]: + """Return beginning and ending indexes of word pairs """ i = 0 for w in words: @@ -32,11 +31,10 @@ def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]: def find_badwords( - tokenize: Callable[[str], List[str]], + tokenize: Callable[[str], list[str]], training_data: Iterable[Iterable[str]], -) -> Set[str]: - """ - Find words that do not work well with the `tokenize` function +) -> set[str]: + """Find words that do not work well with the `tokenize` function for the provided `training_data`. :param Callable[[str], List[str]] tokenize: a tokenize function @@ -68,12 +66,11 @@ def find_badwords( def revise_wordset( - tokenize: Callable[[str], List[str]], + tokenize: Callable[[str], list[str]], orig_words: Iterable[str], training_data: Iterable[Iterable[str]], -) -> Set[str]: - """ - Revise a set of words that could improve tokenization performance of +) -> set[str]: + """Revise a set of words that could improve tokenization performance of a dictionary-based `tokenize` function. `orig_words` will be used as a base set for the dictionary. @@ -91,7 +88,7 @@ def revise_wordset( :Example:: :: - + from pythainlp.corpus import thai_words from pythainlp.corpus.util import revise_wordset from pythainlp.tokenize.longest import segment @@ -119,9 +116,8 @@ def revise_wordset( def revise_newmm_default_wordset( training_data: Iterable[Iterable[str]], -) -> Set[str]: - """ - Revise a set of word that could improve tokenization performance of +) -> set[str]: + """Revise a set of word that could improve tokenization performance of `pythainlp.tokenize.newmm`, a dictionary-based tokenizer and a default tokenizer for PyThaiNLP. diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py index de65e1e1f..14bbc0ed3 100644 --- a/pythainlp/corpus/volubilis.py +++ b/pythainlp/corpus/volubilis.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Provides an optional word list from the Volubilis dictionary. """ -Provides an optional word list from the Volubilis dictionary. -""" -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus @@ -13,10 +12,9 @@ _VOLUBILIS_FILENAME = "volubilis_words_th.txt" -def thai_volubilis_words() -> FrozenSet[str]: - """ - Return a frozenset of Thai words from the Volubilis dictionary - +def thai_volubilis_words() -> frozenset[str]: + """Return a frozenset of Thai words from the Volubilis dictionary + See: `dev/pythainlp/corpus/volubilis_words_th.txt\ `_ diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py index 2c0ce7c2e..78283e716 100644 --- a/pythainlp/corpus/wikipedia.py +++ b/pythainlp/corpus/wikipedia.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Provides an optional word list from Thai Wikipedia titles. """ -Provides an optional word list from Thai Wikipedia titles. -""" -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus @@ -13,9 +12,8 @@ _WIKIPEDIA_TITLES_FILENAME = "wikipedia_titles_th.txt" -def thai_wikipedia_titles() -> FrozenSet[str]: - """ - Return a frozenset of words from Thai Wikipedia titles corpus. +def thai_wikipedia_titles() -> frozenset[str]: + """Return a frozenset of words from Thai Wikipedia titles corpus. They are mostly nouns and noun phrases, including event, organization, people, place, and product names. Commonly misspelled words are included intentionally. @@ -31,6 +29,8 @@ def thai_wikipedia_titles() -> FrozenSet[str]: """ global _WIKIPEDIA_TITLES if not _WIKIPEDIA_TITLES: - _WIKIPEDIA_TITLES = get_corpus(_WIKIPEDIA_TITLES_FILENAME, comments=False) + _WIKIPEDIA_TITLES = get_corpus( + _WIKIPEDIA_TITLES_FILENAME, comments=False + ) return _WIKIPEDIA_TITLES diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py index 658314b4a..7373b96e2 100644 --- a/pythainlp/corpus/wordnet.py +++ b/pythainlp/corpus/wordnet.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -NLTK WordNet wrapper +"""NLTK WordNet wrapper API here is exactly the same as NLTK WordNet API, except that the lang (language) argument is "tha" (Thai) by default. @@ -11,6 +9,9 @@ For more on usage, see NLTK Howto: https://www.nltk.org/howto/wordnet.html """ + +from __future__ import annotations + import nltk try: @@ -26,15 +27,14 @@ from nltk.corpus import wordnet -def synsets(word: str, pos: str = None, lang: str = "tha"): - """ - This function returns the synonym set for all lemmas of the given word +def synsets(word: str, pos: str | None = None, lang: str = "tha"): + """This function returns the synonym set for all lemmas of the given word with an optional argument to constrain the part of speech of the word. :param str word: word to find synsets of - :param str pos: constraint of the part of speech (i.e. *n* for Noun, *v* + :param str | None pos: constraint of the part of speech (i.e. *n* for Noun, *v* for Verb, *a* for Adjective, *s* for Adjective - satellites, and *r* for Adverb) + satellites, and *r* for Adverb). Default is None. :param str lang: abbreviation of language (i.e. *eng*, *tha*). By default, it is *tha* @@ -75,8 +75,7 @@ def synsets(word: str, pos: str = None, lang: str = "tha"): def synset(name_synsets): - """ - This function returns the synonym set (synset) given the name of the synset + """This function returns the synonym set (synset) given the name of the synset (i.e. 'dog.n.01', 'chase.v.01'). :param str name_synsets: name of the synset @@ -88,7 +87,7 @@ def synset(name_synsets): >>> from pythainlp.corpus.wordnet import synset >>> - >>> difficult = synset('difficult.a.01') + >>> difficult = synset("difficult.a.01") >>> difficult Synset('difficult.a.01') >>> @@ -99,13 +98,12 @@ def synset(name_synsets): return wordnet.synset(name_synsets) -def all_lemma_names(pos: str = None, lang: str = "tha"): - """ - This function returns all lemma names for all synsets of the given +def all_lemma_names(pos: str | None = None, lang: str = "tha"): + """This function returns all lemma names for all synsets of the given part of speech tag and language. If part of speech tag is not specified, all synsets of all parts of speech will be used. - :param str pos: constraint of the part of speech (i.e. *n* for Noun, + :param str | None pos: constraint of the part of speech (i.e. *n* for Noun, *v* for Verb, *a* for Adjective, *s* for Adjective satellites, and *r* for Adverb). By default, *pos* is **None**. @@ -142,12 +140,11 @@ def all_lemma_names(pos: str = None, lang: str = "tha"): return wordnet.all_lemma_names(pos=pos, lang=lang) -def all_synsets(pos: str = None): - """ - This function iterates over all synsets constrained by the given +def all_synsets(pos: str | None = None): + """This function iterates over all synsets constrained by the given part of speech tag. - :param str pos: part of speech tag + :param str | None pos: part of speech tag. Default is None. :return: list of synsets constrained by the given part of speech tag. :rtype: Iterable[:class:`Synset`] @@ -174,8 +171,7 @@ def all_synsets(pos: str = None): def langs(): - """ - This function returns a set of ISO-639 language codes. + """This function returns a set of ISO-639 language codes. :return: ISO-639 language codes :rtype: list[str] @@ -192,15 +188,14 @@ def langs(): return wordnet.langs() -def lemmas(word: str, pos: str = None, lang: str = "tha"): - """ - This function returns all lemmas given the word with an optional +def lemmas(word: str, pos: str | None = None, lang: str = "tha"): + """This function returns all lemmas given the word with an optional argument to constrain the part of speech of the word. :param str word: word to find lemmas of - :param str pos: constraint of the part of speech (i.e. *n* for Noun, + :param str | None pos: constraint of the part of speech (i.e. *n* for Noun, *v* for Verb, *a* for Adjective, *s* for - Adjective satellites, and *r* for Adverb) + Adjective satellites, and *r* for Adverb). Default is None. :param str lang: abbreviation of language (i.e. *eng*, *tha*). By default, it is *tha*. @@ -237,8 +232,7 @@ def lemmas(word: str, pos: str = None, lang: str = "tha"): def lemma(name_synsets): - """ - This function returns lemma object given the name. + """This function returns lemma object given the name. .. note:: Support only English language (*eng*). @@ -252,21 +246,20 @@ def lemma(name_synsets): >>> from pythainlp.corpus.wordnet import lemma >>> - >>> lemma('practice.v.01.exercise') + >>> lemma("practice.v.01.exercise") Lemma('practice.v.01.exercise') >>> - >>> lemma('drill.v.03.exercise') + >>> lemma("drill.v.03.exercise") Lemma('drill.v.03.exercise') >>> - >>> lemma('exercise.n.01.exercise') + >>> lemma("exercise.n.01.exercise") Lemma('exercise.n.01.exercise') """ return wordnet.lemma(name_synsets) def lemma_from_key(key): - """ - This function returns lemma object given the lemma key. + """This function returns lemma object given the lemma key. This is similar to :func:`lemma` but it needs to be given the key of lemma instead of the name of lemma. @@ -282,7 +275,7 @@ def lemma_from_key(key): >>> from pythainlp.corpus.wordnet import lemma, lemma_from_key >>> - >>> practice = lemma('practice.v.01.exercise') + >>> practice = lemma("practice.v.01.exercise") >>> practice.key() exercise%2:41:00:: >>> lemma_from_key(practice.key()) @@ -292,8 +285,7 @@ def lemma_from_key(key): def path_similarity(synsets1, synsets2): - """ - This function returns similarity between two synsets based on the + """This function returns similarity between two synsets based on the shortest path distance calculated using the equation below. .. math:: @@ -317,9 +309,9 @@ def path_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import path_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> path_similarity(entity, obj) 0.3333333333333333 @@ -332,8 +324,7 @@ def path_similarity(synsets1, synsets2): def lch_similarity(synsets1, synsets2): - """ - This function returns Leacock Chodorow similarity (LCH) + """This function returns Leacock Chodorow similarity (LCH) between two synsets, based on the shortest path distance and the maximum depth of the taxonomy. The equation to calculate LCH similarity is shown below: @@ -355,9 +346,9 @@ def lch_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import lch_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> lch_similarity(entity, obj) 2.538973871058276 @@ -370,8 +361,7 @@ def lch_similarity(synsets1, synsets2): def wup_similarity(synsets1, synsets2): - """ - This function returns Wu-Palmer similarity (WUP) between two synsets, + """This function returns Wu-Palmer similarity (WUP) between two synsets, based on the depth of the two senses in the taxonomy and their Least Common Subsumer (most specific ancestor node). @@ -387,9 +377,9 @@ def wup_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import wup_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> wup_similarity(entity, obj) 0.5 @@ -401,13 +391,13 @@ def wup_similarity(synsets1, synsets2): return wordnet.wup_similarity(synsets1, synsets2) -def morphy(form, pos: str = None): - """ - This function finds a possible base form for the given form, +def morphy(form, pos: str | None = None): + """This function finds a possible base form for the given form, with the given part of speech. :param str form: the form to finds the base form of - :param str pos: part of speech tag of words to be searched + :param str | None pos: part of speech tag of words to be searched. + Default is None. :return: base form of the given form :rtype: str @@ -432,8 +422,7 @@ def morphy(form, pos: str = None): def custom_lemmas(tab_file, lang: str): - """ - This function reads a custom tab file + """This function reads a custom tab file (see: http://compling.hss.ntu.edu.sg/omw/) containing mappings of lemmas in the given language. diff --git a/pythainlp/el/__init__.py b/pythainlp/el/__init__.py index d076062b1..9638fa2fa 100644 --- a/pythainlp/el/__init__.py +++ b/pythainlp/el/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.el +"""pythainlp.el """ __all__ = ["EntityLinker"] diff --git a/pythainlp/el/_multiel.py b/pythainlp/el/_multiel.py index babe2b292..2c73b64b0 100644 --- a/pythainlp/el/_multiel.py +++ b/pythainlp/el/_multiel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index dbb30d97e..17ea4cb76 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -1,14 +1,17 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations class EntityLinker: - def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata"): - """ - EntityLinker + def __init__( + self, + model_name: str = "bela", + device: str = "cuda", + tag: str = "wikidata", + ): + """EntityLinker :param str model_name: model name (bela) :param str device: device for running model on @@ -21,24 +24,29 @@ def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata") self.device = device self.tag = tag if self.model_name not in ["bela"]: - raise NotImplementedError(f"EntityLinker doesn't support {model_name} model.") + raise NotImplementedError( + f"EntityLinker doesn't support {model_name} model." + ) if self.tag not in ["wikidata"]: - raise NotImplementedError(f"EntityLinker doesn't support {tag} tag.") + raise NotImplementedError( + f"EntityLinker doesn't support {tag} tag." + ) from pythainlp.el._multiel import MultiEL + self.model = MultiEL(model_name=self.model_name, device=self.device) - def get_el(self, list_text:Union[List[str], str])->Union[List[dict], str]: - """ - Get Entity Linking from Thai Text - + + def get_el(self, list_text: list[str] | str) -> list[dict] | str: + """Get Entity Linking from Thai Text + :param str Union[List[str], str]: list of Thai text or text :return: list of entity linking :rtype: Union[List[dict], str] - + :Example: :: from pythainlp.el import EntityLinker - + el = EntityLinker(device="cuda") print(el.get_el("จ๊อบเคยเป็นซีอีโอบริษัทแอปเปิล")) # output: [{'offsets': [11, 23], diff --git a/pythainlp/generate/__init__.py b/pythainlp/generate/__init__.py index ab68623f9..456d90467 100644 --- a/pythainlp/generate/__init__.py +++ b/pythainlp/generate/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Text Generation +"""Thai Text Generation """ __all__ = ["Bigram", "Trigram", "Unigram"] diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 14102f14c..bf317e6a7 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text generator using n-gram language model +"""Text generator using n-gram language model codes are from https://towardsdatascience.com/understanding-word-n-grams-and-n-gram-probability-in-natural-language-processing-9d9eef0fa058 """ +from __future__ import annotations + import random -from typing import List, Union from pythainlp.corpus.oscar import ( unigram_word_freqs as oscar_word_freqs_unigram, @@ -22,8 +21,7 @@ class Unigram: - """ - Text generator using Unigram + """Text generator using Unigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -52,9 +50,8 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: - """ - :param str start_seq: word to begin sentence with + ) -> list[str] | str: + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence @@ -110,8 +107,7 @@ def _next_word( class Bigram: - """ - Text generator using Bigram + """Text generator using Bigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -126,8 +122,7 @@ def __init__(self, name: str = "tnc"): self.words = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str) -> float: - """ - probability of word + """Probability of word :param int t1: text 1 :param int t2: text 2 @@ -148,9 +143,8 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: - """ - :param str start_seq: word to begin sentence with + ) -> list[str] | str: + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence @@ -198,8 +192,7 @@ def gen_sentence( class Trigram: - """ - Text generator using Trigram + """Text generator using Trigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -216,8 +209,7 @@ def __init__(self, name: str = "tnc"): self.words = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str, t3: str) -> float: - """ - probability of word + """Probability of word :param int t1: text 1 :param int t2: text 2 @@ -240,9 +232,8 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: - """ - :param str start_seq: word to begin sentence with + ) -> list[str] | str: + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 445a063af..e33d20b18 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -1,19 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai2fit: Thai Wikipeida Language Model for Text Generation +"""Thai2fit: Thai Wikipeida Language Model for Text Generation Codes are from https://github.com/PyThaiNLP/tutorials/blob/master/source/notebooks/text_generation.ipynb """ +from __future__ import annotations + __all__ = ["gen_sentence"] import pickle import random -from typing import List, Union # fastai import fastai @@ -88,9 +87,8 @@ def gen_sentence( N: int = 4, prob: float = 0.001, output_str: bool = True, -) -> Union[List[str], str]: - """ - Text generator using Thai2fit +) -> list[str] | str: + """Text generator using Thai2fit :param str start_seq: word to begin sentence with :param int N: number of words diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index cdb63efad..12289e338 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import re import torch @@ -9,34 +10,31 @@ class WangChanGLM: def __init__(self): - self.exclude_pattern = re.compile(r'[^ก-๙]+') + self.exclude_pattern = re.compile(r"[^ก-๙]+") self.stop_token = "\n" self.PROMPT_DICT = { "prompt_input": ( ": {input}\n: {instruction}\n: " ), - "prompt_no_input": ( - ": {instruction}\n: " - ), - "prompt_chatbot": ( - ": {human}\n: {bot}" - ), + "prompt_no_input": (": {instruction}\n: "), + "prompt_chatbot": (": {human}\n: {bot}"), } - def is_exclude(self, text:str)->bool: + + def is_exclude(self, text: str) -> bool: return bool(self.exclude_pattern.search(text)) + def load_model( self, - model_path:str="pythainlp/wangchanglm-7.5B-sft-en-sharded", - return_dict:bool=True, - load_in_8bit:bool=False, - device:str="cuda", + model_path: str = "pythainlp/wangchanglm-7.5B-sft-en-sharded", + return_dict: bool = True, + load_in_8bit: bool = False, + device: str = "cuda", torch_dtype=torch.float16, - offload_folder:str="./", - low_cpu_mem_usage:bool=True + offload_folder: str = "./", + low_cpu_mem_usage: bool = True, ): - """ - Load model - + """Load model + :param str model_path: model path :param bool return_dict: return dict :param bool load_in_8bit: load model in 8bit @@ -47,6 +45,7 @@ def load_model( """ import pandas as pd from transformers import AutoModelForCausalLM, AutoTokenizer + self.device = device self.torch_dtype = torch_dtype self.model_path = model_path @@ -57,27 +56,29 @@ def load_model( device_map=device, torch_dtype=torch_dtype, offload_folder=offload_folder, - low_cpu_mem_usage=low_cpu_mem_usage + low_cpu_mem_usage=low_cpu_mem_usage, ) self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) - self.df = pd.DataFrame(self.tokenizer.vocab.items(), columns=['text', 'idx']) - self.df['is_exclude'] = self.df.text.map(self.is_exclude) + self.df = pd.DataFrame( + self.tokenizer.vocab.items(), columns=["text", "idx"] + ) + self.df["is_exclude"] = self.df.text.map(self.is_exclude) self.exclude_ids = self.df[self.df.is_exclude is True].idx.tolist() + def gen_instruct( self, - text:str, - max_new_tokens:int=512, - top_p:float=0.95, - temperature:float=0.9, - top_k:int=50, - no_repeat_ngram_size:int=2, - typical_p:float=1., - thai_only:bool=True, - skip_special_tokens:bool=True + text: str, + max_new_tokens: int = 512, + top_p: float = 0.95, + temperature: float = 0.9, + top_k: int = 50, + no_repeat_ngram_size: int = 2, + typical_p: float = 1.0, + thai_only: bool = True, + skip_special_tokens: bool = True, ): - """ - Generate Instruct - + """Generate Instruct + :param str text: text :param int max_new_tokens: maximum number of new tokens :param float top_p: top p @@ -95,45 +96,48 @@ def gen_instruct( if thai_only: output_tokens = self.model.generate( input_ids=batch["input_ids"], - max_new_tokens=max_new_tokens, # 512 - begin_suppress_tokens = self.exclude_ids, + max_new_tokens=max_new_tokens, # 512 + begin_suppress_tokens=self.exclude_ids, no_repeat_ngram_size=no_repeat_ngram_size, - #oasst k50 + # oasst k50 top_k=top_k, - top_p=top_p, # 0.95 + top_p=top_p, # 0.95 typical_p=typical_p, - temperature=temperature, # 0.9 + temperature=temperature, # 0.9 ) else: output_tokens = self.model.generate( input_ids=batch["input_ids"], - max_new_tokens=max_new_tokens, # 512 + max_new_tokens=max_new_tokens, # 512 no_repeat_ngram_size=no_repeat_ngram_size, - #oasst k50 + # oasst k50 top_k=top_k, - top_p=top_p, # 0.95 + top_p=top_p, # 0.95 typical_p=typical_p, - temperature=temperature, # 0.9 + temperature=temperature, # 0.9 ) - return self.tokenizer.decode(output_tokens[0][len(batch["input_ids"][0]):], skip_special_tokens=skip_special_tokens) + return self.tokenizer.decode( + output_tokens[0][len(batch["input_ids"][0]) :], + skip_special_tokens=skip_special_tokens, + ) + def instruct_generate( self, instruct: str, - context: str = None, + context: str = "", max_new_tokens=512, - temperature: float =0.9, + temperature: float = 0.9, top_p: float = 0.95, - top_k:int=50, - no_repeat_ngram_size:int=2, - typical_p:float=1, - thai_only:bool=True, - skip_special_tokens:bool=True + top_k: int = 50, + no_repeat_ngram_size: int = 2, + typical_p: float = 1, + thai_only: bool = True, + skip_special_tokens: bool = True, ): - """ - Generate Instruct - + """Generate Instruct + :param str instruct: Instruct - :param str context: context + :param str context: context (optional, default is empty string) :param int max_new_tokens: maximum number of new tokens :param float top_p: top p :param float temperature: temperature @@ -153,7 +157,7 @@ def instruct_generate( model = WangChanGLM() - model.load_model(device="cpu",torch_dtype=torch.bfloat16) + model.load_model(device="cpu", torch_dtype=torch.bfloat16) print(model.instruct_generate(instruct="ขอวิธีลดน้ำหนัก")) # output: ลดน้ําหนักให้ได้ผล ต้องทําอย่างค่อยเป็นค่อยไป @@ -165,13 +169,13 @@ def instruct_generate( # และเครื่องดื่มแอลกอฮอล์ """ - if context in (None, ""): - prompt = self.PROMPT_DICT['prompt_no_input'].format_map( - {'instruction': instruct, 'input': ''} + if not context: + prompt = self.PROMPT_DICT["prompt_no_input"].format_map( + {"instruction": instruct, "input": ""} ) else: - prompt = self.PROMPT_DICT['prompt_input'].format_map( - {'instruction': instruct, 'input': context} + prompt = self.PROMPT_DICT["prompt_input"].format_map( + {"instruction": instruct, "input": context} ) result = self.gen_instruct( prompt, @@ -182,6 +186,6 @@ def instruct_generate( no_repeat_ngram_size=no_repeat_ngram_size, typical_p=typical_p, thai_only=thai_only, - skip_special_tokens=skip_special_tokens + skip_special_tokens=skip_special_tokens, ) return result diff --git a/pythainlp/khavee/__init__.py b/pythainlp/khavee/__init__.py index b8959d272..e4882747b 100644 --- a/pythainlp/khavee/__init__.py +++ b/pythainlp/khavee/__init__.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -__all__ = ["KhaveeVerifier"] - -from pythainlp.khavee.core import KhaveeVerifier +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +__all__ = ["KhaveeVerifier"] + +from pythainlp.khavee.core import KhaveeVerifier diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 81e05b662..5f8b0281f 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -1,691 +1,682 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: C901 - -from typing import List, Union - -from pythainlp import thai_consonants -from pythainlp.tokenize import subword_tokenize -from pythainlp.util import remove_tonemark, sound_syllable - - -class KhaveeVerifier: - def __init__(self): - """ - KhaveeVerifier: Thai Poetry verifier - """ - - def _has_true_final_yl(self, word: str) -> bool: - """ - Check if ย or ล is a true final consonant - (not just part of the vowel sound with ไ/ใ) - - :param str word: Thai word - :return: True if ย or ล is a true final consonant - :rtype: bool - """ - if len(word) < 2: - return False - # Count consonants in the word - consonant_count = sum(1 for c in word if c in thai_consonants) - # If there are 2+ consonants and word ends with ย or ล, it's a true final - return consonant_count >= 2 and word[-1] in ["ย", "ล"] - - def check_sara(self, word: str) -> str: - """ - Check the vowels in the Thai word. - - :param str word: Thai word - :return: vowel name of the word - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_sara("เริง")) - # output: 'เออ' - """ - sara = [] - countoa = 0 - - # In case of การันย์ - if "์" in word[-1]: - word = word[:-2] - - # In case of สระเดี่ยว - for i in word: - if i in ("ะ", "ั"): - sara.append("อะ") - elif i == "ิ": - sara.append("อิ") - elif i == "ุ": - sara.append("อุ") - elif i == "ึ": - sara.append("อึ") - elif i == "ี": - sara.append("อี") - elif i == "ู": - sara.append("อู") - elif i == "ื": - sara.append("อือ") - elif i == "เ": - sara.append("เอ") - elif i == "แ": - sara.append("แอ") - elif i == "า": - sara.append("อา") - elif i == "โ": - sara.append("โอ") - elif i == "ำ": - sara.append("อำ") - elif i == "อ": - countoa += 1 - sara.append("ออ") - elif i == "ั" and "ว" in word: - sara.append("อัว") - elif i in ("ไ", "ใ"): - sara.append("ไอ") - elif i == "็": - sara.append("ออ") - elif "รร" in word: - if self.check_marttra(word) == "กม": - sara.append("อำ") - else: - sara.append("อะ") - - # In case of ออ - if countoa == 1 and "อ" in word[-1] and "เ" not in word: - sara.remove("ออ") - - # In case of เอ เอ - countA = 0 - for i in sara: - if i == "เอ": - countA = countA + 1 - if countA > 1: - sara.remove("เอ") - sara.remove("เอ") - sara.append("แ") - - # In case of สระประสม - if "เอ" in sara and "อะ" in sara: - sara.remove("เอ") - sara.remove("อะ") - sara.append("เอะ") - elif "แอ" in sara and "อะ" in sara: - sara.remove("แอ") - sara.remove("อะ") - sara.append("แอะ") - - if "เอะ" in sara and "ออ" in sara: - sara.remove("เอะ") - sara.remove("ออ") - sara.append("เออะ") - elif "เอ" in sara and "อิ" in sara: - sara.remove("เอ") - sara.remove("อิ") - sara.append("เออ") - elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: - sara.remove("เอ") - sara.remove("ออ") - sara.append("เออ") - elif "โอ" in sara and "อะ" in sara: - sara.remove("โอ") - sara.remove("อะ") - sara.append("โอะ") - elif "เอ" in sara and "อี" in sara: - sara.remove("เอ") - sara.remove("อี") - sara.append("เอีย") - elif "เอ" in sara and "อือ" in sara: - sara.remove("เอ") - sara.remove("อือ") - sara.append("อัว") - elif "เอ" in sara and "อา" in sara: - sara.remove("เอ") - sara.remove("อา") - sara.append("เอา") - elif "เ" in word and "า" in word and "ะ" in word: - sara = [] - sara.append("เอาะ") - - if "อือ" in sara and "เออ" in sara: - sara.remove("เออ") - sara.remove("อือ") - sara.append("เอือ") - elif "ออ" in sara and len(sara) > 1: - sara.remove("ออ") - elif "ว" in word and len(sara) == 0: - sara.append("อัว") - - if "ั" in word and self.check_marttra(word) == "กา": - sara = [] - sara.append("ไอ") - - # In case of อ - if word == "เออะ": - sara = [] - sara.append("เออะ") - elif word == "เออ": - sara = [] - sara.append("เออ") - elif word == "เอ": - sara = [] - sara.append("เอ") - elif word == "เอะ": - sara = [] - sara.append("เอะ") - elif word == "เอา": - sara = [] - sara.append("เอา") - elif word == "เอาะ": - sara = [] - sara.append("เอาะ") - - if "ฤา" in word or "ฦา" in word: - sara = [] - sara.append("อือ") - elif "ฤ" in word or "ฦ" in word: - sara = [] - sara.append("อึ") - - # In case of กน - if not sara and len(word) == 2: - if word[-1] != "ร": - sara.append("โอะ") - else: - sara.append("ออ") - elif not sara and len(word) == 3: - sara.append("ออ") - - # In case of บ่ - if word == "บ่": - sara = [] - sara.append("ออ") - - if "ํ" in word: - sara = [] - sara.append("อำ") - - if "เ" in word and "ื" in word and "อ" in word: - sara = [] - sara.append("เอือ") - - if not sara: - return "Can't find Sara in this word" - - return sara[0] - - def check_marttra(self, word: str) -> str: - """ - Check the Thai spelling Section in the Thai word. - - :param str word: Thai word - :return: name of spelling Section of the word. - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_marttra("สาว")) - # output: 'เกอว' - """ - # Handle consonant clusters ending with ร - # ตร, ทร → remove ร (treat as final ต/ท sound) - # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) - # But single syllable words like "กร" should keep ร - if len(word) >= 3 and word[-1] == "ร": - if word[-2] in ["ต", "ท"]: - word = word[:-1] - elif word[-2] in ["ก", "ข", "ค", "ฆ"]: - word = word[:-1] - - word = self.handle_karun_sound_silence(word) - word = remove_tonemark(word) - - # Check for ำ at the end (represents "am" sound, ends with m) - if word[-1] == "ำ": - return "กม" - - # Check for vowels and special patterns that indicate open syllables (กา) - # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel - if "ไ" in word or "ใ" in word: - if word[-1] not in ["ย", "ล"]: - return "กา" - elif not self._has_true_final_yl(word): - # ย/ล is part of the vowel sound, not a true final - return "กา" - # else: ย/ล is a true final, continue to consonant classification below - - if ( - ("ํ" in word and "า" in word) - ): - return "กา" - elif ( - word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] - or ("ี" in word and "ย" in word[-1]) - or ("ื" in word and "อ" in word[-1]) - ): - return "กา" - elif word[-1] in ["ง"]: - return "กง" - elif word[-1] in ["ม"]: - return "กม" - elif word[-1] in ["ย"]: - return "เกย" - elif word[-1] in ["ล"]: - return "เกย" - elif word[-1] in ["ว"]: - return "เกอว" - elif word[-1] in ["ก", "ข", "ค", "ฆ"]: - return "กก" - elif word[-1] in [ - "จ", - "ช", - "ซ", - "ฎ", - "ฏ", - "ฐ", - "ฑ", - "ฒ", - "ด", - "ต", - "ถ", - "ท", - "ธ", - "ศ", - "ษ", - "ส", - ]: - return "กด" - elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: - return "กน" - elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: - return "กบ" - else: - if "็" in word: - return "กา" - else: - return "Cant find Marttra in this word" - - def is_sumpus(self, word1: str, word2: str) -> bool: - """ - Check the rhyme between two words. - - :param str word1: Thai word - :param str word2: Thai word - :return: boolean - :rtype: bool - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.is_sumpus("สรร", "อัน")) - # output: True - - print(kv.is_sumpus("สรร", "แมว")) - # output: False - """ - marttra1 = self.check_marttra(word1) - marttra2 = self.check_marttra(word2) - sara1 = self.check_sara(word1) - sara2 = self.check_sara(word2) - if sara1 == "อะ" and marttra1 == "เกย": - sara1 = "ไอ" - marttra1 = "กา" - elif sara2 == "อะ" and marttra2 == "เกย": - sara2 = "ไอ" - marttra2 = "กา" - if sara1 == "อำ" and marttra1 == "กม": - sara1 = "อำ" - marttra1 = "กา" - elif sara2 == "อำ" and marttra2 == "กม": - sara2 = "อำ" - marttra2 = "กา" - return bool(marttra1 == marttra2 and sara1 == sara2) - - def check_karu_lahu(self, text): - if ( - self.check_marttra(text) != "กา" - or ( - self.check_marttra(text) == "กา" - and self.check_sara(text) - in [ - "อา", - "อี", - "อือ", - "อู", - "เอ", - "แอ", - "โอ", - "ออ", - "เออ", - "เอีย", - "เอือ", - "อัว", - ] - ) - or self.check_sara(text) in ["อำ", "ไอ", "เอา"] - ) and text not in ["บ่", "ณ", "ธ", "ก็"]: - return "karu" - else: - return "lahu" - - def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: - """ - Check the suitability of the poem according to Thai principles. - - :param str text: Thai poem - :param int k_type: type of Thai poem - :return: the check results of the suitability of the poem according to Thai principles. - :rtype: Union[List[str], str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ - มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: The poem is correct according to the principle. - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ - เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: [ - "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", - "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" - ] - """ - if k_type == 8: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 10: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 10 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append( - [ - sub_sent[1], - sub_sent[2], - sub_sent[3], - sub_sent[4], - ] - ) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 3: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form of klon 8." - elif k_type == 4: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 5: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 4 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 1: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form." - - else: - return "Something went wrong. Make sure you enter it in the correct form." - - def check_aek_too( - self, text: Union[List[str], str], dead_syllable_as_aek: bool = False - ) -> Union[List[bool], List[str], bool, str]: - """ - Checker of Thai tonal words - - :param Union[List[str], str] text: Thai word or list of Thai words - :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek - :return: the check result if the word is aek or too or False (not both) or list of check results if input is list - :rtype: Union[List[bool], List[str], bool, str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - # การเช็คคำเอกโท - print( - kv.check_aek_too("เอง"), - kv.check_aek_too("เอ่ง"), - kv.check_aek_too("เอ้ง"), - ) - # -> False, aek, too - print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน - # -> [False, 'aek', 'too'] - - - """ - if isinstance(text, list): - return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] - - if not isinstance(text, str): - raise TypeError("text must be str or iterable list[str]") - - word_characters = [*text] - if "่" in word_characters and "้" not in word_characters: - return "aek" - elif "้" in word_characters and "่" not in word_characters: - return "too" - if dead_syllable_as_aek and sound_syllable(text) == "dead": - return "aek" - else: - return False - - def handle_karun_sound_silence(self, word: str) -> str: - """ - Handle silent sounds in Thai words using '์' character (Karun) - by stripping all characters before the 'Karun' character that should be silenced - - :param str text: Thai word - :return: Thai word with silent words stripped - :rtype: str - """ - sound_silenced = word.endswith("์") - if not sound_silenced: - return word - # Remove ์ and the silent consonant before it - # การันต์ (์) marks the consonant immediately before it as silent - word = word[:-2] - return word +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: C901 +from __future__ import annotations + +from pythainlp import thai_consonants +from pythainlp.tokenize import subword_tokenize +from pythainlp.util import remove_tonemark, sound_syllable + + +class KhaveeVerifier: + def __init__(self): + """KhaveeVerifier: Thai Poetry verifier + """ + + def _has_true_final_yl(self, word: str) -> bool: + """Check if ย or ล is a true final consonant + (not just part of the vowel sound with ไ/ใ) + + :param str word: Thai word + :return: True if ย or ล is a true final consonant + :rtype: bool + """ + if len(word) < 2: + return False + # Count consonants in the word + consonant_count = sum(1 for c in word if c in thai_consonants) + # If there are 2+ consonants and word ends with ย or ล, it's a true final + return consonant_count >= 2 and word[-1] in ["ย", "ล"] + + def check_sara(self, word: str) -> str: + """Check the vowels in the Thai word. + + :param str word: Thai word + :return: vowel name of the word + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_sara("เริง")) + # output: 'เออ' + """ + sara = [] + countoa = 0 + + # In case of การันย์ + if "์" in word[-1]: + word = word[:-2] + + # In case of สระเดี่ยว + for i in word: + if i in ("ะ", "ั"): + sara.append("อะ") + elif i == "ิ": + sara.append("อิ") + elif i == "ุ": + sara.append("อุ") + elif i == "ึ": + sara.append("อึ") + elif i == "ี": + sara.append("อี") + elif i == "ู": + sara.append("อู") + elif i == "ื": + sara.append("อือ") + elif i == "เ": + sara.append("เอ") + elif i == "แ": + sara.append("แอ") + elif i == "า": + sara.append("อา") + elif i == "โ": + sara.append("โอ") + elif i == "ำ": + sara.append("อำ") + elif i == "อ": + countoa += 1 + sara.append("ออ") + elif i == "ั" and "ว" in word: + sara.append("อัว") + elif i in ("ไ", "ใ"): + sara.append("ไอ") + elif i == "็": + sara.append("ออ") + elif "รร" in word: + if self.check_marttra(word) == "กม": + sara.append("อำ") + else: + sara.append("อะ") + + # In case of ออ + if countoa == 1 and "อ" in word[-1] and "เ" not in word: + sara.remove("ออ") + + # In case of เอ เอ + countA = 0 + for i in sara: + if i == "เอ": + countA = countA + 1 + if countA > 1: + sara.remove("เอ") + sara.remove("เอ") + sara.append("แ") + + # In case of สระประสม + if "เอ" in sara and "อะ" in sara: + sara.remove("เอ") + sara.remove("อะ") + sara.append("เอะ") + elif "แอ" in sara and "อะ" in sara: + sara.remove("แอ") + sara.remove("อะ") + sara.append("แอะ") + + if "เอะ" in sara and "ออ" in sara: + sara.remove("เอะ") + sara.remove("ออ") + sara.append("เออะ") + elif "เอ" in sara and "อิ" in sara: + sara.remove("เอ") + sara.remove("อิ") + sara.append("เออ") + elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: + sara.remove("เอ") + sara.remove("ออ") + sara.append("เออ") + elif "โอ" in sara and "อะ" in sara: + sara.remove("โอ") + sara.remove("อะ") + sara.append("โอะ") + elif "เอ" in sara and "อี" in sara: + sara.remove("เอ") + sara.remove("อี") + sara.append("เอีย") + elif "เอ" in sara and "อือ" in sara: + sara.remove("เอ") + sara.remove("อือ") + sara.append("อัว") + elif "เอ" in sara and "อา" in sara: + sara.remove("เอ") + sara.remove("อา") + sara.append("เอา") + elif "เ" in word and "า" in word and "ะ" in word: + sara = [] + sara.append("เอาะ") + + if "อือ" in sara and "เออ" in sara: + sara.remove("เออ") + sara.remove("อือ") + sara.append("เอือ") + elif "ออ" in sara and len(sara) > 1: + sara.remove("ออ") + elif "ว" in word and len(sara) == 0: + sara.append("อัว") + + if "ั" in word and self.check_marttra(word) == "กา": + sara = [] + sara.append("ไอ") + + # In case of อ + if word == "เออะ": + sara = [] + sara.append("เออะ") + elif word == "เออ": + sara = [] + sara.append("เออ") + elif word == "เอ": + sara = [] + sara.append("เอ") + elif word == "เอะ": + sara = [] + sara.append("เอะ") + elif word == "เอา": + sara = [] + sara.append("เอา") + elif word == "เอาะ": + sara = [] + sara.append("เอาะ") + + if "ฤา" in word or "ฦา" in word: + sara = [] + sara.append("อือ") + elif "ฤ" in word or "ฦ" in word: + sara = [] + sara.append("อึ") + + # In case of กน + if not sara and len(word) == 2: + if word[-1] != "ร": + sara.append("โอะ") + else: + sara.append("ออ") + elif not sara and len(word) == 3: + sara.append("ออ") + + # In case of บ่ + if word == "บ่": + sara = [] + sara.append("ออ") + + if "ํ" in word: + sara = [] + sara.append("อำ") + + if "เ" in word and "ื" in word and "อ" in word: + sara = [] + sara.append("เอือ") + + if not sara: + return "Can't find Sara in this word" + + return sara[0] + + def check_marttra(self, word: str) -> str: + """Check the Thai spelling Section in the Thai word. + + :param str word: Thai word + :return: name of spelling Section of the word. + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_marttra("สาว")) + # output: 'เกอว' + """ + # Handle consonant clusters ending with ร + # ตร, ทร → remove ร (treat as final ต/ท sound) + # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) + # But single syllable words like "กร" should keep ร + if len(word) >= 3 and word[-1] == "ร": + if word[-2] in ["ต", "ท"]: + word = word[:-1] + elif word[-2] in ["ก", "ข", "ค", "ฆ"]: + word = word[:-1] + + word = self.handle_karun_sound_silence(word) + word = remove_tonemark(word) + + # Check for ำ at the end (represents "am" sound, ends with m) + if word[-1] == "ำ": + return "กม" + + # Check for vowels and special patterns that indicate open syllables (กา) + # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel + if "ไ" in word or "ใ" in word: + if word[-1] not in ["ย", "ล"]: + return "กา" + elif not self._has_true_final_yl(word): + # ย/ล is part of the vowel sound, not a true final + return "กา" + # else: ย/ล is a true final, continue to consonant classification below + + if "ํ" in word and "า" in word: + return "กา" + elif ( + word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] + or ("ี" in word and "ย" in word[-1]) + or ("ื" in word and "อ" in word[-1]) + ): + return "กา" + elif word[-1] in ["ง"]: + return "กง" + elif word[-1] in ["ม"]: + return "กม" + elif word[-1] in ["ย"]: + return "เกย" + elif word[-1] in ["ล"]: + return "เกย" + elif word[-1] in ["ว"]: + return "เกอว" + elif word[-1] in ["ก", "ข", "ค", "ฆ"]: + return "กก" + elif word[-1] in [ + "จ", + "ช", + "ซ", + "ฎ", + "ฏ", + "ฐ", + "ฑ", + "ฒ", + "ด", + "ต", + "ถ", + "ท", + "ธ", + "ศ", + "ษ", + "ส", + ]: + return "กด" + elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: + return "กน" + elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: + return "กบ" + else: + if "็" in word: + return "กา" + else: + return "Cant find Marttra in this word" + + def is_sumpus(self, word1: str, word2: str) -> bool: + """Check the rhyme between two words. + + :param str word1: Thai word + :param str word2: Thai word + :return: boolean + :rtype: bool + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.is_sumpus("สรร", "อัน")) + # output: True + + print(kv.is_sumpus("สรร", "แมว")) + # output: False + """ + marttra1 = self.check_marttra(word1) + marttra2 = self.check_marttra(word2) + sara1 = self.check_sara(word1) + sara2 = self.check_sara(word2) + if sara1 == "อะ" and marttra1 == "เกย": + sara1 = "ไอ" + marttra1 = "กา" + elif sara2 == "อะ" and marttra2 == "เกย": + sara2 = "ไอ" + marttra2 = "กา" + if sara1 == "อำ" and marttra1 == "กม": + sara1 = "อำ" + marttra1 = "กา" + elif sara2 == "อำ" and marttra2 == "กม": + sara2 = "อำ" + marttra2 = "กา" + return bool(marttra1 == marttra2 and sara1 == sara2) + + def check_karu_lahu(self, text): + if ( + self.check_marttra(text) != "กา" + or ( + self.check_marttra(text) == "กา" + and self.check_sara(text) + in [ + "อา", + "อี", + "อือ", + "อู", + "เอ", + "แอ", + "โอ", + "ออ", + "เออ", + "เอีย", + "เอือ", + "อัว", + ] + ) + or self.check_sara(text) in ["อำ", "ไอ", "เอา"] + ) and text not in ["บ่", "ณ", "ธ", "ก็"]: + return "karu" + else: + return "lahu" + + def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: + """Check the suitability of the poem according to Thai principles. + + :param str text: Thai poem + :param int k_type: type of Thai poem + :return: the check results of the suitability of the poem according to Thai principles. + :rtype: Union[List[str], str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ + มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: The poem is correct according to the principle. + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ + เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: [ + "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", + "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" + ] + """ + if k_type == 8: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 10: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 10 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append( + [ + sub_sent[1], + sub_sent[2], + sub_sent[3], + sub_sent[4], + ] + ) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 3: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return ("Something went wrong. " + "Make sure you enter it in the correct form of klon 8.") + elif k_type == 4: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 5: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 4 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 1: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form." + + else: + return "Something went wrong. Make sure you enter it in the correct form." + + def check_aek_too( + self, text: list[str] | str, dead_syllable_as_aek: bool = False + ) -> list[bool] | list[str] | bool | str: + """Checker of Thai tonal words + + :param Union[List[str], str] text: Thai word or list of Thai words + :param bool dead_syllable_as_aek: if True, dead syllable will be + considered as aek + :return: the check result if the word is aek or too or False (not both) + or list of check results if input is list + :rtype: Union[List[bool], List[str], bool, str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + # การเช็คคำเอกโท + print( + kv.check_aek_too("เอง"), + kv.check_aek_too("เอ่ง"), + kv.check_aek_too("เอ้ง"), + ) + # -> False, aek, too + print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน + # -> [False, 'aek', 'too'] + + + """ + if isinstance(text, list): + return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] + + if not isinstance(text, str): + raise TypeError("text must be str or iterable list[str]") + + word_characters = [*text] + if "่" in word_characters and "้" not in word_characters: + return "aek" + elif "้" in word_characters and "่" not in word_characters: + return "too" + if dead_syllable_as_aek and sound_syllable(text) == "dead": + return "aek" + else: + return False + + def handle_karun_sound_silence(self, word: str) -> str: + """Handle silent sounds in Thai words using '์' character (Karun) + by stripping all characters before the 'Karun' character that should be silenced + + :param str text: Thai word + :return: Thai word with silent words stripped + :rtype: str + """ + sound_silenced = word.endswith("์") + if not sound_silenced: + return word + # Remove ์ and the silent consonant before it + # การันต์ (์) marks the consonant immediately before it as silent + word = word[:-2] + return word diff --git a/pythainlp/lm/__init__.py b/pythainlp/lm/__init__.py index 60d8b503c..259f101d2 100644 --- a/pythainlp/lm/__init__.py +++ b/pythainlp/lm/__init__.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -__all__ = [ - "calculate_ngram_counts", - "remove_repeated_ngrams" -] +__all__ = ["calculate_ngram_counts", "remove_repeated_ngrams"] -from pythainlp.lm.text_util import calculate_ngram_counts, remove_repeated_ngrams +from pythainlp.lm.text_util import ( + calculate_ngram_counts, + remove_repeated_ngrams, +) diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 946fd8451..26fd4fa94 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -1,18 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 # ruff: noqa: C901 - -from typing import List, Tuple, Dict +from __future__ import annotations def calculate_ngram_counts( - list_words: List[str], - n_min: int = 2, - n_max: int = 4) -> Dict[Tuple[str], int]: - """ - Calculates the counts of n-grams in the list words for the specified range. + list_words: list[str], n_min: int = 2, n_max: int = 4 +) -> dict[tuple[str], int]: + """Calculates the counts of n-grams in the list words for the specified range. :param List[str] list_words: List of string :param int n_min: The minimum n-gram size (default: 2). @@ -21,20 +17,18 @@ def calculate_ngram_counts( :return: A dictionary where keys are n-grams and values are their counts. :rtype: Dict[Tuple[str], int] """ - ngram_counts = {} for n in range(n_min, n_max + 1): for i in range(len(list_words) - n + 1): - ngram = tuple(list_words[i:i + n]) + ngram = tuple(list_words[i : i + n]) ngram_counts[ngram] = ngram_counts.get(ngram, 0) + 1 return ngram_counts -def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: - """ - Remove repeated n-grams +def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]: + """Remove repeated n-grams :param List[str] string_list: List of string :param int n: n-gram size @@ -46,7 +40,7 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: from pythainlp.lm import remove_repeated_ngrams - remove_repeated_ngrams(['เอา', 'เอา', 'แบบ', 'ไหน'], n=1) + remove_repeated_ngrams(["เอา", "เอา", "แบบ", "ไหน"], n=1) # output: ['เอา', 'แบบ', 'ไหน'] """ if not string_list or n <= 0: @@ -58,12 +52,14 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: for i in range(len(string_list)): if i + n <= len(string_list): - ngram = tuple(string_list[i:i + n]) + ngram = tuple(string_list[i : i + n]) if ngram not in unique_ngrams: unique_ngrams.add(ngram) - if not output_list or output_list[-(n - 1):] != list(ngram[:-1]): + if not output_list or output_list[-(n - 1) :] != list( + ngram[:-1] + ): output_list.extend(ngram) else: output_list.append(ngram[-1]) diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py index d40baa777..b46042636 100644 --- a/pythainlp/morpheme/__init__.py +++ b/pythainlp/morpheme/__init__.py @@ -1,14 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""PyThaiNLP morpheme """ -PyThaiNLP morpheme -""" -__all__ = [ - "nighit", - "is_native_thai" -] + +__all__ = ["nighit", "is_native_thai"] from pythainlp.morpheme.thaiwordcheck import is_native_thai from pythainlp.morpheme.word_formation import nighit diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py index 61c02baff..6891b1cf0 100644 --- a/pythainlp/morpheme/thaiwordcheck.py +++ b/pythainlp/morpheme/thaiwordcheck.py @@ -1,18 +1,21 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Check if a word is a "native Thai word" +"""Check if a word is a "native Thai word" Adapted from https://github.com/wannaphong/open-thai-nlp-document/blob/master/check_thai_word.md References +---------- - ทีมงานทรูปลูกปัญญา 2015. ลักษณะของคำไทยแท้ \ http://www.trueplookpanya.com/learning/detail/30589-043067 - วารุณี บำรุงรส 2010. คำไทยแท้ https://www.gotoknow.org/posts/377619 + """ + +from __future__ import annotations + import re _THANTHAKHAT_CHAR = "\u0e4c" # Thanthakhat (cancellation of sound) @@ -65,8 +68,7 @@ def is_native_thai(word: str) -> bool: - """ - Check if a word is an "native Thai word" (Thai: "คำไทยแท้") + """Check if a word is an "native Thai word" (Thai: "คำไทยแท้") This function is based on a simple heuristic algorithm and cannot be entirely reliable. diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index ede6c248d..212c3e23a 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp import thai_consonants def nighit(w1: str, w2: str) -> str: - """ - Nighit (นิคหิต or ํ ) is the niggahita in Thai language for create new \ + """Nighit (นิคหิต or ํ ) is the niggahita in Thai language for create new \ words from Pali language in Thai. The function use simple method to create new Thai word from two words \ that the root is from Pali language. @@ -31,7 +31,7 @@ def nighit(w1: str, w2: str) -> str: assert nighit("สํ","ปทา")=="สัมปทา" assert nighit("สํ","โยค")=="สังโยค" """ - if not str(w1).endswith('ํ') and len(w1) != 2: + if not str(w1).endswith("ํ") and len(w1) != 2: raise NotImplementedError(f"The function doesn't support {w1}.") list_w1 = list(w1) list_w2 = list(w2) @@ -56,4 +56,4 @@ def nighit(w1: str, w2: str) -> str: The function doesn't support {w1} and {w2}. """) newword.extend(list_w2) - return ''.join(newword) + return "".join(newword) diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py index d7aae6f1c..01fbdada9 100644 --- a/pythainlp/parse/__init__.py +++ b/pythainlp/parse/__init__.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""PyThaiNLP Parse """ -PyThaiNLP Parse -""" + __all__ = ["dependency_parsing"] from pythainlp.parse.core import dependency_parsing diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index 2a64b4bd5..619a5a8df 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - -from typing import List, Union +from __future__ import annotations _tagger = None _tagger_name = "" @@ -11,12 +9,11 @@ def dependency_parsing( text: str, - model: Union[str, None] = None, + model: str | None = None, tag: str = "str", engine: str = "esupar", -) -> Union[List[List[str]], str]: - """ - Dependency Parsing +) -> list[list[str]] | str: + """Dependency Parsing :param str text: text to apply dependency parsing to :param str model: model for using with engine \ diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 259cb00b7..232051a16 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- -""" -esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages +"""esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages GitHub: https://github.com/KoichiYasuoka/esupar """ -from typing import List, Union + +from __future__ import annotations try: import esupar @@ -18,9 +17,7 @@ def __init__(self, model: str = "th") -> None: model = "th" self.nlp = esupar.load(model) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: _data = str(self.nlp(text)) if tag == "list": _temp = _data.splitlines() diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py index 6d0eef7ab..30aeb0112 100644 --- a/pythainlp/parse/spacy_thai_engine.py +++ b/pythainlp/parse/spacy_thai_engine.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- -""" -spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies. +"""spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies. GitHub: https://github.com/KoichiYasuoka/spacy-thai """ -from typing import List, Union + +from __future__ import annotations import spacy_thai @@ -14,9 +13,7 @@ class Parse: def __init__(self, model: str = "th") -> None: self.nlp = spacy_thai.load() - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: doc = self.nlp(text) _text = [] if tag == "list": diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 9d1ffca5b..71737ae2c 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -""" -TransformersUD +"""TransformersUD Author: Prof. Koichi Yasuoka @@ -10,8 +8,10 @@ GitHub: https://github.com/KoichiYasuoka """ + +from __future__ import annotations + import os -from typing import List, Union import numpy import torch @@ -56,9 +56,7 @@ def __init__( model=t, tokenizer=self.tokenizer ) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: w = [ (t["start"], t["end"], t["entity_group"]) for t in self.deprel(text) diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index fc258d41d..05dabb59e 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -""" -UDgoeswith +"""UDgoeswith Author: Prof. Koichi Yasuoka @@ -10,7 +8,8 @@ GitHub: https://github.com/KoichiYasuoka """ -from typing import List, Union + +from __future__ import annotations import numpy as np import torch @@ -27,9 +26,7 @@ def __init__( self.tokenizer = AutoTokenizer.from_pretrained(model) self.model = AutoModelForTokenClassification.from_pretrained(model) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: w = self.tokenizer(text, return_offsets_mapping=True) v = w["input_ids"] x = [ diff --git a/pythainlp/phayathaibert/__init__.py b/pythainlp/phayathaibert/__init__.py index 2d50efabd..c2a2e24b2 100644 --- a/pythainlp/phayathaibert/__init__.py +++ b/pythainlp/phayathaibert/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PhayaThaiBERT +"""PhayaThaiBERT """ __all__ = [ diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 5631b766c..0619b16b0 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import random import re import warnings -from typing import Callable, List, Tuple, Union +from collections.abc import Callable from transformers import ( CamembertTokenizer, @@ -32,8 +32,7 @@ def __init__(self): self.SPACE_SPECIAL_TOKEN = "<_>" def replace_url(self, text: str) -> str: - """ - Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) + """Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) :param str text: text to replace url :return: text where urls are replaced :rtype: str @@ -44,8 +43,7 @@ def replace_url(self, text: str) -> str: return re.sub(_PAT_URL, self._TK_URL, text) def rm_brackets(self, text: str) -> str: - """ - Remove all empty brackets and artifacts within brackets from `text`. + """Remove all empty brackets and artifacts within brackets from `text`. :param str text: text to remove useless brackets :return: text where all useless brackets are removed :rtype: str @@ -84,8 +82,7 @@ def rm_brackets(self, text: str) -> str: return new_line def replace_newlines(self, text: str) -> str: - """ - Replace newlines in `text` with spaces. + """Replace newlines in `text` with spaces. :param str text: text to replace all newlines with spaces :return: text where all newlines are replaced with spaces :rtype: str @@ -93,12 +90,10 @@ def replace_newlines(self, text: str) -> str: >>> rm_useless_spaces("hey whats\n\nup") hey whats up """ - return re.sub(r"[\n]", " ", text.strip()) def rm_useless_spaces(self, text: str) -> str: - """ - Remove multiple spaces in `text`. (code from `fastai`) + """Remove multiple spaces in `text`. (code from `fastai`) :param str text: text to replace useless spaces :return: text where all spaces are reduced to one :rtype: str @@ -109,8 +104,7 @@ def rm_useless_spaces(self, text: str) -> str: return re.sub(" {2,}", " ", text) def replace_spaces(self, text: str, space_token: str = "<_>") -> str: - """ - Replace spaces with _ + """Replace spaces with _ :param str text: text to replace spaces :return: text where all spaces replaced with _ :rtype: str @@ -121,8 +115,7 @@ def replace_spaces(self, text: str, space_token: str = "<_>") -> str: return re.sub(" ", space_token, text) def replace_rep_after(self, text: str) -> str: - """ - Replace repetitions at the character level in `text` + """Replace repetitions at the character level in `text` :param str text: input text to replace character repetition :return: text with repetitive tokens removed. :rtype: str @@ -139,9 +132,8 @@ def _replace_rep(m): re_rep = re.compile(r"(\S)(\1{3,})") return re_rep.sub(_replace_rep, text) - def replace_wrep_post(self, toks: List[str]) -> List[str]: - """ - Replace repetitive words post tokenization; + def replace_wrep_post(self, toks: list[str]) -> list[str]: + """Replace repetitive words post tokenization; fastai `replace_wrep` does not work well with Thai. :param List[str] toks: list of tokens :return: list of tokens where repetitive words are removed. @@ -166,9 +158,8 @@ def replace_wrep_post(self, toks: List[str]) -> List[str]: return res[1:] - def remove_space(self, toks: List[str]) -> List[str]: - """ - Do not include space for bag-of-word models. + def remove_space(self, toks: list[str]) -> list[str]: + """Do not include space for bag-of-word models. :param List[str] toks: list of tokens :return: List of tokens where space tokens (" ") are filtered out :rtype: List[str] @@ -189,7 +180,7 @@ def remove_space(self, toks: List[str]) -> List[str]: def preprocess( self, text: str, - pre_rules: List[Callable] = [ + pre_rules: list[Callable] = [ rm_brackets, replace_newlines, rm_useless_spaces, @@ -253,9 +244,8 @@ def augment( text: str, num_augs: int = 3, sample: bool = False, - ) -> List[str]: - """ - Text augmentation from PhayaThaiBERT + ) -> list[str]: + """Text augmentation from PhayaThaiBERT :param str text: Thai text :param int num_augs: an amount of augmentation text needed as an output @@ -315,9 +305,8 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: def get_tag( self, sentence: str, strategy: str = "simple" - ) -> List[List[Tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. + ) -> list[list[tuple[str, str]]]: + """Marks sentences with part-of-speech (POS) tags. :param str sentence: a list of lists of tokenized words :return: a list of lists of tuples (word, POS tag) @@ -363,9 +352,8 @@ def get_ner( tag: bool = False, pos: bool = False, strategy: str = "simple", - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: - """ - This function tags named entities in text in IOB format. + ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: + """This function tags named entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: output with part-of-speech tags.\ @@ -435,9 +423,8 @@ def get_ner( return sample_output -def segment(sentence: str) -> List[str]: - """ - Subword tokenize of PhayaThaiBERT, \ +def segment(sentence: str) -> list[str]: + """Subword tokenize of PhayaThaiBERT, \ sentencepiece from WangchanBERTa model with vocabulary expansion. :param str sentence: text to be tokenized diff --git a/pythainlp/soundex/__init__.py b/pythainlp/soundex/__init__.py index ed924db9f..4e002ebcd 100644 --- a/pythainlp/soundex/__init__.py +++ b/pythainlp/soundex/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex +"""Thai soundex Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ diff --git a/pythainlp/soundex/core.py b/pythainlp/soundex/core.py index 8ecbcb1b7..5446c3d77 100644 --- a/pythainlp/soundex/core.py +++ b/pythainlp/soundex/core.py @@ -1,12 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex +"""Thai soundex Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ + +from __future__ import annotations + from pythainlp.soundex import DEFAULT_SOUNDEX_ENGINE from pythainlp.soundex.lk82 import lk82 from pythainlp.soundex.metasound import metasound @@ -20,8 +21,7 @@ def soundex( text: str, engine: str = DEFAULT_SOUNDEX_ENGINE, length: int = 4 ) -> str: - """ - This function converts Thai text into phonetic code. + """This function converts Thai text into phonetic code. :param str text: word :param str engine: soundex engine diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py index fd3010e69..47f8aaddc 100644 --- a/pythainlp/soundex/lk82.py +++ b/pythainlp/soundex/lk82.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - LK82 system +"""Thai soundex - LK82 system Original paper: Vichit Lorchirachoonkul. 1982. A Thai soundex @@ -15,6 +13,9 @@ by Korakot Chaovavanich https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8 """ + +from __future__ import annotations + import re from pythainlp.util import remove_tonemark @@ -37,8 +38,7 @@ def lk82(text: str) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the Thai soundex algorithm named **LK82** [#lk82]_. :param str text: Thai word diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py index 7729d520c..ef764eb52 100644 --- a/pythainlp/soundex/metasound.py +++ b/pythainlp/soundex/metasound.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - MetaSound system +"""Thai soundex - MetaSound system References: Snae & Brückner. (2009). Novel Phonetic Name Matching Algorithm with a Statistical Ontology for Analysing Names Given in Accordance with Thai Astrology. https://pdfs.semanticscholar.org/3983/963e87ddc6dfdbb291099aa3927a0e3e4ea6.pdf + """ +from __future__ import annotations + _CONS_THANTHAKHAT = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ์" _THANTHAKHAT = "์" # \u0e4c _C1 = "กขฃคฆฅ" # sound K -> coded letter 1 @@ -25,8 +26,7 @@ def metasound(text: str, length: int = 4) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the matching technique called **MetaSound** [#metasound]_ (combination between Soundex and Metaphone algorithms). MetaSound algorithm was developed specifically for the Thai language. diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py index 5ba0908d7..8adea3142 100644 --- a/pythainlp/soundex/prayut_and_somchaip.py +++ b/pythainlp/soundex/prayut_and_somchaip.py @@ -1,14 +1,19 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-English Cross-Language Transliterated Word Retrieval +"""Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique References: -Prayut Suwanvisat, Somchai Prasitjutrakul.Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf +Prayut Suwanvisat, Somchai Prasitjutrakul. +Thai-English Cross-Language Transliterated Word Retrieval using Soundex +Technique. In 1998 [cited 2022 Sep 8]. +Available from: +https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf """ + +from __future__ import annotations + from pythainlp import thai_characters _C0 = "AEIOUHWYอ" @@ -26,8 +31,7 @@ def prayut_and_somchaip(text: str, length: int = 4) -> str: - """ - This function converts English-Thai Cross-Language Transliterated Word into + """This function converts English-Thai Cross-Language Transliterated Word into phonetic code with the matching technique called **Soundex** [#prayut_and_somchaip]_. :param str text: English-Thai Cross-Language Transliterated Word diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py index 355949d90..6cf5e9924 100644 --- a/pythainlp/soundex/sound.py +++ b/pythainlp/soundex/sound.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations import panphon import panphon.distance @@ -13,19 +12,29 @@ _ft = panphon.FeatureTable() _dst = panphon.distance.Distance() + def _clean_ipa(ipa: str) -> str: - """ - Clean IPA by removing tones and space between phonetic codes + """Clean IPA by removing tones and space between phonetic codes :param str ipa: IPA text :return: IPA with tones removed from the text :rtype: str """ - return ipa.replace("˩˩˦","").replace("˥˩","").replace("˨˩","").replace("˦˥","").replace("˧","").replace("˧","").replace(" .",".").replace(". ",".").strip() + return ( + ipa.replace("˩˩˦", "") + .replace("˥˩", "") + .replace("˨˩", "") + .replace("˦˥", "") + .replace("˧", "") + .replace("˧", "") + .replace(" .", ".") + .replace(". ", ".") + .strip() + ) + def word2audio(word: str) -> str: - """ - Convert word to IPA + """Convert word to IPA :param str word: Thai word :return: IPA with tones removed from the text @@ -41,12 +50,14 @@ def word2audio(word: str) -> str: """ _word = word_tokenize(word) _phone = [pronunciate(w, engine="w2p") for w in _word] - _ipa = [_clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone] - return '.'.join(_ipa) + _ipa = [ + _clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone + ] + return ".".join(_ipa) -def audio_vector(word: str) -> List[List[int]]: - """ - Convert audio to vector list + +def audio_vector(word: str) -> list[list[int]]: + """Convert audio to vector list :param str word: Thai word :return: List of features from panphon @@ -62,9 +73,9 @@ def audio_vector(word: str) -> List[List[int]]: """ return _ft.word_to_vector_list(word2audio(word), numeric=True) -def word_approximation(word: str, list_word: List[str]) -> List[float]: - """ - Thai Word Approximation + +def word_approximation(word: str, list_word: list[str]) -> list[float]: + """Thai Word Approximation :param str word: Thai word :param str list_word: Thai word @@ -81,5 +92,7 @@ def word_approximation(word: str, list_word: List[str]) -> List[float]: """ _word = word2audio(word) _list_word = [word2audio(w) for w in list_word] - _distance = [_dst.weighted_feature_edit_distance(_word, w) for w in _list_word] + _distance = [ + _dst.weighted_feature_edit_distance(_word, w) for w in _list_word + ] return _distance diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py index 7b401bb01..cfb6a72f8 100644 --- a/pythainlp/soundex/udom83.py +++ b/pythainlp/soundex/udom83.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - Udom83 system +"""Thai soundex - Udom83 system Original paper: Wannee Udompanich. String searching for Thai alphabet @@ -16,6 +14,9 @@ by Korakot Chaovavanich https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8 """ + +from __future__ import annotations + import re from pythainlp import thai_consonants @@ -48,8 +49,7 @@ def udom83(text: str) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the Thai soundex algorithm named **Udom83** [#udom83]_. :param str text: Thai word @@ -77,7 +77,6 @@ def udom83(text: str) -> str: udom83("ปัจจุบัน") # output: 'ป775300' """ - if not text or not isinstance(text, str): return "" diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 33bee579e..8850ceb62 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Spell checking and correction. +"""Spell checking and correction. """ __all__ = [ @@ -18,8 +16,10 @@ from pythainlp.spell.pn import NorvigSpellChecker -DEFAULT_SPELL_CHECKER = NorvigSpellChecker() +DEFAULT_SPELL_CHECKER = NorvigSpellChecker # these imports are placed here to avoid circular imports from pythainlp.spell.core import correct, correct_sent, spell, spell_sent -from pythainlp.spell.words_spelling_correction import get_words_spell_suggestion +from pythainlp.spell.words_spelling_correction import ( + get_words_spell_suggestion, +) diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index 8c874d5ce..2cca2dc81 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -1,20 +1,25 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Spell checking functions """ -Spell checking functions -""" + +from __future__ import annotations import itertools -from typing import List +from functools import lru_cache from pythainlp.spell import DEFAULT_SPELL_CHECKER -def spell(word: str, engine: str = "pn") -> List[str]: - """ - Provides a list of possible correct spellings of the given word. +@lru_cache +def default_spell_checker(): + """Lazy load default spell checker with cache""" + return DEFAULT_SPELL_CHECKER() + + +def spell(word: str, engine: str = "pn") -> list[str]: + """Provides a list of possible correct spellings of the given word. The list of words are from the words in the dictionary that incurs an edit distance value of 1 or 2. The result is a list of words sorted by their occurrences @@ -37,13 +42,13 @@ def spell(word: str, engine: str = "pn") -> List[str]: from pythainlp.spell import spell - spell("เส้นตรบ", engine="pn") + spell("เส้นตรบ", engine="pn") # output: ['เส้นตรง'] spell("เส้นตรบ") # output: ['เส้นตรง'] - spell("เส้นตรบ", engine="tltk") + spell("เส้นตรบ", engine="tltk") # output: ['เส้นตรง'] spell("ครัช") @@ -72,14 +77,13 @@ def spell(word: str, engine: str = "pn") -> List[str]: text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.spell(word) + text_correct = default_spell_checker().spell(word) return text_correct def correct(word: str, engine: str = "pn") -> str: - """ - Corrects the spelling of the given word by returning + """Corrects the spelling of the given word by returning the correctly spelled word. :param str word: word to correct spelling of @@ -120,19 +124,20 @@ def correct(word: str, engine: str = "pn") -> str: text_correct = SPELL_CHECKER(word) elif engine == "wanchanberta_thai_grammarly": - from pythainlp.spell.wanchanberta_thai_grammarly import correct as SPELL_CHECKER + from pythainlp.spell.wanchanberta_thai_grammarly import ( + correct as SPELL_CHECKER, + ) text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.correct(word) + text_correct = default_spell_checker().correct(word) return text_correct -def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: - """ - Provides a list of possible correct spellings of sentence +def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: + """Provides a list of possible correct spellings of sentence :param List[str] list_words: list of words in sentence :param str engine: @@ -147,7 +152,7 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: from pythainlp.spell import spell_sent - spell_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy') + spell_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # output: [['เด็ก', 'อินเทอร์เน็ต', 'แรง']] """ if engine == "symspellpy": @@ -168,9 +173,8 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: return list_new -def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]: - """ - Corrects and returns the spelling of the given sentence +def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]: + """Corrects and returns the spelling of the given sentence :param List[str] list_words: list of words in sentence :param str engine: @@ -186,7 +190,7 @@ def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]: from pythainlp.spell import correct_sent - correct_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy') + correct_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # output: ['เด็ก', 'อินเทอร์เน็ต', 'แรง'] """ return spell_sent(list_words, engine=engine)[0] diff --git a/pythainlp/spell/phunspell.py b/pythainlp/spell/phunspell.py index 7a293816d..c42fd54e3 100644 --- a/pythainlp/spell/phunspell.py +++ b/pythainlp/spell/phunspell.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Phunspell +"""Phunspell A pure Python spell checker utilizing spylls, a port of Hunspell. @@ -11,17 +9,20 @@ * \ https://github.com/dvwright/phunspell """ -from typing import List + +from __future__ import annotations try: import phunspell except ImportError: - raise ImportError("Import Error; Install phunspell by pip install phunspell") + raise ImportError( + "Import Error; Install phunspell by pip install phunspell" + ) pspell = phunspell.Phunspell("th_TH") -def spell(text: str) -> List[str]: +def spell(text: str) -> list[str]: return list(pspell.suggest(text)) diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 4cdfab5d7..695bc7cf1 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -1,24 +1,15 @@ -# -*- coding: utf-8 -*- -""" -Spell checker, using Peter Norvig algorithm. +"""Spell checker, using Peter Norvig algorithm. Spelling dictionary can be customized. Default spelling dictionary is based on Thai National Corpus. Based on Peter Norvig's Python code from http://norvig.com/spell-correct.html """ + +from __future__ import annotations + from collections import Counter +from collections.abc import Callable, ItemsView, Iterable from string import digits -from typing import ( - Callable, - Dict, - ItemsView, - Iterable, - List, - Optional, - Set, - Tuple, - Union, -) from pythainlp import thai_digits, thai_letters from pythainlp.corpus import tnc @@ -39,14 +30,13 @@ def _is_thai_and_not_num(word: str) -> bool: def _keep( - word_freq: Tuple[str, int], + word_freq: tuple[str, int], min_freq: int, min_len: int, max_len: int, dict_filter: Callable[[str], bool], ) -> bool: - """ - Checks whether a given word has the required minimum frequency min_freq + """Checks whether a given word has the required minimum frequency min_freq and its character length is between min_len and max_len (inclusive). """ if not word_freq or word_freq[1] < min_freq: @@ -59,9 +49,8 @@ def _keep( return dict_filter(word) -def _edits1(word: str) -> Set[str]: - """ - Returns a set of words with an edit distance of 1 from the input word +def _edits1(word: str) -> set[str]: + """Returns a set of words with an edit distance of 1 from the input word """ splits = [(word[:i], word[i:]) for i in range(len(word) + 1)] deletes = [L + R[1:] for L, R in splits if R] @@ -72,24 +61,20 @@ def _edits1(word: str) -> Set[str]: return set(deletes + transposes + replaces + inserts) -def _edits2(word: str) -> Set[str]: - """ - Returns a set of words with an edit distance of 2 from the input word +def _edits2(word: str) -> set[str]: + """Returns a set of words with an edit distance of 2 from the input word """ return set(e2 for e1 in _edits1(word) for e2 in _edits1(e1)) def _convert_custom_dict( - custom_dict: Union[ - Dict[str, int], Iterable[str], Iterable[Tuple[str, int]] - ], + custom_dict: dict[str, int] | Iterable[str] | Iterable[tuple[str, int]], min_freq: int, min_len: int, max_len: int, - dict_filter: Optional[Callable[[str], bool]], -) -> List[Tuple[str, int]]: - """ - Converts a custom dictionary to a list of (str, int) tuples + dict_filter: Callable[[str], bool] | None, +) -> list[tuple[str, int]]: + """Converts a custom dictionary to a list of (str, int) tuples """ if isinstance(custom_dict, dict): custom_dict = list(custom_dict.items()) @@ -123,16 +108,15 @@ def _convert_custom_dict( class NorvigSpellChecker: def __init__( self, - custom_dict: Union[ - Dict[str, int], Iterable[str], Iterable[Tuple[str, int]] - ] = None, + custom_dict: dict[str, int] + | Iterable[str] + | Iterable[tuple[str, int]] = None, min_freq: int = 2, min_len: int = 2, max_len: int = 40, - dict_filter: Optional[Callable[[str], bool]] = _is_thai_and_not_num, + dict_filter: Callable[[str], bool] | None = _is_thai_and_not_num, ): - """ - Initializes Peter Norvig's spell checker object. + """Initializes Peter Norvig's spell checker object. Spelling dictionary can be customized. By default, spelling dictionary is from `Thai National Corpus `_ @@ -180,8 +164,7 @@ def __init__( self.__WORDS_TOTAL = sum(self.__WORDS.values()) def dictionary(self) -> ItemsView[str, int]: - """ - Returns the spelling dictionary currently used by this spell checker + """Returns the spelling dictionary currently used by this spell checker :return: spelling dictionary of this instance :rtype: list[tuple[str, int]] @@ -191,7 +174,7 @@ def dictionary(self) -> ItemsView[str, int]: from pythainlp.spell import NorvigSpellChecker - dictionary= [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] + dictionary = [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] checker = NorvigSpellChecker(custom_dict=dictionary) checker.dictionary() @@ -199,9 +182,8 @@ def dictionary(self) -> ItemsView[str, int]: """ return self.__WORDS.items() - def known(self, words: Iterable[str]) -> List[str]: - """ - Returns a list of given words found in the spelling dictionary + def known(self, words: Iterable[str]) -> list[str]: + """Returns a list of given words found in the spelling dictionary :param list[str] words: A list of words to check if they exist in the spelling dictionary @@ -220,7 +202,7 @@ def known(self, words: Iterable[str]) -> List[str]: checker.known(["เพยน", "เพล", "เพลง"]) # output: ['เพล', 'เพลง'] - checker.known(['ยกไ', 'ไฟล์ม']) + checker.known(["ยกไ", "ไฟล์ม"]) # output: [] checker.known([]) @@ -229,8 +211,7 @@ def known(self, words: Iterable[str]) -> List[str]: return list(w for w in words if w in self.__WORDS) def prob(self, word: str) -> float: - """ - Returns the probability of an input word, + """Returns the probability of an input word, according to the spelling dictionary :param str word: A word to check occurrence probability of @@ -257,8 +238,7 @@ def prob(self, word: str) -> float: return self.__WORDS[word] / self.__WORDS_TOTAL def freq(self, word: str) -> int: - """ - Returns the frequency of an input word, + """Returns the frequency of an input word, according to the spelling dictionary :param str word: A word to check frequency of @@ -280,9 +260,8 @@ def freq(self, word: str) -> int: """ return self.__WORDS[word] - def spell(self, word: str) -> List[str]: - """ - Returns a list of all correctly-spelled words whose spelling + def spell(self, word: str) -> list[str]: + """Returns a list of all correctly-spelled words whose spelling is similar to the given word by edit distance metrics. The returned list of words will be sorted by decreasing order of word frequencies in the word spelling dictionary. @@ -331,8 +310,7 @@ def spell(self, word: str) -> List[str]: return candidates def correct(self, word: str) -> str: - """ - Returns the most possible word, using the probability from + """Returns the most possible word, using the probability from the spelling dictionary :param str word: A word to correct spelling of diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index b3c5abb81..8ba10fa22 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -symspellpy +"""symspellpy symspellpy is a Python port of SymSpell v6.5. We used unigram & bigram from Thai National Corpus (TNC). @@ -13,12 +11,14 @@ https://github.com/mammothb/symspellpy """ -from typing import List +from __future__ import annotations try: from symspellpy import SymSpell, Verbosity except ImportError: - raise ImportError("Import Error; Install symspellpy by pip install symspellpy") + raise ImportError( + "Import Error; Install symspellpy by pip install symspellpy" + ) from pythainlp.corpus import get_corpus_path, path_pythainlp_corpus @@ -42,7 +42,7 @@ ) -def spell(text: str, max_edit_distance: int = 2) -> List[str]: +def spell(text: str, max_edit_distance: int = 2) -> list[str]: return [ str(i).split(",", maxsplit=1)[0] for i in list( @@ -58,8 +58,8 @@ def correct(text: str, max_edit_distance: int = 1) -> str: def spell_sent( - list_words: List[str], max_edit_distance: int = 2 -) -> List[List[str]]: + list_words: list[str], max_edit_distance: int = 2 +) -> list[list[str]]: temp = [ str(i).split(",", maxsplit=1)[0].split(" ") for i in list( @@ -77,7 +77,7 @@ def spell_sent( return list_new -def correct_sent(list_words: List[str], max_edit_distance=1) -> List[str]: +def correct_sent(list_words: list[str], max_edit_distance=1) -> list[str]: return [ i[0] for i in spell_sent(list_words, max_edit_distance=max_edit_distance) diff --git a/pythainlp/spell/tltk.py b/pythainlp/spell/tltk.py index dd120aa3d..225a20c02 100644 --- a/pythainlp/spell/tltk.py +++ b/pythainlp/spell/tltk.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -TLTK +"""TLTK Thai Language Toolkit @@ -11,12 +9,16 @@ * \ https://pypi.org/project/tltk/ """ + +from __future__ import annotations + try: from tltk.nlp import spell_candidates except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") -from typing import List + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) -def spell(text: str) -> List[str]: +def spell(text: str) -> list[str]: return spell_candidates(text) diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 2707467f8..116b949e7 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Two-stage Thai Misspelling Correction based on Pre-trained Language Models +"""Two-stage Thai Misspelling Correction based on Pre-trained Language Models :See Also: * Paper: \ @@ -11,7 +9,9 @@ * GitHub: \ https://github.com/bookpanda/Two-stage-Thai-Misspelling-Correction-Based-on-Pre-trained-Language-Models """ -from typing import List + +from __future__ import annotations + import torch from transformers import ( AutoModelForMaskedLM, @@ -21,28 +21,41 @@ use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") -tokenizer = AutoTokenizer.from_pretrained("airesearch/wangchanberta-base-att-spm-uncased") +tokenizer = AutoTokenizer.from_pretrained( + "airesearch/wangchanberta-base-att-spm-uncased" +) + class BertModel(torch.nn.Module): def __init__(self): super().__init__() - self.bert = BertForTokenClassification.from_pretrained('bookpanda/wangchanberta-base-att-spm-uncased-tagging') + self.bert = BertForTokenClassification.from_pretrained( + "bookpanda/wangchanberta-base-att-spm-uncased-tagging" + ) def forward(self, input_id, mask, label): - output = self.bert(input_ids=input_id, attention_mask=mask, labels=label, return_dict=False) + output = self.bert( + input_ids=input_id, + attention_mask=mask, + labels=label, + return_dict=False, + ) return output + tagging_model = BertModel() if use_cuda: tagging_model = tagging_model.to(device=device) -ids_to_labels = {0: 'f', 1: 'i'} +ids_to_labels = {0: "f", 1: "i"} -def align_word_ids(texts: str) -> List[int]: - tokenized_inputs = tokenizer(texts, padding='max_length', max_length=512, truncation=True) + +def align_word_ids(texts: str) -> list[int]: + tokenized_inputs = tokenizer( + texts, padding="max_length", max_length=512, truncation=True + ) word_ids = tokenized_inputs.word_ids() label_ids = [] for word_idx in word_ids: - if word_idx is None: label_ids.append(-100) else: @@ -53,10 +66,17 @@ def align_word_ids(texts: str) -> List[int]: return label_ids + def evaluate_one_text(model, sentence): - text = tokenizer(sentence, padding='max_length', max_length = 512, truncation=True, return_tensors="pt") - mask = text['attention_mask'][0].unsqueeze(0).to(device) - input_id = text['input_ids'][0].unsqueeze(0).to(device) + text = tokenizer( + sentence, + padding="max_length", + max_length=512, + truncation=True, + return_tensors="pt", + ) + mask = text["attention_mask"][0].unsqueeze(0).to(device) + input_id = text["input_ids"][0].unsqueeze(0).to(device) label_ids = torch.Tensor(align_word_ids(sentence)).unsqueeze(0).to(device) logits = tagging_model(input_id, mask, None) @@ -67,39 +87,53 @@ def evaluate_one_text(model, sentence): return prediction_label -mlm_model = AutoModelForMaskedLM.from_pretrained("bookpanda/wangchanberta-base-att-spm-uncased-masking") +mlm_model = AutoModelForMaskedLM.from_pretrained( + "bookpanda/wangchanberta-base-att-spm-uncased-masking" +) if use_cuda: mlm_model = mlm_model.to(device=device) + def correct(text: str) -> str: ans = [] i_f = evaluate_one_text(tagging_model, text) a = tokenizer(text) i_f_len = len(i_f) for j in range(i_f_len): - if i_f[j] == 'i': - ph = a['input_ids'][j+1] - a['input_ids'][j+1] = 25004 - b = {'input_ids': torch.Tensor([a['input_ids']]).type(torch.int64).to(device), 'attention_mask': torch.Tensor([a['attention_mask']]).type(torch.int64).to(device)} + if i_f[j] == "i": + ph = a["input_ids"][j + 1] + a["input_ids"][j + 1] = 25004 + b = { + "input_ids": torch.Tensor([a["input_ids"]]) + .type(torch.int64) + .to(device), + "attention_mask": torch.Tensor([a["attention_mask"]]) + .type(torch.int64) + .to(device), + } token_logits = mlm_model(**b).logits - mask_token_index = torch.where(b["input_ids"] == tokenizer.mask_token_id)[1] + mask_token_index = torch.where( + b["input_ids"] == tokenizer.mask_token_id + )[1] mask_token_logits = token_logits[0, mask_token_index, :] - top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist() + top_5_tokens = ( + torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist() + ) ans.append((j, top_5_tokens[0])) - text = ''.join(tokenizer.convert_ids_to_tokens(a['input_ids'])) - a['input_ids'][j+1] = ph - for x,y in ans: - a['input_ids'][x+1] = y - final_output = tokenizer.convert_ids_to_tokens(a['input_ids']) + text = "".join(tokenizer.convert_ids_to_tokens(a["input_ids"])) + a["input_ids"][j + 1] = ph + for x, y in ans: + a["input_ids"][x + 1] = y + final_output = tokenizer.convert_ids_to_tokens(a["input_ids"]) if "" in final_output: final_output.remove("") if "" in final_output: final_output.remove("") if "" in final_output: final_output.remove("") - if final_output[0] == '▁': + if final_output[0] == "▁": final_output.pop(0) - final_output = ''.join(final_output) + final_output = "".join(final_output) final_output = final_output.replace("▁", " ") final_output = final_output.replace("", "") return final_output diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index 10f0d7991..92b4467c5 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -1,24 +1,32 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import os + from pythainlp.corpus import get_hf_hub -from typing import List, Union class FastTextEncoder: - """ - A class to load pre-trained FastText-like word embeddings, - compute word and sentence vectors, and interact with an ONNX + """A class to load pre-trained FastText-like word embeddings, + compute word and sentence vectors, and interact with an ONNX model for nearest neighbor suggestions. """ # --- Initialization and Data Loading --- - - def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_words=2000000, minn=5, maxn=5): - """ - Initializes the FastTextEncoder, loading embeddings, vocabulary, + + def __init__( + self, + model_dir, + nn_model_path, + words_list, + bucket=2000000, + nb_words=2000000, + minn=5, + maxn=5, + ): + """Initializes the FastTextEncoder, loading embeddings, vocabulary, nearest neighbor model, and suggestion words list. Args: @@ -29,17 +37,18 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word nb_words (int): The number of words in the vocabulary (used as an offset for subword indices). minn (int): Minimum character length for subwords. maxn (int): Maximum character length for subwords. + """ try: - import numpy as np # reduce load - import onnxruntime + import numpy as np + self.np = np except ModuleNotFoundError: raise ModuleNotFoundError(""" Please installing the package via 'pip install numpy onnxruntime'. """) except Exception as e: - raise Exception(f"An unexpected error occurred: {e}") + raise RuntimeError(f"An unexpected error occurred: {e}") from e self.model_dir = model_dir self.nn_model_path = nn_model_path self.bucket = bucket @@ -55,10 +64,12 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word def _load_embeddings(self): """Loads embeddings matrix and vocabulary list.""" - input_matrix = self.np.load(os.path.join(self.model_dir, "embeddings.npy")) + input_matrix = self.np.load( + os.path.join(self.model_dir, "embeddings.npy") + ) words = [] vocab_path = os.path.join(self.model_dir, "vocabulary.txt") - with open(vocab_path, "r", encoding='utf-8') as f: + with open(vocab_path, encoding="utf-8") as f: for line in f.readlines(): words.append(line.rstrip()) return words, input_matrix @@ -72,7 +83,10 @@ def _load_onnx_session(self, onnx_path): """Loads the ONNX inference session.""" # Note: Using providers=["CPUExecutionProvider"] for platform independence import onnxruntime as rt - sess = rt.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) + + sess = rt.InferenceSession( + onnx_path, providers=["CPUExecutionProvider"] + ) return sess # --- Helper Methods for Encoding --- @@ -103,9 +117,11 @@ def _get_subwords(self, word): for ngram_start in range(0, len(_word)): for ngram_length in range(self.minn, self.maxn + 1): if ngram_start + ngram_length <= len(_word): - _candidate_subword = _word[ngram_start:ngram_start + ngram_length] + _candidate_subword = _word[ + ngram_start : ngram_start + ngram_length + ] # Only append if not already included (e.g., as the full word) - if _candidate_subword not in _subwords: + if _candidate_subword not in _subwords: _subwords.append(_candidate_subword) _subword_ids.append(self._get_hash(_candidate_subword)) @@ -115,20 +131,22 @@ def get_word_vector(self, word): """Computes the normalized vector for a single word.""" # subword_ids[1] contains the array of indices for the word and its subwords subword_ids = self._get_subwords(word)[1] - + # Check if the array of subword indices is empty if subword_ids.size == 0: # Return a 300-dimensional zero vector if no word/subword is found. return self.np.zeros(self.embedding_dim) # Compute the mean of the embeddings for all subword indices - vector = self.np.mean([self.embeddings[s] for s in subword_ids], axis=0) - + vector = self.np.mean( + [self.embeddings[s] for s in subword_ids], axis=0 + ) + # Normalize the vector norm = self.np.linalg.norm(vector) if norm > 0: vector /= norm - + return vector def _tokenize(self, sentence): @@ -136,11 +154,11 @@ def _tokenize(self, sentence): tokens = [] word = "" for c in sentence: - if c in [' ', '\n', '\r', '\t', '\v', '\f', '\0']: + if c in [" ", "\n", "\r", "\t", "\v", "\f", "\0"]: if word: tokens.append(word) word = "" - if c == '\n': + if c == "\n": tokens.append("") else: word += c @@ -156,7 +174,7 @@ def get_sentence_vector(self, line): # get_word_vector already handles normalization, so no need to do it again here vec = self.get_word_vector(t) vectors.append(vec) - + # If the sentence was empty and resulted in no vectors, return a zero vector if not vectors: return self.np.zeros(self.embedding_dim) @@ -166,17 +184,17 @@ def get_sentence_vector(self, line): # --- Nearest Neighbor Method --- def get_word_suggestion(self, list_word): - """ - Queries the ONNX model to find the nearest neighbor word(s) + """Queries the ONNX model to find the nearest neighbor word(s) for the given word or list of words. Args: - list_word (str or list of str): A single word or a list of words + list_word (str or list of str): A single word or a list of words to get suggestions for. Returns: - str or list of str: The nearest neighbor word(s) from the + str or list of str: The nearest neighbor word(s) from the pre-loaded suggestion list. + """ if isinstance(list_word, str): input_words = [list_word] @@ -184,23 +202,26 @@ def get_word_suggestion(self, list_word): else: input_words = list_word return_single = False - + # Compute sentence vector for each input word/phrase - # The original code's `get_sentence_vector(' '.join(list(word)))` seems - # intended to treat a list of characters/tokens as a sentence. - # I'll stick to a more standard usage: treat each item in `input_words` + # The original code's `get_sentence_vector(' '.join(list(word)))` seems + # intended to treat a list of characters/tokens as a sentence. + # I'll stick to a more standard usage: treat each item in `input_words` # as a separate phrase/word to encode. - word_input_vecs = [self.get_sentence_vector(' '.join(list(word))) for word in input_words] + word_input_vecs = [ + self.get_sentence_vector(" ".join(list(word))) + for word in input_words + ] # Convert to numpy array for ONNX input (ensure float32) input_data = self.np.array(word_input_vecs, dtype=self.np.float32) # Run ONNX inference indices = self.nn_session.run(None, {"X": input_data})[0] - + # Look up suggestions suggestions = [self.words_for_suggestion[i].tolist() for i in indices] - + return suggestions[0] if return_single else suggestions @@ -209,7 +230,11 @@ def __init__(self): self.model_name = "pythainlp/word-spelling-correction-char2vec" self.model_path = get_hf_hub(self.model_name) self.model_onnx = get_hf_hub(self.model_name, "nearest_neighbors.onnx") - with open(get_hf_hub(self.model_name, "list_word-spelling-correction-char2vec.txt")) as f: + with open( + get_hf_hub( + self.model_name, "list_word-spelling-correction-char2vec.txt" + ) + ) as f: self.list_word = [i.strip() for i in f.readlines()] super().__init__(self.model_path, self.model_onnx, self.list_word) @@ -217,9 +242,10 @@ def __init__(self): _WSC = None -def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[str], List[List[str]]]: - """ - Get words spell suggestion +def get_words_spell_suggestion( + list_words: str | list[str], +) -> list[str] | list[list[str]]: + """Get words spell suggestion The function is designed to retrieve spelling suggestions \ for one or more input Thai words. @@ -243,6 +269,6 @@ def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[ # ['กระเพาะ', 'กระพา', 'กะเพรา', 'กระเพาะปลา', 'พระประธาน']] """ global _WSC - if _WSC==None: + if _WSC is None: _WSC = Words_Spelling_Correction() return _WSC.get_word_suggestion(list_words) diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py index 70c1e7e97..6294498aa 100644 --- a/pythainlp/summarize/__init__.py +++ b/pythainlp/summarize/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text summarization +"""Text summarization """ __all__ = [ diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index a72959a85..7a644c184 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Text summarization and keyword extraction """ -Text summarization and keyword extraction -""" -from typing import Iterable, List, Optional, Tuple +from __future__ import annotations + +from collections.abc import Iterable from pythainlp.summarize import ( CPE_KMUTT_THAI_SENTENCE_SUM, @@ -22,9 +22,8 @@ def summarize( n: int = 1, engine: str = DEFAULT_SUMMARIZE_ENGINE, tokenizer: str = "newmm", -) -> List[str]: - """ - This function summarizes text based on frequency of words. +) -> list[str]: + """This function summarizes text based on frequency of words. Under the hood, this function first tokenizes sentences from the given text with :func:`pythainlp.tokenize.sent_tokenize`. @@ -119,15 +118,14 @@ def summarize( def extract_keywords( text: str, - keyphrase_ngram_range: Tuple[int, int] = (1, 2), + keyphrase_ngram_range: tuple[int, int] = (1, 2), max_keywords: int = 5, min_df: int = 1, engine: str = DEFAULT_KEYWORD_EXTRACTION_ENGINE, tokenizer: str = "newmm", - stop_words: Optional[Iterable[str]] = None, -) -> List[str]: - """ - This function returns most-relevant keywords (and/or keyphrases) from the input document. + stop_words: Iterable[str] | None = None, +) -> list[str]: + """This function returns most-relevant keywords (and/or keyphrases) from the input document. Each algorithm may produce completely different keywords from each other, so please be careful when choosing the algorithm. @@ -197,7 +195,7 @@ def rank_by_frequency( max_keywords: int = 5, min_df: int = 5, tokenizer: str = "newmm", - stop_words: Optional[Iterable[str]] = None, + stop_words: Iterable[str] | None = None, ): from pythainlp.tokenize import word_tokenize from pythainlp.util.keywords import rank diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index 3adc828f2..01c15c595 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Summarization by frequency of words """ -Summarization by frequency of words -""" + +from __future__ import annotations + from collections import defaultdict from heapq import nlargest from string import punctuation -from typing import List from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import sent_tokenize, word_tokenize @@ -27,7 +27,7 @@ def __rank(ranking, n: int): return nlargest(n, ranking, key=ranking.get) def __compute_frequencies( - self, word_tokenized_sents: List[List[str]] + self, word_tokenized_sents: list[list[str]] ) -> defaultdict: word_freqs = defaultdict(int) for sent in word_tokenized_sents: @@ -48,7 +48,7 @@ def __compute_frequencies( def summarize( self, text: str, n: int, tokenizer: str = "newmm" - ) -> List[str]: + ) -> list[str]: sents = sent_tokenize(text, engine="whitespace+newline") word_tokenized_sents = [ word_tokenize(sent, engine=tokenizer) for sent in sents diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index acfd981c3..efed9e721 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Minimal re-implementation of KeyBERT. +"""Minimal re-implementation of KeyBERT. KeyBERT is a minimal and easy-to-use keyword extraction technique that leverages BERT embeddings to create keywords and keyphrases @@ -11,8 +9,11 @@ https://github.com/MaartenGr/KeyBERT """ + +from __future__ import annotations + from collections import Counter -from typing import Iterable, List, Optional, Tuple, Union +from collections.abc import Iterable import numpy as np from transformers import pipeline @@ -35,15 +36,14 @@ def __init__( def extract_keywords( self, text: str, - keyphrase_ngram_range: Tuple[int, int] = (1, 2), + keyphrase_ngram_range: tuple[int, int] = (1, 2), max_keywords: int = 5, min_df: int = 1, tokenizer: str = "newmm", return_similarity=False, - stop_words: Optional[Iterable[str]] = None, - ) -> Union[List[str], List[Tuple[str, float]]]: - """ - Extract Thai keywords and/or keyphrases with KeyBERT algorithm. + stop_words: Iterable[str] | None = None, + ) -> list[str] | list[tuple[str, float]]: + """Extract Thai keywords and/or keyphrases with KeyBERT algorithm. See https://github.com/MaartenGr/KeyBERT. :param str text: text to be summarized @@ -87,7 +87,9 @@ def extract_keywords( # 'ควบคุมการเปลี่ยนแปลง', # 'มีพิษ'] - keywords = kb.extract_keyword(text, max_keywords=10, return_similarity=True) + keywords = kb.extract_keyword( + text, max_keywords=10, return_similarity=True + ) # output: [('อวัยวะต่างๆ', 0.3228477063109462), # ('ซ่อมแซมส่วน', 0.31320597838000375), @@ -132,9 +134,8 @@ def extract_keywords( else: return [kw for kw, _ in keywords] - def embed(self, docs: Union[str, List[str]]) -> np.ndarray: - """ - Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. + def embed(self, docs: str | list[str]) -> np.ndarray: + """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. """ embs = self.ft_pipeline(docs) if isinstance(docs, str) or len(docs) == 1: @@ -152,11 +153,11 @@ def embed(self, docs: Union[str, List[str]]) -> np.ndarray: def _generate_ngrams( doc: str, - keyphrase_ngram_range: Tuple[int, int], + keyphrase_ngram_range: tuple[int, int], min_df: int, tokenizer_engine: str, stop_words: Iterable[str], -) -> List[str]: +) -> list[str]: assert keyphrase_ngram_range[0] >= 1, ( f"`keyphrase_ngram_range` must start from 1. " f"current value={keyphrase_ngram_range}." @@ -167,7 +168,7 @@ def _generate_ngrams( f"current value={keyphrase_ngram_range}." ) - def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]: + def _join_ngram(ngrams: list[tuple[str, str]]) -> list[str]: ngrams_joined = [] for ng in ngrams: joined = "".join(ng) @@ -201,18 +202,18 @@ def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]: def _rank_keywords( doc_vector: np.ndarray, word_vectors: np.ndarray, - keywords: List[str], + keywords: list[str], max_keywords: int, -) -> List[Tuple[str, float]]: +) -> list[tuple[str, float]]: def l2_norm(v: np.ndarray) -> np.ndarray: vec_size = v.shape[1] result = np.divide( v, np.linalg.norm(v, axis=1).reshape(-1, 1).repeat(vec_size, axis=1), ) - assert np.isclose( - np.linalg.norm(result, axis=1), 1 - ).all(), "Cannot normalize a vector to unit vector." + assert np.isclose(np.linalg.norm(result, axis=1), 1).all(), ( + "Cannot normalize a vector to unit vector." + ) return result def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index bc5e572ac..bce375fea 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Summarization by mT5 model """ -Summarization by mT5 model -""" -from typing import List + +from __future__ import annotations from transformers import MT5ForConditionalGeneration, T5Tokenizer @@ -21,10 +20,26 @@ def __init__( min_length: int = 30, max_length: int = 100, skip_special_tokens: bool = True, - pretrained_mt5_model_name: str = None, + pretrained_mt5_model_name: str = "", ): + """Initialize mT5 Summarizer. + + :param str model_size: Size of the model ("small", "base", "large", + "xl", "xxl"). Default is "small". + :param int num_beams: Number of beams for beam search. Default is 4. + :param int no_repeat_ngram_size: Size of n-grams to avoid repeating. + Default is 2. + :param int min_length: Minimum length of generated summary. + Default is 30. + :param int max_length: Maximum length of generated summary. + Default is 100. + :param bool skip_special_tokens: Whether to skip special tokens in + output. Default is True. + :param str pretrained_mt5_model_name: Name of pretrained model. + If empty (default), uses google/mt5-{model_size}. + """ model_name = "" - if pretrained_mt5_model_name is None: + if not pretrained_mt5_model_name: if model_size not in ["small", "base", "large", "xl", "xxl"]: raise ValueError( f"""model_size \"{model_size}\" not found. @@ -45,7 +60,7 @@ def __init__( self.max_length = max_length self.skip_special_tokens = skip_special_tokens - def summarize(self, text: str) -> List[str]: + def summarize(self, text: str) -> list[str]: preprocess_text = text.strip().replace("\n", "") if self.model_name == f"thanathorn/{CPE_KMUTT_THAI_SENTENCE_SUM}": t5_prepared_Text = "simplify: " + preprocess_text diff --git a/pythainlp/tag/__init__.py b/pythainlp/tag/__init__.py index c7ed6412f..23b02980c 100644 --- a/pythainlp/tag/__init__.py +++ b/pythainlp/tag/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Linguistic and other taggers. +"""Linguistic and other taggers. Tagging each token in a sentence with supplementary information, such as its part-of-speech (POS) tag, and named entity (NE) tag. diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index 8a6a07701..d815d64ea 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Perceptron Tagger. +"""Perceptron Tagger. This tagger is a port of the Textblob Averaged Perceptron Tagger Author: Matthew Honnibal , @@ -17,14 +15,16 @@ This tagger is provided under the terms of the MIT License. """ + +from __future__ import annotations + import json from collections import defaultdict -from typing import Dict, Iterable, List, Tuple, Union +from collections.abc import Iterable -class AveragedPerceptron(): - """ - An averaged perceptron, as implemented by Matthew Honnibal. +class AveragedPerceptron: + """An averaged perceptron, as implemented by Matthew Honnibal. See more implementation details here: http://honnibal.wordpress.com/2013/09/11/a-good-part-of-speechpos-tagger-in-about-200-lines-of-python/ @@ -45,9 +45,8 @@ def __init__(self) -> None: # Number of instances seen self.i = 0 - def predict(self, features: Dict): - """ - Dot-product the features and current weights and return the best + def predict(self, features: dict): + """Dot-product the features and current weights and return the best label. """ scores = defaultdict(float) @@ -60,7 +59,7 @@ def predict(self, features: Dict): # Do a secondary alphabetic sort, for stability return max(self.classes, key=lambda label: (scores[label], label)) - def update(self, truth, guess, features: Dict) -> None: + def update(self, truth, guess, features: dict) -> None: """Update the feature weights.""" def upd_feat(c, f, w, v): @@ -92,8 +91,7 @@ def average_weights(self) -> None: class PerceptronTagger: - """ - Greedy Averaged Perceptron tagger, as implemented by Matthew Honnibal. + """Greedy Averaged Perceptron tagger, as implemented by Matthew Honnibal. See more implementation details here: http://honnibal.wordpress.com/2013/09/11/a-good-part-of-speechpos-tagger-in-about-200-lines-of-python/ @@ -118,8 +116,7 @@ class PerceptronTagger: AP_MODEL_LOC = "" def __init__(self, path: str = "") -> None: - """ - :param str path: model path + """:param str path: model path """ self.model = AveragedPerceptron() self.tagdict = {} @@ -128,7 +125,7 @@ def __init__(self, path: str = "") -> None: self.AP_MODEL_LOC = path self.load(self.AP_MODEL_LOC) - def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]: + def tag(self, tokens: Iterable[str]) -> list[tuple[str, str]]: """Tags a string `tokens`.""" prev, prev2 = self.START output = [] @@ -146,12 +143,11 @@ def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]: def train( self, - sentences: Iterable[Iterable[Tuple[str, str]]], - save_loc: Union[str, None] = None, + sentences: Iterable[Iterable[tuple[str, str]]], + save_loc: str | None = None, nr_iter: int = 5, ) -> None: - """ - Train a model from sentences, and save it at ``save_loc``. + """Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. @@ -198,24 +194,22 @@ def train( json.dump(data, f, ensure_ascii=False) def load(self, loc: str) -> None: - """ - Load a pickled model. + """Load a pickled model. :param str loc: model path """ try: - with open(loc, "r", encoding="utf-8-sig") as f: + with open(loc, encoding="utf-8-sig") as f: w_td_c = json.load(f) - except IOError: + except OSError: msg = "Missing trontagger.json file." - raise IOError(msg) + raise OSError(msg) self.model.weights = w_td_c["weights"] self.tagdict = w_td_c["tagdict"] self.classes = w_td_c["classes"] self.model.classes = set(self.classes) def _normalize(self, word: str) -> str: - """ - Normalization used in pre-processing. + """Normalization used in pre-processing. - All words are lower cased - Digits in the range 1800-2100 are represented as !YEAR; @@ -233,10 +227,9 @@ def _normalize(self, word: str) -> str: return word.lower() def _get_features( - self, i: int, word: str, context: List[str], prev: str, prev2: str - ) -> Dict: - """ - Map tokens into a feature representation, implemented as a + self, i: int, word: str, context: list[str], prev: str, prev2: str + ) -> dict: + """Map tokens into a feature representation, implemented as a {hashable: float} dict. If the features change, a new model must be trained. """ @@ -265,7 +258,7 @@ def add(name: str, *args): return features def _make_tagdict( - self, sentences: Iterable[Iterable[Tuple[str, str]]] + self, sentences: Iterable[Iterable[tuple[str, str]]] ) -> None: """Make a tag dictionary for single-tag words.""" counts = defaultdict(lambda: defaultdict(int)) diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index afa771087..2a3938f14 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations # defined strings for special characters CHAR_TO_ESCAPE = {" ": "_"} @@ -32,9 +31,8 @@ } -def pre_process(words: List[str]) -> List[str]: - """ - Convert signs and symbols with their defined strings. +def pre_process(words: list[str]) -> list[str]: + """Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, before the actual POS tagging. """ @@ -44,10 +42,9 @@ def pre_process(words: List[str]) -> List[str]: def post_process( - word_tags: List[Tuple[str, str]], to_ud: bool = False -) -> List[Tuple[str, str]]: - """ - Convert defined strings back to corresponding signs and symbols. + word_tags: list[tuple[str, str]], to_ud: bool = False +) -> list[tuple[str, str]]: + """Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, after the POS tagging. """ diff --git a/pythainlp/tag/chunk.py b/pythainlp/tag/chunk.py index cd5722b7f..278da56a9 100644 --- a/pythainlp/tag/chunk.py +++ b/pythainlp/tag/chunk.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations def chunk_parse( - sent: List[Tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp" -) -> List[str]: - """ - This function parses Thai sentence to phrase structure in IOB format. + sent: list[tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp" +) -> list[str]: + """This function parses Thai sentence to phrase structure in IOB format. :param list sent: list [(word, part-of-speech)] :param str engine: chunk parse engine (now, it has crf only) diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index f8dfbc950..1e949a817 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from __future__ import annotations from pycrfsuite import Tagger as CRFTagger @@ -13,9 +12,8 @@ def _is_stopword(word: str) -> bool: # check Thai stopword return word in thai_stopwords() -def _doc2features(tokens: List[Tuple[str, str]], index: int) -> Dict: - """ - `tokens` = a POS-tagged sentence [(w1, t1), ...] +def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict: + """`tokens` = a POS-tagged sentence [(w1, t1), ...] `index` = the index of the token we want to extract features for """ word, pos = tokens[index] @@ -67,6 +65,6 @@ def load_model(self, corpus: str): self.path = path_pythainlp_corpus("crfchunk_orchidpp.model") self.tagger.open(self.path) - def parse(self, token_pos: List[Tuple[str, str]]) -> List[str]: + def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) return self.tagger.tag(self.xseq) diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py index b70072ba5..9b2143c0a 100644 --- a/pythainlp/tag/locations.py +++ b/pythainlp/tag/locations.py @@ -1,19 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Recognizes locations in text +"""Recognizes locations in text """ -from typing import List, Tuple +from __future__ import annotations from pythainlp.corpus import provinces -def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]: - """ - This function recognizes Thailand provinces in text. +def tag_provinces(tokens: list[str]) -> list[tuple[str, str]]: + """This function recognizes Thailand provinces in text. Note that it uses exact match and considers no context. @@ -26,7 +23,7 @@ def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]: from pythainlp.tag import tag_provinces - text = ['หนองคาย', 'น่าอยู่'] + text = ["หนองคาย", "น่าอยู่"] tag_provinces(text) # output: [('หนองคาย', 'B-LOCATION'), ('น่าอยู่', 'O')] """ diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index e9628a7b4..71a0876c8 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -1,16 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Named-entity recognizer """ -Named-entity recognizer -""" -from typing import List, Tuple, Union + +from __future__ import annotations class NER: - """ - Class of named-entity recognizer + """Class of named-entity recognizer :param str engine: engine of named-entity recognizer :param str corpus: corpus @@ -26,7 +24,9 @@ class NER: **Note**: The tltk engine supports NER models from tltk only. """ - def __init__(self, engine: str = "thainer-v2", corpus: str = "thainer") -> None: + def __init__( + self, engine: str = "thainer-v2", corpus: str = "thainer" + ) -> None: self.load_engine(engine=engine, corpus=corpus) def load_engine(self, engine: str, corpus: str) -> None: @@ -38,7 +38,10 @@ def load_engine(self, engine: str, corpus: str) -> None: self.engine = ThaiNameTagger() elif engine == "thainer-v2" and corpus == "thainer": from pythainlp.wangchanberta import NamedEntityRecognition - self.engine = NamedEntityRecognition(model="pythainlp/thainer-corpus-v2-base-model") + + self.engine = NamedEntityRecognition( + model="pythainlp/thainer-corpus-v2-base-model" + ) elif engine == "tltk": from pythainlp.tag import tltk @@ -53,18 +56,13 @@ def load_engine(self, engine: str, corpus: str) -> None: self.engine = NamedEntityTagger() else: raise ValueError( - "NER class not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"NER class not support {engine} engine or {corpus} corpus." ) - def tag(self, - text, - pos=False, - tag=False - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: - """ - This function tags named entities in text in IOB format. + def tag( + self, text, pos=False, tag=False + ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: + """This function tags named entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: output with part-of-speech tags.\ @@ -98,8 +96,7 @@ def tag(self, class NNER: - """ - Nested Named Entity Recognition + """Nested Named Entity Recognition :param str engine: engine of nested named entity recognizer :param str corpus: corpus @@ -116,9 +113,8 @@ def load_engine(self, engine: str = "thai_nner") -> None: self.engine = Thai_NNER() - def tag(self, text) -> Tuple[List[str], List[dict]]: - """ - This function tags nested named entities. + def tag(self, text) -> tuple[list[str], list[dict]]: + """This function tags nested named entities. :param str text: text in Thai to be tagged diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 678c7a37e..69fcb0c07 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Data preprocessing for ORCHID corpus """ -Data preprocessing for ORCHID corpus -""" -from typing import List, Tuple + +from __future__ import annotations # defined strings for special characters, # from Table 4 in ORCHID paper @@ -126,9 +125,8 @@ def ud_exception(w: str, tag: str) -> str: return tag -def pre_process(words: List[str]) -> List[str]: - """ - Convert signs and symbols with their defined strings. +def pre_process(words: list[str]) -> list[str]: + """Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, before the actual POS tagging. """ @@ -138,10 +136,9 @@ def pre_process(words: List[str]) -> List[str]: def post_process( - word_tags: List[Tuple[str, str]], to_ud: bool = False -) -> List[Tuple[str, str]]: - """ - Convert defined strings back to corresponding signs and symbols. + word_tags: list[tuple[str, str]], to_ud: bool = False +) -> list[tuple[str, str]]: + """Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, after the actual POS tagging. """ diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index 71e9b720e..c856a864b 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Perceptron part-of-speech tagger """ -Perceptron part-of-speech tagger -""" + +from __future__ import annotations import os -from typing import List, Tuple from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import PerceptronTagger, blackboard, orchid @@ -69,9 +68,8 @@ def _tud_tagger(): return _TUD_TAGGER -def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]: - """ - :param list words: a list of tokenized words +def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: + """:param list words: a list of tokenized words :param str corpus: corpus name (orchid, pud) :return: a list of tuples (word, POS tag) :rtype: list[tuple[str, str]] diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index a83a3330f..6fbc3f2f0 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations def pos_tag( - words: List[str], engine: str = "perceptron", corpus: str = "orchid" -) -> List[Tuple[str, str]]: - """ - Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'. + words: list[str], engine: str = "perceptron", corpus: str = "orchid" +) -> list[tuple[str, str]]: + """Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'. :param list words: a list of tokenized words :param str engine: @@ -117,9 +115,7 @@ def pos_tag( from pythainlp.tag.unigram import tag as tag_ else: raise ValueError( - "pos_tag not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"pos_tag not support {engine} engine or {corpus} corpus." ) word_tags = tag_(words, corpus=corpus) @@ -128,12 +124,11 @@ def pos_tag( def pos_tag_sents( - sentences: List[List[str]], + sentences: list[list[str]], engine: str = "perceptron", corpus: str = "orchid", -) -> List[List[Tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. +) -> list[list[tuple[str, str]]]: + """Marks sentences with part-of-speech (POS) tags. :param list sentences: a list of lists of tokenized words :param str engine: @@ -180,22 +175,27 @@ def pos_tag_transformers( sentence: str, engine: str = "bert", corpus: str = "blackboard", -) -> List[List[Tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. +) -> list[list[tuple[str, str]]]: + """Marks sentences with part-of-speech (POS) tags. :param str sentence: a list of lists of tokenized words :param str engine: * *bert* - BERT: Bidirectional Encoder Representations from Transformers (default) - * *wangchanberta* - fine-tuned version of airesearch/wangchanberta-base-att-spm-uncased on pud corpus (support PUD cotpus only) + * *wangchanberta* - fine-tuned version of \ + airesearch/wangchanberta-base-att-spm-uncased on pud corpus \ + (support PUD cotpus only) * *phayathaibert* - fine-tuned version of clicknext/phayathaibert \ on blackboard corpus (support blackboard cotpus only) - * *mdeberta* - mDeBERTa: Multilingual Decoding-enhanced BERT with disentangled attention (support PUD corpus only) - :param str corpus: the corpus that is used to create the language model for tagger - * *blackboard* - `blackboard treebank (support bert engine only) `_ + * *mdeberta* - mDeBERTa: Multilingual Decoding-enhanced BERT \ + with disentangled attention (support PUD corpus only) + :param str corpus: the corpus that is used to create the language model + for tagger + * *blackboard* - `blackboard treebank (support bert engine only) \ + `_ * *pud* - `Parallel Universal Dependencies (PUD)\ `_ \ - treebanks, natively use Universal POS tags (support wangchanberta and mdeberta engine) + treebanks, natively use Universal POS tags \ + (support wangchanberta and mdeberta engine) :return: a list of lists of tuples (word, POS tag) :rtype: list[list[tuple[str, str]]] @@ -210,7 +210,6 @@ def pos_tag_transformers( # output: # [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] """ - try: from transformers import ( AutoModelForTokenClassification, @@ -245,15 +244,14 @@ def pos_tag_transformers( tokenizer = AutoTokenizer.from_pretrained(base_model) else: raise ValueError( - "pos_tag_transformers not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"pos_tag_transformers not support {engine} engine or {corpus} corpus." ) - pipeline = TokenClassificationPipeline(model=model, - tokenizer=tokenizer, - aggregation_strategy="simple", - ) + pipeline = TokenClassificationPipeline( + model=model, + tokenizer=tokenizer, + aggregation_strategy="simple", + ) outputs = pipeline(sentence) word_tags = [[(tag["word"], tag["entity_group"]) for tag in outputs]] diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index ad0a2623a..09432e7f1 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from thai_nner import NNER @@ -13,5 +12,5 @@ class Thai_NNER: def __init__(self, path_model=get_corpus_path("thai_nner", "1.0")) -> None: self.model = NNER(path_model=path_model) - def tag(self, text) -> Tuple[List[str], List[dict]]: + def tag(self, text) -> tuple[list[str], list[dict]]: return self.model.get_tag(text) diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index 8558eef67..106cb82b1 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Named-entity recognizer """ -Named-entity recognizer -""" + +from __future__ import annotations __all__ = ["ThaiNameTagger"] -from typing import Dict, List, Tuple, Union from pythainlp.corpus import get_corpus_path, thai_stopwords from pythainlp.tag import pos_tag @@ -22,7 +21,7 @@ def _is_stopword(word: str) -> bool: # เช็คว่าเป็นคำ return word in thai_stopwords() -def _doc2features(doc, i) -> Dict: +def _doc2features(doc, i) -> dict: word = doc[i][0] postag = doc[i][1] @@ -74,8 +73,7 @@ def _doc2features(doc, i) -> Dict: class ThaiNameTagger: - """ - Thai named-entity recognizer or Thai NER. + """Thai named-entity recognizer or Thai NER. This function supports Thai NER 1.4 and 1.5 only. :param str version: Thai NER version. It supports Thai NER 1.4 & 1.5. @@ -91,8 +89,7 @@ class ThaiNameTagger: """ def __init__(self, version: str = "1.4") -> None: - """ - Thai named-entity recognizer. + """Thai named-entity recognizer. :param str version: Thai NER version. It's support Thai NER 1.4 & 1.5. @@ -111,9 +108,8 @@ def __init__(self, version: str = "1.4") -> None: def get_ner( self, text: str, pos: bool = True, tag: bool = False - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]]]: - """ - This function tags named-entities in text in IOB format. + ) -> list[tuple[str, str]] | list[tuple[str, str, str]]: + """This function tags named-entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: To include POS tags in the results (`True`) or diff --git a/pythainlp/tag/tltk.py b/pythainlp/tag/tltk.py index f957a1832..c8cba01d1 100644 --- a/pythainlp/tag/tltk.py +++ b/pythainlp/tag/tltk.py @@ -1,22 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple, Union +from __future__ import annotations try: from tltk import nlp except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) from pythainlp.tokenize import word_tokenize nlp.pos_load() nlp.ner_load() -def pos_tag(words: List[str], corpus: str = "tnc") -> List[Tuple[str, str]]: +def pos_tag(words: list[str], corpus: str = "tnc") -> list[tuple[str, str]]: if corpus != "tnc": - raise ValueError("tltk not support {0} corpus.".format(0)) + raise ValueError(f"tltk not support {0} corpus.") return nlp.pos_tag_wordlist(words) @@ -26,9 +27,8 @@ def _post_process(text: str) -> str: def get_ner( text: str, pos: bool = True, tag: bool = False -) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: - """ - Named-entity recognizer from **TLTK** +) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: + """Named-entity recognizer from **TLTK** This function tags named-entities in text in IOB format. diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 34384072b..ea3fa3c43 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Unigram Part-Of-Speech tagger """ -Unigram Part-Of-Speech tagger -""" + +from __future__ import annotations + import json import os -from typing import List, Tuple from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import blackboard, orchid @@ -75,8 +75,8 @@ def _tud_tagger(): def _find_tag( - words: List[str], dictdata: dict, default_tag: str = "" -) -> List[Tuple[str, str]]: + words: list[str], dictdata: dict, default_tag: str = "" +) -> list[tuple[str, str]]: keys = list(dictdata.keys()) return [ (word, dictdata[word]) if word in keys else (word, default_tag) @@ -84,9 +84,8 @@ def _find_tag( ] -def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]: - """ - :param list words: a list of tokenized words +def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: + """:param list words: a list of tokenized words :param str corpus: corpus name (orchid or pud) :return: a list of tuples (word, POS tag) :rtype: list[tuple[str, str]] diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py index c3744d1a1..ccc12f15a 100644 --- a/pythainlp/tag/wangchanberta_onnx.py +++ b/pythainlp/tag/wangchanberta_onnx.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import json -from typing import List import numpy as np @@ -16,7 +16,7 @@ def __init__( model_name: str, model_version: str, file_onnx: str, - providers: List[str] = ["CPUExecutionProvider"], + providers: list[str] = ["CPUExecutionProvider"], ) -> None: import sentencepiece as spm from onnxruntime import ( diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 160996d7f..3f9867571 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -1,13 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Tokenizers at different levels of linguistic analysis. +"""Tokenizers at different levels of linguistic analysis. """ __all__ = [ - "THAI2FIT_TOKENIZER", + "thai2fit_tokenizer", "Tokenizer", "Trie", "paragraph_tokenize", @@ -19,6 +17,8 @@ "display_cell_tokenize", ] +from functools import lru_cache + from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie @@ -27,23 +27,27 @@ DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo" -DEFAULT_WORD_DICT_TRIE = Trie(thai_words()) -DEFAULT_SYLLABLE_DICT_TRIE = Trie(thai_syllables()) -DEFAULT_DICT_TRIE = DEFAULT_WORD_DICT_TRIE + +@lru_cache +def word_dict_trie(): + """Lazy load default word dict trie with cache""" + return Trie(thai_words()) + + +@lru_cache +def syllable_dict_trie(): + """Lazy load default syllable dict trie with cache""" + return Trie(thai_syllables()) + from pythainlp.tokenize.core import ( Tokenizer, + display_cell_tokenize, paragraph_tokenize, sent_tokenize, subword_tokenize, syllable_tokenize, word_detokenize, word_tokenize, - display_cell_tokenize, -) - -from pythainlp.corpus import get_corpus as _get_corpus - -THAI2FIT_TOKENIZER = Tokenizer( - custom_dict=_get_corpus("words_th_thai2fit_201810.txt"), engine="mm" ) +from pythainlp.tokenize.thai2fit import thai2fit_tokenizer diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index 463b45fb0..2d8cadee7 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -1,22 +1,21 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Utility functions for tokenize module. """ -Utility functions for tokenize module. -""" + +from __future__ import annotations import re -from typing import Callable, List +from collections.abc import Callable _DIGITS_WITH_SEPARATOR = re.compile(r"(\d+[\.\,:])+\d+") def apply_postprocessors( - segments: List[str], postprocessors: Callable[[List[str]], List[str]] -) -> List[str]: - """ - A list of callables to apply to a raw segmentation result. + segments: list[str], postprocessors: Callable[[list[str]], list[str]] +) -> list[str]: + """A list of callables to apply to a raw segmentation result. """ for func in postprocessors: segments = func(segments) @@ -24,9 +23,8 @@ def apply_postprocessors( return segments -def rejoin_formatted_num(segments: List[str]) -> List[str]: - """ - Rejoin well-known formatted numeric that are over-tokenized. +def rejoin_formatted_num(segments: list[str]) -> list[str]: + """Rejoin well-known formatted numeric that are over-tokenized. The formatted numeric are numbers separated by ":", ",", or ".", such as time, decimal numbers, comma-added numbers, and IP addresses. @@ -73,9 +71,8 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]: return tokens_joined -def strip_whitespace(segments: List[str]) -> List[str]: - """ - Strip whitespace(s) off each token and remove whitespace tokens. +def strip_whitespace(segments: list[str]) -> list[str]: + """Strip whitespace(s) off each token and remove whitespace tokens. :param List[str] segments: result from word tokenizer :return: a list of tokens :rtype: List[str] diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py index e5c59c44f..f7e65cbdd 100644 --- a/pythainlp/tokenize/attacut.py +++ b/pythainlp/tokenize/attacut.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -* # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai +"""Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai :See Also: * `GitHub repository `_ """ -from typing import Dict, List + +from __future__ import annotations from attacut import Tokenizer @@ -22,16 +21,15 @@ def __init__(self, model="attacut-sc"): self._tokenizer = Tokenizer(model=self._MODEL_NAME) - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: return self._tokenizer.tokenize(text) -_tokenizers: Dict[str, AttacutTokenizer] = {} +_tokenizers: dict[str, AttacutTokenizer] = {} -def segment(text: str, model: str = "attacut-sc") -> List[str]: - """ - Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai +def segment(text: str, model: str = "attacut-sc") -> list[str]: + """Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai :param str text: text to be tokenized to words :param str model: model of word tokenizer model :return: list of words, tokenized from the text diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py index 5237da2a9..b958b9438 100644 --- a/pythainlp/tokenize/budoux.py +++ b/pythainlp/tokenize/budoux.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for BudouX tokenizer (https://github.com/google/budoux) +"""Wrapper for BudouX tokenizer (https://github.com/google/budoux) This module provides a small, defensive wrapper around the Python `budoux` package. The wrapper lazy-imports the package so importing @@ -11,7 +9,8 @@ used and `budoux` is missing, a clear ImportError is raised with an installation hint. """ -from typing import List + +from __future__ import annotations _parser = None @@ -32,7 +31,7 @@ def _init_parser(): return budoux.load_default_thai_parser() -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """Segment `text` into tokens using budoux. The function returns a list of strings. If `budoux` is not available diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index f3d316a2e..9259041dc 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -1,22 +1,22 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Generic functions of tokenizers """ -Generic functions of tokenizers -""" + +from __future__ import annotations import copy import re -from typing import Iterable, List, Union +from collections.abc import Iterable from pythainlp.tokenize import ( DEFAULT_SENT_TOKENIZE_ENGINE, DEFAULT_SUBWORD_TOKENIZE_ENGINE, - DEFAULT_SYLLABLE_DICT_TRIE, DEFAULT_SYLLABLE_TOKENIZE_ENGINE, - DEFAULT_WORD_DICT_TRIE, DEFAULT_WORD_TOKENIZE_ENGINE, + syllable_dict_trie, + word_dict_trie, ) from pythainlp.tokenize._utils import ( apply_postprocessors, @@ -27,10 +27,9 @@ def word_detokenize( - segments: Union[List[List[str]], List[str]], output: str = "str" -) -> Union[List[str], str]: - """ - Word detokenizer. + segments: list[list[str]] | list[str], output: str = "str" +) -> list[str] | str: + """Word detokenizer. This function will detokenize the list of words in each sentence into text. @@ -97,13 +96,12 @@ def word_detokenize( def word_tokenize( text: str, - custom_dict: Trie = Trie([]), + custom_dict: Trie | None = None, engine: str = DEFAULT_WORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, join_broken_num: bool = True, -) -> List[str]: - """ - Word tokenizer. +) -> list[str]: + """Word tokenizer. Tokenizes running text into words (list of strings). @@ -128,7 +126,7 @@ def word_tokenize( * *icu* - wrapper for a word tokenizer in `PyICU `_., from ICU (International Components for Unicode), - dictionary-based + dictionary-based * *longest* - dictionary-based, longest matching * *mm* - "multi-cut", dictionary-based, maximum matching * *nercut* - dictionary-based, maximal matching, @@ -223,6 +221,9 @@ def word_tokenize( segments = [] + if custom_dict is None: + custom_dict = Trie([]) + if custom_dict and engine in ( "attacut", "icu", @@ -358,12 +359,11 @@ def map_indices_to_words(index_list, sentences): def sent_tokenize( - text: Union[str, List[str]], + text: str | list[str], engine: str = DEFAULT_SENT_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: - """ - Sentence tokenizer. +) -> list[str]: + """Sentence tokenizer. Tokenizes running text into "sentences". Supports both string and list of strings. @@ -428,7 +428,6 @@ def sent_tokenize( # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ ', 'และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค'] """ - if not text or not isinstance(text, (str, list)): return [] @@ -529,9 +528,8 @@ def paragraph_tokenize( engine: str = "wtp-mini", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[List[str]]: - """ - Paragraph tokenizer. +) -> list[list[str]]: + """Paragraph tokenizer. Tokenizes text into paragraphs. @@ -561,7 +559,7 @@ def paragraph_tokenize( paragraph_tokenize(sent) # output: [ - # ['(1) '], + # ['(1) '], # [ # 'บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต ', # 'มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด ', @@ -597,9 +595,8 @@ def subword_tokenize( text: str, engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: - """ - Subword tokenizer for tokenizing text into units smaller than syllables. +) -> list[str]: + """Subword tokenizer for tokenizing text into units smaller than syllables. Tokenizes text into inseparable units of Thai contiguous characters, namely @@ -689,9 +686,7 @@ def subword_tokenize( words = word_tokenize(text) for word in words: segments.extend( - word_tokenize( - text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE - ) + word_tokenize(text=word, custom_dict=syllable_dict_trie()) ) elif engine == "ssg": from pythainlp.tokenize.ssg import segment @@ -720,9 +715,8 @@ def syllable_tokenize( text: str, engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: - """ - Syllable tokenizer +) -> list[str]: + """Syllable tokenizer Tokenizes text into inseparable units of Thai syllables. @@ -752,9 +746,8 @@ def syllable_tokenize( ) -def display_cell_tokenize(text: str) -> List[str]: - """ - Display cell tokenizer. +def display_cell_tokenize(text: str) -> list[str]: + """Display cell tokenizer. Tokenizes Thai text into display cells without splitting tone marks. @@ -793,8 +786,7 @@ def display_cell_tokenize(text: str) -> List[str]: class Tokenizer: - """ - Tokenizer class for a custom tokenizer. + """Tokenizer class for a custom tokenizer. This class allows users to pre-define custom dictionary along with tokenizer and encapsulate them into one single object. @@ -861,13 +853,12 @@ class Tokenizer: def __init__( self, - custom_dict: Union[Trie, Iterable[str], str] = [], + custom_dict: Trie | Iterable[str] | str = [], engine: str = "newmm", keep_whitespace: bool = True, join_broken_num: bool = True, ): - """ - Initialize tokenizer object. + """Initialize tokenizer object. :param str custom_dict: a file path, a list of vocaburaies* to be used to create a trie, or an instantiated @@ -881,7 +872,7 @@ def __init__( if custom_dict: self.__trie_dict = dict_trie(custom_dict) else: - self.__trie_dict = DEFAULT_WORD_DICT_TRIE + self.__trie_dict = word_dict_trie() self.__engine = engine if self.__engine not in ["newmm", "mm", "longest", "deepcut"]: raise NotImplementedError( @@ -893,9 +884,8 @@ def __init__( self.__keep_whitespace = keep_whitespace self.__join_broken_num = join_broken_num - def word_tokenize(self, text: str) -> List[str]: - """ - Main tokenization function. + def word_tokenize(self, text: str) -> list[str]: + """Main tokenization function. :param str text: text to be tokenized :return: list of words, tokenized from the text @@ -910,8 +900,7 @@ def word_tokenize(self, text: str) -> List[str]: ) def set_tokenize_engine(self, engine: str) -> None: - """ - Set the tokenizer's engine. + """Set the tokenizer's engine. :param str engine: choose between different options of tokenizer engines (i.e. *newmm*, *mm*, *longest*, *deepcut*) diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 9746d0113..b4cb9e0be 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -CRFCut - Thai sentence segmenter. +"""CRFCut - Thai sentence segmenter. Thai sentence segmentation using conditional random field, with default model trained on TED dataset @@ -17,8 +15,9 @@ POS features are not used due to unreliable POS tagging available """ +from __future__ import annotations + import os -from typing import List import pycrfsuite @@ -128,10 +127,9 @@ def extract_features( - doc: List[str], window: int = 2, max_n_gram: int = 3 -) -> List[List[str]]: - """ - Extract features for CRF by sliding `max_n_gram` of tokens + doc: list[str], window: int = 2, max_n_gram: int = 3 +) -> list[list[str]]: + """Extract features for CRF by sliding `max_n_gram` of tokens for +/- `window` from the current token :param List[str] doc: tokens from which features are to be extracted @@ -168,12 +166,12 @@ def extract_features( # ngram features for n_gram in range(1, min(max_n_gram + 1, 2 + window * 2)): for j in range(i - window, i + window + 2 - n_gram): - feature_position = f"{n_gram}_{j-i}_{j-i+n_gram}" - word_ = f'{"|".join(doc[j:(j+n_gram)])}' + feature_position = f"{n_gram}_{j - i}_{j - i + n_gram}" + word_ = f"{'|'.join(doc[j : (j + n_gram)])}" word_features += [f"word_{feature_position}={word_}"] - ender_ = f'{"|".join(doc_ender[j:(j+n_gram)])}' + ender_ = f"{'|'.join(doc_ender[j : (j + n_gram)])}" word_features += [f"ender_{feature_position}={ender_}"] - starter_ = f'{"|".join(doc_starter[j:(j+n_gram)])}' + starter_ = f"{'|'.join(doc_starter[j : (j + n_gram)])}" word_features += [f"starter_{feature_position}={starter_}"] # append to feature per word doc_features.append(word_features) @@ -186,9 +184,8 @@ def extract_features( _tagger.open(os.path.join(corpus_path(), _CRFCUT_DATA_FILENAME)) -def segment(text: str) -> List[str]: - """ - CRF-based sentence segmentation. +def segment(text: str) -> list[str]: + """CRF-based sentence segmentation. :param str text: text to be tokenized into sentences :return: list of words, tokenized from the text @@ -206,7 +203,7 @@ def segment(text: str) -> List[str]: if toks[idx].strip().endswith(("!", ".", "?")): labs[idx] = "E" # Spaces or empty strings would no longer be treated as end of sentence. - elif (idx == 0 or labs[idx-1] == "E") and toks[idx].strip() == "": + elif (idx == 0 or labs[idx - 1] == "E") and toks[idx].strip() == "": labs[idx] = "I" sentences = [] diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py index 4b008de40..6e677c241 100644 --- a/pythainlp/tokenize/deepcut.py +++ b/pythainlp/tokenize/deepcut.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for deepcut Thai word segmentation. deepcut is a +"""Wrapper for deepcut Thai word segmentation. deepcut is a Thai word segmentation library using 1D Convolution Neural Network. User need to install deepcut (and its dependency: tensorflow) by themselves. @@ -12,7 +10,7 @@ * `GitHub repository `_ """ -from typing import List, Union +from __future__ import annotations try: from deepcut import tokenize @@ -21,9 +19,7 @@ from pythainlp.util import Trie -def segment( - text: str, custom_dict: Union[Trie, List[str], str] = [] -) -> List[str]: +def segment(text: str, custom_dict: Trie | list[str] | str = []) -> list[str]: if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 8d8769515..9e97f92bf 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Segmenting text into Enhanced Thai Character Clusters (ETCCs) +"""Segmenting text into Enhanced Thai Character Clusters (ETCCs) Python implementation by Wannaphong Phatthiyaphaibun This implementation relies on a dictionary of ETCC created from etcc.txt @@ -19,19 +17,28 @@ and backward longest matching techniques." In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001. """ + +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp import thai_follow_vowels from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer -_cut_etcc = Tokenizer(get_corpus("etcc.txt"), engine="longest") + +@lru_cache +def _cut_etcc(): + """Lazy load ETCC tokenizer with cache""" + return Tokenizer(get_corpus("etcc.txt"), engine="longest") + + _PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]" _RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR) -def _cut_subword(tokens: List[str]) -> List[str]: +def _cut_subword(tokens: list[str]) -> list[str]: len_tokens = len(tokens) i = 0 while True: @@ -45,9 +52,8 @@ def _cut_subword(tokens: List[str]) -> List[str]: return tokens -def segment(text: str) -> List[str]: - """ - Segmenting text into ETCCs. +def segment(text: str) -> list[str]: + """Segmenting text into ETCCs. Enhanced Thai Character Cluster (ETCC) is a kind of subword unit. The concept was presented in Inrut, Jeeragone, Patiroop Yuanghirun, @@ -60,8 +66,7 @@ def segment(text: str) -> List[str]: :return: list of clusters, tokenized from the text :return: List[str] """ - if not text or not isinstance(text, str): return [] - return _cut_subword(_cut_etcc.word_tokenize(text)) + return _cut_subword(_cut_etcc().word_tokenize(text)) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 30d8180bc..4c11ce684 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2019 Ponrawee Prasertsom # SPDX-License-Identifier: Apache-2.0 -""" -🪿 Han-solo: Thai syllable segmenter +"""🪿 Han-solo: Thai syllable segmenter GitHub: https://github.com/PyThaiNLP/Han-solo """ -from typing import List + +from __future__ import annotations from pythainlp.corpus import path_pythainlp_corpus @@ -119,7 +118,7 @@ def featurize( _to_feature = Featurizer() -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: x = _to_feature.featurize(text)["X"] y_pred = tagger.tag(x) list_cut = [] diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index f4cfc571d..1059cd976 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Dictionary-based longest-matching Thai word segmentation. Implementation is based +"""Dictionary-based longest-matching Thai word segmentation. Implementation is based on the codes from Patorn Utenpattanun. :See Also: @@ -11,11 +9,13 @@ `_ """ + +from __future__ import annotations + import re -from typing import Dict, List, Union from pythainlp import thai_tonemarks -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie _FRONT_DEP_CHAR = [ @@ -48,7 +48,7 @@ def __init__(self, trie: Trie): self.__trie = trie @staticmethod - def __search_nonthai(text: str) -> Union[None, str]: + def __search_nonthai(text: str) -> None | str: match = _RE_NONTHAI.search(text) if match.group(0): return match.group(0).lower() @@ -137,24 +137,27 @@ def __segment(self, text: str): # Group consecutive spaces into one token grouped_tokens = [] for token in tokens: - if token.isspace() and grouped_tokens and grouped_tokens[-1].isspace(): + if ( + token.isspace() + and grouped_tokens + and grouped_tokens[-1].isspace() + ): grouped_tokens[-1] += token else: grouped_tokens.append(token) return grouped_tokens - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: tokens = self.__segment(text) return tokens -_tokenizers: Dict[int, LongestMatchTokenizer] = {} +_tokenizers: dict[int, LongestMatchTokenizer] = {} -def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: - """ - Dictionary-based longest matching word segmentation. +def segment(text: str, custom_dict: Trie | None = None) -> list[str]: + """Dictionary-based longest matching word segmentation. :param str text: text to be tokenized into words :param pythainlp.util.Trie custom_dict: dictionary for tokenization @@ -164,7 +167,7 @@ def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() global _tokenizers custom_dict_ref_id = id(custom_dict) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index afd57e88a..3f88f5b6a 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Multi cut -- Thai word segmentation with maximum matching. +"""Multi cut -- Thai word segmentation with maximum matching. Original codes from Korakot Chaovavanich. :See Also: @@ -13,11 +11,13 @@ `_ """ +from __future__ import annotations + import re from collections import defaultdict -from typing import Iterator, List +from collections.abc import Iterator -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie @@ -48,12 +48,11 @@ def __init__(self, value, multi=None, in_dict=True): def _multicut( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Trie | None = None ) -> Iterator[LatticeString]: """Return LatticeString""" if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE - + custom_dict = word_dict_trie() len_text = len(text) words_at = defaultdict(list) # main data structure @@ -101,7 +100,7 @@ def serialize(p, p2): # helper function q.add(i) -def mmcut(text: str) -> List[str]: +def mmcut(text: str) -> list[str]: res = [] for w in _multicut(text): mm = min(w.multi, key=lambda x: x.count("/")) @@ -109,7 +108,7 @@ def mmcut(text: str) -> List[str]: return res -def _combine(ww: List[LatticeString]) -> Iterator[str]: +def _combine(ww: list[LatticeString]) -> Iterator[str]: if ww == []: yield "" else: @@ -122,15 +121,13 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]: yield m.replace("/", "|") + "|" + tail -def segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE -) -> List[str]: +def segment(text: str, custom_dict: Trie | None = None) -> list[str]: """Dictionary-based maximum matching word segmentation. :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to a Trie generated from pythainlp.corpus.thai_words :type custom_dict: Trie, optional :return: list of segmented tokens :rtype: List[str] @@ -138,18 +135,19 @@ def segment( if not text or not isinstance(text, str): return [] + if not custom_dict: + custom_dict = word_dict_trie() + return list(_multicut(text, custom_dict=custom_dict)) -def find_all_segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE -) -> List[str]: +def find_all_segment(text: str, custom_dict: Trie | None = None) -> list[str]: """Get all possible segment variations. :param text: input string to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :return: list of segment variations :rtype: List[str] @@ -157,6 +155,9 @@ def find_all_segment( if not text or not isinstance(text, str): return [] + if not custom_dict: + custom_dict = word_dict_trie() + ww = list(_multicut(text, custom_dict=custom_dict)) return list(_combine(ww)) diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py index 1ac0fa8ab..50f33d8ba 100644 --- a/pythainlp/tokenize/nercut.py +++ b/pythainlp/tokenize/nercut.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -nercut 0.2 +"""nercut 0.2 Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are @@ -11,7 +9,10 @@ Code by Wannaphong Phatthiyaphaibun """ -from typing import Iterable, List + +from __future__ import annotations + +from collections.abc import Iterable from pythainlp.tag.named_entity import NER @@ -29,9 +30,8 @@ def segment( "TIME", ], tagger=_thainer, -) -> List[str]: - """ - Dictionary-based maximal matching word segmentation, constrained by +) -> list[str]: + """Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are parts of the same named-entity. diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 935da7453..2bcbfa3be 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Dictionary-based maximal matching word segmentation, constrained by +"""Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries with improved rules. The codes are based on the notebooks created by Korakot Chaovavanich, @@ -15,12 +13,15 @@ * \ https://colab.research.google.com/drive/14Ibg-ngZXj15RKwjNwoZlOT32fQBOrBx#scrollTo=MYZ7NzAR7Dmw """ + +from __future__ import annotations + import re from collections import defaultdict +from collections.abc import Generator from heapq import heappop, heappush -from typing import Generator, List -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.tokenize.tcc_p import tcc_pos from pythainlp.util import Trie @@ -57,7 +58,7 @@ def _bfs_paths_graph( graph: defaultdict, start: int, goal: int -) -> Generator[List[int], None, None]: +) -> Generator[list[int], None, None]: queue = [(start, [start])] while queue: (vertex, path) = queue.pop(0) @@ -140,9 +141,9 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: def segment( text: str, - custom_dict: Trie = DEFAULT_WORD_DICT_TRIE, + custom_dict: Trie | None = None, safe_mode: bool = False, -) -> List[str]: +) -> list[str]: """Maximal-matching word segmentation constrained by Thai Character Cluster. A dictionary-based word segmentation using maximal matching algorithm, @@ -153,7 +154,7 @@ def segment( :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :param safe_mode: reduce chance for long processing time for long text\ with many ambiguous breaking points, defaults to False @@ -165,7 +166,7 @@ def segment( return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() if not safe_mode or len(text) < _TEXT_SCAN_END: return list(_onecut(text, custom_dict)) diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index c29dc0874..310928ee1 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from sys import stderr -from typing import List from nlpo3 import load_dict as nlpo3_load_dict from nlpo3 import segment as nlpo3_segment @@ -45,7 +45,7 @@ def segment( custom_dict: str = _NLPO3_DEFAULT_DICT_NAME, safe_mode: bool = False, parallel_mode: bool = False, -) -> List[str]: +) -> list[str]: """Break text into tokens. Python binding for nlpO3. It is newmm engine in Rust. diff --git a/pythainlp/tokenize/oskut.py b/pythainlp/tokenize/oskut.py index 3c6037a15..82701f36d 100644 --- a/pythainlp/tokenize/oskut.py +++ b/pythainlp/tokenize/oskut.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper OSKut (Out-of-domain StacKed cut for Word Segmentation). +"""Wrapper OSKut (Out-of-domain StacKed cut for Word Segmentation). Handling Cross- and Out-of-Domain Samples in Thai Word Segmentation Stacked Ensemble Framework and DeepCut as Baseline model (ACL 2021 Findings) :See Also: * `GitHub repository `_ """ -from typing import List + +from __future__ import annotations import oskut @@ -18,7 +17,7 @@ oskut.load_model(engine=DEFAULT_ENGINE) -def segment(text: str, engine: str = "ws") -> List[str]: +def segment(text: str, engine: str = "ws") -> list[str]: global DEFAULT_ENGINE if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py index 5e27116f6..1b8c52f58 100644 --- a/pythainlp/tokenize/pyicu.py +++ b/pythainlp/tokenize/pyicu.py @@ -1,22 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for PyICU word segmentation. This wrapper module uses +"""Wrapper for PyICU word segmentation. This wrapper module uses :class:`icu.BreakIterator` with Thai as :class:`icu.Local` to locate boundaries between words in the text. :See Also: * `GitHub repository `_ """ + +from __future__ import annotations + import re -from typing import List from icu import BreakIterator, Locale bd = BreakIterator.createWordInstance(Locale("th")) + def _gen_words(text: str) -> str: global bd bd.setText(text) @@ -26,14 +27,13 @@ def _gen_words(text: str) -> str: p = q -def segment(text: str) -> List[str]: - """ - :param str text: text to be tokenized into words +def segment(text: str) -> list[str]: + """:param str text: text to be tokenized into words :return: list of words, tokenized from the text """ if not text or not isinstance(text, str): return [] - text = re.sub("([^\u0E00-\u0E7F\n ]+)", " \\1 ", text) + text = re.sub("([^\u0e00-\u0e7f\n ]+)", " \\1 ", text) return list(_gen_words(text)) diff --git a/pythainlp/tokenize/sefr_cut.py b/pythainlp/tokenize/sefr_cut.py index e8434ba49..3381fa692 100644 --- a/pythainlp/tokenize/sefr_cut.py +++ b/pythainlp/tokenize/sefr_cut.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for SEFR CUT Thai word segmentation. SEFR CUT is a +"""Wrapper for SEFR CUT Thai word segmentation. SEFR CUT is a Thai Word Segmentation Models using Stacked Ensemble. :See Also: * `GitHub repository `_ """ -from typing import List + +from __future__ import annotations import sefr_cut @@ -17,7 +16,7 @@ sefr_cut.load_model(engine=DEFAULT_ENGINE) -def segment(text: str, engine: str = "ws1000") -> List[str]: +def segment(text: str, engine: str = "ws1000") -> list[str]: global DEFAULT_ENGINE if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py index 6ea6daade..ccaa56550 100644 --- a/pythainlp/tokenize/ssg.py +++ b/pythainlp/tokenize/ssg.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations from ssg import syllable_tokenize -def segment(text: str) -> List[str]: - """ - Syllable tokenizer using ssg +def segment(text: str) -> list[str]: + """Syllable tokenizer using ssg """ if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py index 81a92b30c..b23fd0586 100644 --- a/pythainlp/tokenize/tcc.py +++ b/pythainlp/tokenize/tcc.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of tokenizer according to Thai Character Clusters (TCCs) +"""The implementation of tokenizer according to Thai Character Clusters (TCCs) rules proposed by `Theeramunkong et al. 2000. \ `_ @@ -13,8 +11,10 @@ `_) * Python code: Korakot Chaovavanich """ + +from __future__ import annotations + import re -from typing import List, Set _RE_TCC = ( """\ @@ -48,9 +48,7 @@ ก็ อึ หึ -""".replace( - "k", "(cc?[d|ิ]?[์])?" - ) +""".replace("k", "(cc?[d|ิ]?[์])?") .replace("c", "[ก-ฮ]") .replace("t", "[่-๋]?") .replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel @@ -61,8 +59,7 @@ def tcc(text: str) -> str: - """ - TCC generator which generates Thai Character Clusters + """TCC generator which generates Thai Character Clusters :param str text: text to be tokenized into character clusters :return: subwords (character clusters) @@ -83,9 +80,8 @@ def tcc(text: str) -> str: p += n -def tcc_pos(text: str) -> Set[int]: - """ - TCC positions +def tcc_pos(text: str) -> set[int]: + """TCC positions :param str text: text to be tokenized into character clusters :return: list of the ending position of subwords @@ -103,14 +99,12 @@ def tcc_pos(text: str) -> Set[int]: return p_set -def segment(text: str) -> List[str]: - """ - Subword segmentation +def segment(text: str) -> list[str]: + """Subword segmentation :param str text: text to be tokenized into character clusters :return: list of subwords (character clusters), tokenized from the text :rtype: list[str] """ - return list(tcc(text)) diff --git a/pythainlp/tokenize/tcc_p.py b/pythainlp/tokenize/tcc_p.py index fe4376bbb..80199475b 100644 --- a/pythainlp/tokenize/tcc_p.py +++ b/pythainlp/tokenize/tcc_p.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of tokenizer according to Thai Character Clusters (TCCs) +"""The implementation of tokenizer according to Thai Character Clusters (TCCs) rules proposed by `Theeramunkong et al. 2000. \ `_ and improved rules that are used in newmm @@ -14,8 +12,10 @@ `_) * Python code: Korakot Chaovavanich """ + +from __future__ import annotations + import re -from typing import List, Set _RE_TCC = ( """\ @@ -48,9 +48,7 @@ ก็ อึ หึ -""".replace( - "k", "(cc?[dิ]?[์])?" - ) +""".replace("k", "(cc?[dิ]?[์])?") .replace("c", "[ก-ฮ]") .replace("t", "[่-๋]?") .replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel @@ -61,8 +59,7 @@ def tcc(text: str) -> str: - """ - TCC generator which generates Thai Character Clusters + """TCC generator which generates Thai Character Clusters :param str text: text to be tokenized into character clusters :return: subwords (character clusters) @@ -83,9 +80,8 @@ def tcc(text: str) -> str: p += n -def tcc_pos(text: str) -> Set[int]: - """ - TCC positions +def tcc_pos(text: str) -> set[int]: + """TCC positions :param str text: text to be tokenized into character clusters :return: list of the ending position of subwords @@ -103,14 +99,12 @@ def tcc_pos(text: str) -> Set[int]: return p_set -def segment(text: str) -> List[str]: - """ - Subword segmentation +def segment(text: str) -> list[str]: + """Subword segmentation :param str text: text to be tokenized into character clusters :return: list of subwords (character clusters), tokenized from the text :rtype: list[str] """ - return list(tcc(text)) diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py new file mode 100644 index 000000000..e6f1059f3 --- /dev/null +++ b/pythainlp/tokenize/thai2fit.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +from functools import lru_cache + +from pythainlp.corpus import get_corpus +from pythainlp.tokenize import Tokenizer + + +@lru_cache +def thai2fit_tokenizer(): + """Lazy load Thai2Fit tokenizer with cache""" + return Tokenizer( + custom_dict=get_corpus("words_th_thai2fit_201810.txt"), engine="mm" + ) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 2c74f8875..35dda6ec0 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2020 Nakhun Chumpolsathien # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of sentence segmentator from Nakhun Chumpolsathien, 2020 +"""The implementation of sentence segmentator from Nakhun Chumpolsathien, 2020 original codes are from: https://github.com/nakhunchumpolsathien/ThaiSum Cite: @@ -15,21 +13,22 @@ school={Beijing Institute of Technology} """ +from __future__ import annotations + import math import operator import re -from typing import List from pythainlp.tokenize import word_tokenize -def list_to_string(list: List[str]) -> str: +def list_to_string(list: list[str]) -> str: string = "".join(list) string = " ".join(string.split()) return string -def middle_cut(sentences: List[str]) -> List[str]: +def middle_cut(sentences: list[str]) -> list[str]: new_text = "" for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) @@ -86,7 +85,7 @@ def middle_cut(sentences: List[str]) -> List[str]: class ThaiSentenceSegmentor: def split_into_sentences( self, text: str, isMiddleCut: bool = False - ) -> List[str]: + ) -> list[str]: # Declare Variables th_alphabets = "([ก-๙])" th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)" diff --git a/pythainlp/tokenize/tltk.py b/pythainlp/tokenize/tltk.py index fb044caf0..90b1ab4b2 100644 --- a/pythainlp/tokenize/tltk.py +++ b/pythainlp/tokenize/tltk.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations try: from tltk.nlp import syl_segment from tltk.nlp import word_segment as tltk_segment except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: if not text or not isinstance(text, str): return [] text = text.replace(" ", "") @@ -22,7 +23,7 @@ def segment(text: str) -> List[str]: return _temp -def syllable_tokenize(text: str) -> List[str]: +def syllable_tokenize(text: str) -> list[str]: if not text or not isinstance(text, str): return [] _temp = syl_segment(text) @@ -32,7 +33,7 @@ def syllable_tokenize(text: str) -> List[str]: return _temp -def sent_tokenize(text: str) -> List[str]: +def sent_tokenize(text: str) -> list[str]: text = text.replace(" ", "") _temp = tltk_segment(text).replace("", " ").replace("|", "") _temp = _temp.split("") diff --git a/pythainlp/tokenize/wtsplit.py b/pythainlp/tokenize/wtsplit.py index 952f605a8..c4d688c33 100644 --- a/pythainlp/tokenize/wtsplit.py +++ b/pythainlp/tokenize/wtsplit.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Where's the Point? Self-Supervised Multilingual Punctuation-Agnostic Sentence Segmentation +"""Where's the Point? Self-Supervised Multilingual Punctuation-Agnostic Sentence Segmentation GitHub: https://github.com/bminixhofer/wtpsplit """ -from typing import List + +from __future__ import annotations from wtpsplit import WtP @@ -22,7 +21,7 @@ def _tokenize( tokenize: str = "sentence", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[str]: +) -> list[str]: global _MODEL_NAME, _MODEL if _MODEL_NAME != model: @@ -60,7 +59,7 @@ def tokenize( tokenize: str = "sentence", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[str]: +) -> list[str]: _model_load = "" if size == "tiny": _model_load = "wtp-bert-tiny" diff --git a/pythainlp/tools/__init__.py b/pythainlp/tools/__init__.py index 1d36048d1..3d6743c66 100644 --- a/pythainlp/tools/__init__.py +++ b/pythainlp/tools/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/tools/core.py b/pythainlp/tools/core.py index f7c190745..c4fc9e6c4 100644 --- a/pythainlp/tools/core.py +++ b/pythainlp/tools/core.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Generic support functions for PyThaiNLP. """ -Generic support functions for PyThaiNLP. -""" + +from __future__ import annotations import sys import warnings @@ -33,6 +33,7 @@ def warn_deprecation( message += f" Please use '{replacing_func}' instead." warnings.warn(message, DeprecationWarning, stacklevel=2) + def safe_print(text: str): """Print text to console, handling UnicodeEncodeError. diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index c940d2929..89c445eae 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations -import numpy as np +import math +import random THAI_CHARACTERS_WITHOUT_SHIFT = [ "ผปแอิืทมใฝ", @@ -51,7 +51,7 @@ def search_location_of_character(char: str): def find_neighbour_locations( loc: tuple, char: str, - kernel: List = [(-1, -1), (-1, 0), (1, 1), (0, 1), (0, -1), (1, 0)], + kernel: list = [(-1, -1), (-1, 0), (1, 1), (0, 1), (0, -1), (1, 0)], ): language_ix, is_shift, row, pos = loc @@ -105,8 +105,7 @@ def find_misspell_candidates(char: str, verbose: bool = False): def misspell(sentence: str, ratio: float = 0.05): - """ - Simulate some misspellings of the input sentence. + """Simulate some misspellings of the input sentence. The number of misspelled locations is governed by ratio. :params str sentence: sentence to be misspelled @@ -126,10 +125,8 @@ def misspell(sentence: str, ratio: float = 0.05): # output: ภาษาไทยปรากฏครั้งแรกในกุทธศักราช 1727 """ - num_misspells = np.floor(len(sentence) * ratio).astype(int) - positions = np.random.choice( - len(sentence), size=num_misspells, replace=False - ) + num_misspells = math.floor(len(sentence) * ratio) + positions = random.sample(range(len(sentence)), k=num_misspells) # convert strings to array of characters misspelled = list(sentence) @@ -138,7 +135,7 @@ def misspell(sentence: str, ratio: float = 0.05): if potential_candidates is None: continue - candidate = np.random.choice(potential_candidates) + candidate = random.choice(potential_candidates) misspelled[pos] = candidate diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index ddf81a6e7..9af51e60a 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -1,12 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PyThaiNLP data tools +"""PyThaiNLP data tools For text processing and text conversion, see pythainlp.util """ + +from __future__ import annotations + import os from pythainlp import __file__ as pythainlp_file @@ -15,8 +16,7 @@ def get_full_data_path(path: str) -> str: - """ - This function joins path of :mod:`pythainlp` data directory and the + """This function joins path of :mod:`pythainlp` data directory and the given path, and returns the full path. :return: full path given the name of dataset @@ -27,15 +27,14 @@ def get_full_data_path(path: str) -> str: from pythainlp.tools import get_full_data_path - get_full_data_path('ttc_freq.txt') + get_full_data_path("ttc_freq.txt") # output: '/root/pythainlp-data/ttc_freq.txt' """ return os.path.join(get_pythainlp_data_path(), path) def get_pythainlp_data_path() -> str: - """ - Returns the full path where PyThaiNLP keeps its (downloaded) data. + """Returns the full path where PyThaiNLP keeps its (downloaded) data. If the directory does not yet exist, it will be created. The path can be specified through the environment variable :envvar:`PYTHAINLP_DATA_DIR`. By default, `~/pythainlp-data` @@ -61,8 +60,7 @@ def get_pythainlp_data_path() -> str: def get_pythainlp_path() -> str: - """ - This function returns full path of PyThaiNLP codes + """This function returns full path of PyThaiNLP codes :return: full path of :mod:`pythainlp` codes :rtype: str diff --git a/pythainlp/translate/__init__.py b/pythainlp/translate/__init__.py index ce83658cc..f126f56db 100644 --- a/pythainlp/translate/__init__.py +++ b/pythainlp/translate/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Language translation. +"""Language translation. """ __all__ = ["Translate", "ThZhTranslator", "ZhThTranslator", "word_translate"] diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index f7387271f..d49cbb289 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -1,13 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations class Translate: - """ - Machine Translation + """Machine Translation """ def __init__( @@ -17,8 +15,7 @@ def __init__( engine: str = "default", use_gpu: bool = False, ) -> None: - """ - :param str src_lang: source language + """:param str src_lang: source language :param str target_lang: target language :param str engine: machine translation engine :param bool use_gpu: load model using GPU (Default is False) @@ -86,8 +83,7 @@ def load_model(self): raise ValueError("Not support language!") def translate(self, text: str) -> str: - """ - Translate text + """Translate text :param str text: input text in source language :return: translated text in target language @@ -99,13 +95,9 @@ def translate(self, text: str) -> str: def word_translate( - word: str, - src: str, - target: str, - engine: str="word2word" - ) -> Union[List[str], None]: - """ - Translate word from source language to target language. + word: str, src: str, target: str, engine: str = "word2word" +) -> list[str] | None: + """Translate word from source language to target language. :param str word: text :param str src: src language @@ -119,18 +111,21 @@ def word_translate( Translate word from Thai to English:: from pythainlp.translate import word_translate - print(word_translate("แมว","th","en")) + + print(word_translate("แมว", "th", "en")) # output: ['cat', 'cats', 'kitty', 'kitten', 'Cat'] Translate word from English to Thai:: from pythainlp.translate import word_translate - print(word_translate("cat","en","th")) + + print(word_translate("cat", "en", "th")) # output: ['แมว', 'แมวป่า', 'ข่วน', 'เลี้ยง', 'อาหาร'] """ - if engine=="word2word": + if engine == "word2word": from .word2word_translate import translate + return translate(word=word, src=src, target=target) else: raise NotImplementedError( diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index f23065daf..2a734b4dd 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -1,25 +1,30 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -English-Thai Machine Translation +"""English-Thai Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute Website: https://airesearch.in.th/releases/machine-translation-models/ """ + +from __future__ import annotations + import os try: from fairseq.models.transformer import TransformerModel except ImportError: - raise ImportError("Not found fairseq! Please install fairseq by pip install fairseq") + raise ImportError( + "Not found fairseq! Please install fairseq by pip install fairseq" + ) try: from sacremoses import MosesTokenizer except ImportError: - raise ImportError("Not found sacremoses! Please install sacremoses by pip install sacremoses") + raise ImportError( + "Not found sacremoses! Please install sacremoses by pip install sacremoses" + ) from pythainlp.corpus import download, get_corpus_path @@ -42,16 +47,14 @@ def _download_install(name: str) -> None: def download_model_all() -> None: - """ - Download all translation models in advance + """Download all translation models in advance """ _download_install(_EN_TH_MODEL_NAME) _download_install(_TH_EN_MODEL_NAME) class EnThTranslator: - """ - English-Thai Machine Translation + """English-Thai Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute @@ -83,8 +86,7 @@ def __init__(self, use_gpu: bool = False): self._model = self._model.cuda() def translate(self, text: str) -> str: - """ - Translate text from English to Thai + """Translate text from English to Thai :param str text: input text in source language :return: translated text in target language @@ -108,8 +110,7 @@ def translate(self, text: str) -> str: class ThEnTranslator: - """ - Thai-English Machine Translation + """Thai-English Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute @@ -146,8 +147,7 @@ def __init__(self, use_gpu: bool = False): self._model.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to English + """Translate text from Thai to English :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index 2699a8dd4..b774b9a50 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from transformers import M2M100ForConditionalGeneration from .tokenization_small100 import SMALL100Tokenizer class Small100Translator: - """ - Machine Translation using small100 model + """Machine Translation using small100 model - Huggingface https://huggingface.co/alirezamsh/small100 @@ -22,14 +22,15 @@ def __init__( pretrained: str = "alirezamsh/small100", ) -> None: self.pretrained = pretrained - self.model = M2M100ForConditionalGeneration.from_pretrained(self.pretrained) + self.model = M2M100ForConditionalGeneration.from_pretrained( + self.pretrained + ) self.tgt_lang = None if use_gpu: self.model = self.model.cuda() - def translate(self, text: str, tgt_lang: str="en") -> str: - """ - Translate text from X to X + def translate(self, text: str, tgt_lang: str = "en") -> str: + """Translate text from X to X :param str text: input text in source language :param str tgt_lang: target language @@ -57,10 +58,14 @@ def translate(self, text: str, tgt_lang: str="en") -> str: # output: 'Test du système' """ - if tgt_lang!=self.tgt_lang: - self.tokenizer = SMALL100Tokenizer.from_pretrained(self.pretrained, tgt_lang=tgt_lang) + if tgt_lang != self.tgt_lang: + self.tokenizer = SMALL100Tokenizer.from_pretrained( + self.pretrained, tgt_lang=tgt_lang + ) self.tgt_lang = tgt_lang self.translated = self.model.generate( **self.tokenizer(text, return_tensors="pt") ) - return self.tokenizer.batch_decode(self.translated, skip_special_tokens=True)[0] + return self.tokenizer.batch_decode( + self.translated, skip_special_tokens=True + )[0] diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 4857143f7..a408c7896 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-French Machine Translation +"""Thai-French Machine Translation Trained by OPUS Corpus @@ -14,10 +12,11 @@ - Huggingface https://huggingface.co/Helsinki-NLP/opus-mt-th-fr """ +from __future__ import annotations + class ThFrTranslator: - """ - Thai-French Machine Translation + """Thai-French Machine Translation Trained by OPUS Corpus @@ -36,14 +35,14 @@ def __init__( pretrained: str = "Helsinki-NLP/opus-mt-th-fr", ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + self.tokenizer_thzh = AutoTokenizer.from_pretrained(pretrained) self.model_thzh = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: self.model_thzh = self.model_thzh.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to French + """Translate text from Thai to French :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 9a11dc2da..96d641179 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -21,11 +21,13 @@ # limitations under the License. """Tokenization classes for SMALL100.""" +from __future__ import annotations + import json import os from pathlib import Path from shutil import copyfile -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any import sentencepiece from transformers.tokenization_utils import BatchEncoding, PreTrainedTokenizer @@ -62,10 +64,10 @@ class SMALL100Tokenizer(PreTrainedTokenizer): - """ - Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). + """Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. + Args: vocab_file (`str`): Path to the vocabulary file. @@ -99,6 +101,7 @@ class SMALL100Tokenizer(PreTrainedTokenizer): using forward-filtering-and-backward-sampling algorithm. - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. + Examples: ```python >>> from tokenization_small100 import SMALL100Tokenizer @@ -107,15 +110,17 @@ class SMALL100Tokenizer(PreTrainedTokenizer): >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria" >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt") >>> model(**model_inputs) # should work - ```""" + ``` + + """ vocab_files_names = VOCAB_FILES_NAMES max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP model_input_names = ["input_ids", "attention_mask"] - prefix_tokens: List[int] = [] - suffix_tokens: List[int] = [] + prefix_tokens: list[int] = [] + suffix_tokens: list[int] = [] def __init__( self, @@ -128,7 +133,7 @@ def __init__( pad_token="", unk_token="", language_codes="m2m100", - sp_model_kwargs: Optional[Dict[str, Any]] = None, + sp_model_kwargs: dict[str, Any] | None = None, num_madeup_words=8, **kwargs, ) -> None: @@ -136,9 +141,13 @@ def __init__( self.language_codes = language_codes fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes] - self.lang_code_to_token = {lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code} + self.lang_code_to_token = { + lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code + } - kwargs["additional_special_tokens"] = kwargs.get("additional_special_tokens", []) + kwargs["additional_special_tokens"] = kwargs.get( + "additional_special_tokens", [] + ) kwargs["additional_special_tokens"] += [ self.get_lang_token(lang_code) for lang_code in fairseq_language_code @@ -167,9 +176,13 @@ def __init__( self.encoder_size = len(self.encoder) self.lang_token_to_id = { - self.get_lang_token(lang_code): self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code) + self.get_lang_token(lang_code): self.encoder_size + i + for i, lang_code in enumerate(fairseq_language_code) + } + self.lang_code_to_id = { + lang_code: self.encoder_size + i + for i, lang_code in enumerate(fairseq_language_code) } - self.lang_code_to_id = {lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)} self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()} self._tgt_lang = tgt_lang if tgt_lang is not None else "en" @@ -191,7 +204,7 @@ def tgt_lang(self, new_tgt_lang: str) -> None: self._tgt_lang = new_tgt_lang self.set_lang_special_tokens(self._tgt_lang) - def _tokenize(self, text: str) -> List[str]: + def _tokenize(self, text: str) -> list[str]: return self.sp_model.encode(text, out_type=str) def _convert_token_to_id(self, token): @@ -205,78 +218,99 @@ def _convert_id_to_token(self, index: int) -> str: return self.id_to_lang_token[index] return self.decoder.get(index, self.unk_token) - def convert_tokens_to_string(self, tokens: List[str]) -> str: + def convert_tokens_to_string(self, tokens: list[str]) -> str: """Converts a sequence of tokens (strings for sub-words) in a single string.""" return self.sp_model.decode(tokens) def get_special_tokens_mask( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False - ) -> List[int]: - """ - Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding - special tokens using the tokenizer `prepare_for_model` method. + self, + token_ids_0: list[int], + token_ids_1: list[int] | None = None, + already_has_special_tokens: bool = False, + ) -> list[int]: + """Retrieve sequence IDs from a token list that has no special tokens + added. This method is called when adding special tokens using the + tokenizer `prepare_for_model` method. + Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not the token list is already formatted with special tokens for the model. + Whether or not the token list is already formatted with + special tokens for the model. + Returns: - `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. - """ + `List[int]`: A list of integers in the range [0, 1]: + 1 for a special token, 0 for a sequence token. + """ if already_has_special_tokens: return super().get_special_tokens_mask( - token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, ) prefix_ones = [1] * len(self.prefix_tokens) suffix_ones = [1] * len(self.suffix_tokens) if token_ids_1 is None: return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones - return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones + return ( + prefix_ones + + ([0] * len(token_ids_0)) + + ([0] * len(token_ids_1)) + + suffix_ones + ) def build_inputs_with_special_tokens( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. An MBART sequence has the following format, where `X` represents the sequence: + self, token_ids_0: list[int], token_ids_1: list[int] | None = None + ) -> list[int]: + """Build model inputs from a sequence or a pair of sequence for + sequence classification tasks by concatenating and + adding special tokens. An MBART sequence has the following format, + where `X` represents the sequence: - `input_ids` (for encoder) `X [eos, src_lang_code]` - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]` - BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a - separator. + + BOS is never used. Pairs of sequences are not the expected use case, + but they will be handled without aseparator. + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. + Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + `List[int]`: List of [input IDs](../glossary#input-ids) with the + appropriate special tokens. + """ if token_ids_1 is None: if self.prefix_tokens is None: return token_ids_0 + self.suffix_tokens else: return self.prefix_tokens + token_ids_0 + self.suffix_tokens - # We don't expect to process pairs, but leave the pair logic for API consistency + # We don't expect to process pairs, + # but leave the pair logic for API consistency if self.prefix_tokens is None: return token_ids_0 + token_ids_1 + self.suffix_tokens else: return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens - def get_vocab(self) -> Dict: + def get_vocab(self) -> dict: vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} vocab.update(self.added_tokens_encoder) return vocab - def __getstate__(self) -> Dict: + def __getstate__(self) -> dict: state = self.__dict__.copy() state["sp_model"] = None return state - def __setstate__(self, d: Dict) -> None: + def __setstate__(self, d: dict) -> None: self.__dict__ = d # for backward compatibility @@ -285,20 +319,26 @@ def __setstate__(self, d: Dict) -> None: self.sp_model = load_spm(self.spm_file, self.sp_model_kwargs) - def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + def save_vocabulary( + self, save_directory: str, filename_prefix: str | None = None + ) -> tuple[str]: save_dir = Path(save_directory) if not save_dir.is_dir(): raise OSError(f"{save_directory} should be a directory") vocab_save_path = save_dir / ( - (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"] + (filename_prefix + "-" if filename_prefix else "") + + self.vocab_files_names["vocab_file"] ) spm_save_path = save_dir / ( - (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"] + (filename_prefix + "-" if filename_prefix else "") + + self.vocab_files_names["spm_file"] ) save_json(self.encoder, vocab_save_path) - if os.path.abspath(self.spm_file) != os.path.abspath(spm_save_path) and os.path.isfile(self.spm_file): + if os.path.abspath(self.spm_file) != os.path.abspath( + spm_save_path + ) and os.path.isfile(self.spm_file): copyfile(self.spm_file, spm_save_path) elif not os.path.isfile(self.spm_file): with open(spm_save_path, "wb") as fi: @@ -309,8 +349,8 @@ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = def prepare_seq2seq_batch( self, - src_texts: List[str], - tgt_texts: Optional[List[str]] = None, + src_texts: list[str], + tgt_texts: list[str] | None = None, tgt_lang: str = "ro", **kwargs, ) -> BatchEncoding: @@ -318,8 +358,11 @@ def prepare_seq2seq_batch( self.set_lang_special_tokens(self.tgt_lang) return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs) - def _build_translation_inputs(self, raw_inputs, tgt_lang: Optional[str], **extra_kwargs): - """Used by translation pipeline, to prepare inputs for the generate function""" + def _build_translation_inputs( + self, raw_inputs, tgt_lang: str | None, **extra_kwargs + ): + """Used by translation pipeline, to prepare inputs for the generate + function""" if tgt_lang is None: raise ValueError("Translation requires a `tgt_lang` for this model") self.tgt_lang = tgt_lang @@ -334,7 +377,8 @@ def _switch_to_target_mode(self): self.suffix_tokens = [self.eos_token_id] def set_lang_special_tokens(self, src_lang: str) -> None: - """Reset the special tokens to the tgt lang setting. No prefix and suffix=[eos, tgt_lang_code].""" + """Reset the special tokens to the tgt lang setting. + No prefix and suffix=[eos, tgt_lang_code].""" lang_token = self.get_lang_token(src_lang) self.cur_lang_id = self.lang_token_to_id[lang_token] self.prefix_tokens = [self.cur_lang_id] @@ -348,14 +392,16 @@ def get_lang_id(self, lang: str) -> int: return self.lang_token_to_id[lang_token] -def load_spm(path: str, sp_model_kwargs: Dict[str, Any]) -> sentencepiece.SentencePieceProcessor: +def load_spm( + path: str, sp_model_kwargs: dict[str, Any] +) -> sentencepiece.SentencePieceProcessor: spm = sentencepiece.SentencePieceProcessor(**sp_model_kwargs) spm.Load(str(path)) return spm -def load_json(path: str) -> Union[Dict, List]: - with open(path, "r") as f: +def load_json(path: str) -> dict | list: + with open(path) as f: return json.load(f) diff --git a/pythainlp/translate/word2word_translate.py b/pythainlp/translate/word2word_translate.py index fe027ff33..87bd7aca6 100644 --- a/pythainlp/translate/word2word_translate.py +++ b/pythainlp/translate/word2word_translate.py @@ -1,75 +1,78 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations + from word2word import Word2word -support_list = set(['zh_tw', - 'el', - 'te', - 'hu', - 'eu', - 'ko', - 'ru', - 'lv', - 'bg', - 'sk', - 'vi', - 'gl', - 'et', - 'ta', - 'fa', - 'it', - 'ms', - 'id', - 'pt', - 'fr', - 'sr', - 'mk', - 'sv', - 'si', - 'en', - 'ka', - 'uk', - 'sl', - 'hi', - 'ca', - 'lt', - 'es', - 'no', - 'de', - 'he', - 'cs', - 'ze_zh', - 'fi', - 'pl', - 'tl', - 'is', - 'ze_en', - 'kk', - 'bn', - 'tr', - 'ur', - 'pt_br', - 'ar', - 'ro', - 'bs', - 'ml', - 'zh_cn', - 'da', - 'hr', - 'sq', - 'af', - 'eo', - 'nl', - 'ja', - 'th']) +support_list = set( + [ + "zh_tw", + "el", + "te", + "hu", + "eu", + "ko", + "ru", + "lv", + "bg", + "sk", + "vi", + "gl", + "et", + "ta", + "fa", + "it", + "ms", + "id", + "pt", + "fr", + "sr", + "mk", + "sv", + "si", + "en", + "ka", + "uk", + "sl", + "hi", + "ca", + "lt", + "es", + "no", + "de", + "he", + "cs", + "ze_zh", + "fi", + "pl", + "tl", + "is", + "ze_en", + "kk", + "bn", + "tr", + "ur", + "pt_br", + "ar", + "ro", + "bs", + "ml", + "zh_cn", + "da", + "hr", + "sq", + "af", + "eo", + "nl", + "ja", + "th", + ] +) -def translate(word: str, src: str, target: str) -> Union[List[str], None]: - """ - Word translate +def translate(word: str, src: str, target: str) -> list[str] | None: + """Word translate :param str word: text :param str src: src language @@ -78,10 +81,8 @@ def translate(word: str, src: str, target: str) -> Union[List[str], None]: :rtype: Union[List[str], None] """ if src not in support_list or target not in support_list: - raise NotImplementedError( - f"word2word doesn't support {src}-{target}." - ) - elif src==target: + raise NotImplementedError(f"word2word doesn't support {src}-{target}.") + elif src == target: return [word] _engine = Word2word(src, target) - return _engine(word) \ No newline at end of file + return _engine(word) diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 1ac82eeb5..84a3b75b5 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Lalita Chinese-Thai Machine Translation +"""Lalita Chinese-Thai Machine Translation from AI builder @@ -11,10 +9,11 @@ - Facebook post https://web.facebook.com/aibuildersx/posts/166736255494822 """ +from __future__ import annotations + class ThZhTranslator: - """ - Thai-Chinese Machine Translation + """Thai-Chinese Machine Translation from Lalita @ AI builder @@ -30,14 +29,14 @@ def __init__( pretrained: str = "Lalita/marianmt-th-zh_cn", ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + self.tokenizer_thzh = AutoTokenizer.from_pretrained(pretrained) self.model_thzh = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: self.model_thzh = self.model_thzh.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to Chinese + """Translate text from Thai to Chinese :param str text: input text in source language :return: translated text in target language @@ -65,8 +64,7 @@ def translate(self, text: str) -> str: class ZhThTranslator: - """ - Chinese-Thai Machine Translation + """Chinese-Thai Machine Translation from Lalita @ AI builder @@ -87,8 +85,7 @@ def __init__( self.model_zhth.cuda() def translate(self, text: str) -> str: - """ - Translate text from Chinese to Thai + """Translate text from Chinese to Thai :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 4dd1ab9b9..bee56ad24 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliteration. +"""Transliteration. """ __all__ = [ diff --git a/pythainlp/transliterate/core.py b/pythainlp/transliterate/core.py index 6881b071d..112d0bd4d 100644 --- a/pythainlp/transliterate/core.py +++ b/pythainlp/transliterate/core.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations DEFAULT_ROMANIZE_ENGINE = "royin" DEFAULT_TRANSLITERATE_ENGINE = "thaig2p" @@ -13,8 +13,7 @@ def romanize( engine: str = DEFAULT_ROMANIZE_ENGINE, fallback_engine: str = DEFAULT_ROMANIZE_ENGINE, ) -> str: - """ - This function renders Thai word in the Latin alphabet or "romanization", + """This function renders Thai word in the Latin alphabet or "romanization", using the Royal Thai General System of Transcription (RTGS) [#rtgs_transcription]_. RTGS is the official system published by the Royal Institute of Thailand. (Thai: ถอดเสียงภาษาไทยเป็นอักษรละติน) @@ -22,9 +21,11 @@ def romanize( :param str text: A Thai word to be romanized. \ The input should not include whitespace because \ the function is support subwords by spliting whitespace. - :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx, 'tltk', and 'lookup'. See more in options for engine section. - :param str fallback_engine: If engine equals 'lookup', use `fallback_engine` for words that are not in the transliteration dict. - No effect on other engines. Default to 'royin'. + :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx, + 'tltk', and 'lookup'. See more in options for engine section. + :param str fallback_engine: If engine equals 'lookup', + use `fallback_engine` for words that are not in the lookup dictionary. + No effect on other engines. Default to 'royin'. :return: A string of a Thai word rendered in the Latin alphabet. :rtype: str @@ -92,17 +93,16 @@ def select_romanize_engine(engine: str): else: rom_engine = select_romanize_engine(engine) trans_word = [] - for subword in text.split(' '): + for subword in text.split(" "): trans_word.append(rom_engine(subword)) - new_word = ' '.join(trans_word) + new_word = " ".join(trans_word) return new_word def transliterate( text: str, engine: str = DEFAULT_TRANSLITERATE_ENGINE ) -> str: - """ - This function transliterates Thai text. + """This function transliterates Thai text. :param str text: Thai text to be transliterated :param str engine: 'icu', 'ipa', or 'thaig2p' (default) @@ -161,7 +161,6 @@ def transliterate( transliterate("ภาพยนตร์", engine="iso_11940") # output: 'p̣hāphyntr' """ - if not text or not isinstance(text, str): return "" @@ -186,8 +185,7 @@ def transliterate( def pronunciate(word: str, engine: str = DEFAULT_PRONUNCIATE_ENGINE) -> str: - """ - This function pronunciates Thai word. + """This function pronunciates Thai word. :param str word: Thai text to be pronunciated :param str engine: 'w2p' (default) diff --git a/pythainlp/transliterate/ipa.py b/pythainlp/transliterate/ipa.py index 0193be693..7432347f2 100644 --- a/pythainlp/transliterate/ipa.py +++ b/pythainlp/transliterate/ipa.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating text to International Phonetic Alphabet (IPA) +"""Transliterating text to International Phonetic Alphabet (IPA) Using epitran :See Also: * `GitHub \ `_ """ -from typing import List + +from __future__ import annotations import epitran @@ -21,9 +20,9 @@ def transliterate(text: str) -> str: return _EPI_THA.transliterate(text) -def trans_list(text: str) -> List[str]: +def trans_list(text: str) -> list[str]: return _EPI_THA.trans_list(text) -def xsampa_list(text: str) -> List[str]: +def xsampa_list(text: str) -> list[str]: return _EPI_THA.xsampa_list(text) diff --git a/pythainlp/transliterate/iso_11940.py b/pythainlp/transliterate/iso_11940.py index c4b784ca1..0141e9a2f 100644 --- a/pythainlp/transliterate/iso_11940.py +++ b/pythainlp/transliterate/iso_11940.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating Thai text using ISO 11940 +"""Transliterating Thai text using ISO 11940 :See Also: * `Wikipedia \ `_ """ + +from __future__ import annotations + _consonants = { "ก": "k", "ข": "k̄h", @@ -131,8 +132,7 @@ def transliterate(word: str) -> str: - """ - Use ISO 11940 for transliteration + """Use ISO 11940 for transliteration :param str text: Thai text to be transliterated. :return: A string indicating how the text should be pronounced, according to ISO 11940. """ diff --git a/pythainlp/transliterate/lookup.py b/pythainlp/transliterate/lookup.py index 02b7dd49c..bc1e248dc 100644 --- a/pythainlp/transliterate/lookup.py +++ b/pythainlp/transliterate/lookup.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Look up romanized Thai words in a predefined dictionary compiled by Wannaphong, 2022. +"""Look up romanized Thai words in a predefined dictionary compiled by Wannaphong, 2022. Wannaphong Phatthiyaphaibun. (2022). wannaphong/thai-english-transliteration-dictionary: v1.4 (v1.4). Zenodo. https://doi.org/10.5281/zenodo.6716672 """ -from typing import Callable, Optional +from __future__ import annotations + +from collections.abc import Callable from pythainlp.corpus.th_en_translit import ( TRANSLITERATE_DICT, @@ -21,9 +21,8 @@ _TRANSLITERATE_IDX = 0 -def follow_rtgs(text: str) -> Optional[bool]: - """ - Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). +def follow_rtgs(text: str) -> bool | None: + """Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). :param str text: Text to look up. Must be a self-contained word. :return: True if text follows the definition by RTGS, False otherwise. `None` means unverified or unknown word. @@ -40,8 +39,7 @@ def follow_rtgs(text: str) -> Optional[bool]: def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """ - Romanize one word. Look up first, call `fallback_func` if not found. + """Romanize one word. Look up first, call `fallback_func` if not found. """ try: # try to get 0-th idx of look up result, simply ignore other possible variations. @@ -56,8 +54,7 @@ def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: def romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """ - Render Thai words in Latin alphabet by looking up + """Render Thai words in Latin alphabet by looking up Thai-English transliteration dictionary. :param str text: Thai text to be romanized diff --git a/pythainlp/transliterate/pyicu.py b/pythainlp/transliterate/pyicu.py index 9a99066eb..4dc431e85 100644 --- a/pythainlp/transliterate/pyicu.py +++ b/pythainlp/transliterate/pyicu.py @@ -1,23 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating text to International Phonetic Alphabet (IPA) +"""Transliterating text to International Phonetic Alphabet (IPA) Using International Components for Unicode (ICU) :See Also: * `GitHub \ `_ """ + +from __future__ import annotations + from icu import Transliterator _ICU_THAI_TO_LATIN = Transliterator.createInstance("Thai-Latin") def transliterate(text: str) -> str: - """ - Use ICU (International Components for Unicode) for transliteration + """Use ICU (International Components for Unicode) for transliteration :param str text: Thai text to be transliterated. :return: A string of Internaitonal Phonetic Alphabets indicating how the text should be pronounced. """ diff --git a/pythainlp/transliterate/royin.py b/pythainlp/transliterate/royin.py index 5a3ffc3e7..d8b9e02ba 100644 --- a/pythainlp/transliterate/royin.py +++ b/pythainlp/transliterate/royin.py @@ -1,15 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The Royal Thai General System of Transcription (RTGS) +"""The Royal Thai General System of Transcription (RTGS) is the official system for rendering Thai words in the Latin alphabet. It was published by the Royal Institute of Thailand. :See Also: * `Wikipedia `_ """ + +from __future__ import annotations + import re from pythainlp import thai_consonants, word_tokenize @@ -136,8 +137,7 @@ def _normalize(word: str) -> str: - """ - Remove silence, no sound, and tonal characters. + """Remove silence, no sound, and tonal characters. ตัดอักษรที่ไม่ออกเสียง (การันต์ ไปยาลน้อย ไม้ยมก*) และวรรณยุกต์ทิ้ง """ @@ -157,7 +157,7 @@ def _replace_consonants(word: str, consonants: str) -> str: _LO_LING = "\u0e25" # ล _WO_WAEN = "\u0e27" # ว _DOUBLE_RO_RUA = _RO_RUA + _RO_RUA - + # Consonants that can be second in a cluster _CLUSTER_SECOND = {_RO_RUA, _LO_LING, _WO_WAEN} @@ -168,7 +168,7 @@ def _replace_consonants(word: str, consonants: str) -> str: mod_chars = [] j = 0 # j is the index of consonants vowel_seen = False # Track if we've seen a vowel (non-consonant character) - + for i in range(len(word)): if skip: skip = False @@ -194,24 +194,32 @@ def _replace_consonants(word: str, consonants: str) -> str: elif not vowel_seen: # Building initial consonant cluster # Check if we've added any actual initial consonants (non-empty romanized characters) # We check for non-vowel characters since mod_chars contains romanized output - has_initial = any(c and c not in _ROMANIZED_VOWELS for c in mod_chars) - + has_initial = any( + c and c not in _ROMANIZED_VOWELS for c in mod_chars + ) + if not has_initial: # First consonant in the cluster initial = _CONSONANTS[consonants[j]][0] - if initial: # Only append if not empty (e.g., อ has empty initial) + if ( + initial + ): # Only append if not empty (e.g., อ has empty initial) mod_chars.append(initial) j += 1 else: # Check if this consonant can be part of a cluster is_cluster_consonant = word[i] in _CLUSTER_SECOND - is_last_char = (i + 1 >= len(word)) - has_vowel_next = not is_last_char and word[i+1] not in _CONSONANTS - + is_last_char = i + 1 >= len(word) + has_vowel_next = ( + not is_last_char and word[i + 1] not in _CONSONANTS + ) + # Cluster consonants (ร/r, ล/l, ว/w) are part of initial cluster if: # - followed by a vowel, OR # - not the last character (e.g., กรม/krom: ก/k+ร/r are cluster, ม/m is final) - if is_cluster_consonant and (has_vowel_next or not is_last_char): + if is_cluster_consonant and ( + has_vowel_next or not is_last_char + ): # This is part of initial cluster (ร/r, ล/l, or ว/w after first consonant) mod_chars.append(_CONSONANTS[consonants[j]][0]) j += 1 @@ -244,7 +252,9 @@ def _replace_consonants(word: str, consonants: str) -> str: vowel_seen = True j += 1 else: # After vowel - could be final consonant or start of new syllable - has_vowel_next = (i + 1 < len(word) and word[i+1] not in _CONSONANTS) + has_vowel_next = ( + i + 1 < len(word) and word[i + 1] not in _CONSONANTS + ) if has_vowel_next: # Consonant followed by vowel - start of new syllable mod_chars.append(_CONSONANTS[consonants[j]][0]) @@ -260,9 +270,9 @@ def _replace_consonants(word: str, consonants: str) -> str: # support function for romanize() def _romanize(word: str) -> str: # Special case: single ห character should be empty (silent) - if word == 'ห': - return '' - + if word == "ห": + return "" + word = _replace_vowels(_normalize(word)) consonants = _RE_CONSONANT.findall(word) @@ -276,14 +286,15 @@ def _romanize(word: str) -> str: return word -def _should_add_syllable_separator(prev_word: str, curr_word: str, prev_romanized: str) -> bool: - """ - Determine if 'a' should be added between two romanized syllables. - +def _should_add_syllable_separator( + prev_word: str, curr_word: str, prev_romanized: str +) -> bool: + """Determine if 'a' should be added between two romanized syllables. + This applies when: - Previous word has explicit vowel and ends with consonant - Current word is a 2-consonant cluster with no vowels (e.g., 'กร') - + :param prev_word: The previous Thai word/token :param curr_word: The current Thai word/token :param prev_romanized: The romanized form of the previous word @@ -291,22 +302,24 @@ def _should_add_syllable_separator(prev_word: str, curr_word: str, prev_romanize """ if not prev_romanized or len(curr_word) < 2: return False - + # Check if previous word has explicit vowel prev_normalized = _normalize(prev_word) prev_after_vowels = _replace_vowels(prev_normalized) prev_consonants = _RE_CONSONANT.findall(prev_word) has_explicit_vowel_prev = len(prev_after_vowels) > len(prev_consonants) - + # Check if current word is 2 Thai consonants with no vowel consonants_in_word = _RE_CONSONANT.findall(curr_word) vowels_in_word = len(curr_word) - len(consonants_in_word) - + # Add 'a' if conditions are met - return (has_explicit_vowel_prev and - len(consonants_in_word) == 2 and - vowels_in_word == 0 and - prev_romanized[-1] not in _ROMANIZED_VOWELS) + return ( + has_explicit_vowel_prev + and len(consonants_in_word) == 2 + and vowels_in_word == 0 + and prev_romanized[-1] not in _ROMANIZED_VOWELS + ) def romanize(text: str) -> str: @@ -322,17 +335,17 @@ def romanize(text: str) -> str: """ words = word_tokenize(text) romanized_words = [] - + for i, word in enumerate(words): romanized = _romanize(word) - + # Check if we need to add syllable separator 'a' if i > 0 and romanized: - prev_word = words[i-1] - prev_romanized = romanized_words[-1] if romanized_words else '' + prev_word = words[i - 1] + prev_romanized = romanized_words[-1] if romanized_words else "" if _should_add_syllable_separator(prev_word, word, prev_romanized): - romanized = 'a' + romanized - + romanized = "a" + romanized + romanized_words.append(romanized) - + return "".join(romanized_words) diff --git a/pythainlp/transliterate/spoonerism.py b/pythainlp/transliterate/spoonerism.py index 1c91b3f23..ad57eac4a 100644 --- a/pythainlp/transliterate/spoonerism.py +++ b/pythainlp/transliterate/spoonerism.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp import thai_consonants from pythainlp.transliterate import pronunciate @@ -9,8 +10,7 @@ def puan(word: str, show_pronunciation: bool = True) -> str: - """ - Thai Spoonerism + """Thai Spoonerism This function converts Thai word to spoonerism word. diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 334668b8b..d7a2d6b0e 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Romanization of Thai words based on machine-learnt engine ("thai2rom") """ -Romanization of Thai words based on machine-learnt engine ("thai2rom") -""" + +from __future__ import annotations import random @@ -21,8 +21,7 @@ class ThaiTransliterator: def __init__(self): - """ - Transliteration of Thai words. + """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ @@ -61,8 +60,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for PyTorch + """Prepare input sequence for PyTorch """ idxs = [] for ch in text: @@ -75,8 +73,7 @@ def _prepare_sequence_in(self, text: str): return tensor.to(device) def romanize(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ @@ -139,7 +136,9 @@ def forward(self, sequences, sequences_lengths): sequences = self.dropout(sequences) sequences_packed = nn.utils.rnn.pack_padded_sequence( - sequences, sequences_lengths.clone().to("cpu", torch.int64), batch_first=True + sequences, + sequences_lengths.clone().to("cpu", torch.int64), + batch_first=True, ) sequences_output, hidden = self.rnn(sequences_packed, hidden) @@ -235,7 +234,6 @@ def __init__( def forward(self, input_character, last_hidden, encoder_outputs, mask): """Defines the forward computation of the decoder""" - # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) # encoder_outputs: (batch_size, sequence_len, hidden_dim) diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index efdd6a4ae..19ebb0468 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") """ -Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") -""" + +from __future__ import annotations + import json import numpy as np @@ -19,8 +20,7 @@ class ThaiTransliterator_ONNX: def __init__(self): - """ - Transliteration of Thai words. + """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ @@ -58,8 +58,7 @@ def __init__(self): ) def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for ONNX + """Prepare input sequence for ONNX """ idxs = [] for ch in text: @@ -71,8 +70,7 @@ def _prepare_sequence_in(self, text: str): return np.array(idxs) def romanize(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index 3448b78cb..dea1f3306 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Grapheme-to-Phoneme (Thai G2P) +"""Thai Grapheme-to-Phoneme (Thai G2P) GitHub : https://github.com/wannaphong/thai-g2p """ +from __future__ import annotations + import random import numpy as np @@ -22,8 +22,7 @@ class ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self): @@ -62,8 +61,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for PyTorch. + """Prepare input sequence for PyTorch. """ idxs = [] for ch in text: @@ -76,8 +74,7 @@ def _prepare_sequence_in(self, text: str): return tensor.to(device) def g2p(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ @@ -124,7 +121,6 @@ def __init__( self.dropout = nn.Dropout(dropout) def forward(self, sequences, sequences_lengths): - # sequences: (batch_size, sequence_length=MAX_LENGTH) # sequences_lengths: (batch_size) @@ -198,9 +194,7 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze( - 2 - ) # (batch_size, sequence_len) + ).squeeze(2) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -244,7 +238,6 @@ def __init__( def forward(self, input_character, last_hidden, encoder_outputs, mask): """ "Defines the forward computation of the decoder""" - # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) # encoder_outputs: (batch_size, sequence_len, hidden_dim) @@ -297,7 +290,6 @@ def create_mask(self, source_seq): def forward( self, source_seq, source_seq_len, target_seq, teacher_forcing_ratio=0.5 ): - # source_seq: (batch_size, MAX_LENGTH) # source_seq_len: (batch_size, 1) # target_seq: (batch_size, MAX_LENGTH) diff --git a/pythainlp/transliterate/thaig2p_v2.py b/pythainlp/transliterate/thaig2p_v2.py index 4b0643cea..aad18c0d4 100644 --- a/pythainlp/transliterate/thaig2p_v2.py +++ b/pythainlp/transliterate/thaig2p_v2.py @@ -1,24 +1,27 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Grapheme-to-Phoneme (Thai G2P) +"""Thai Grapheme-to-Phoneme (Thai G2P) huggingface: https://huggingface.co/pythainlp/thaig2p-v2.0 """ # Use a pipeline as a high-level helper +from __future__ import annotations + from transformers import pipeline class ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self, device: str = "cpu"): - self.pipe = pipeline("text2text-generation", model="pythainlp/thaig2p-v2.0", device=device) + self.pipe = pipeline( + "text2text-generation", + model="pythainlp/thaig2p-v2.0", + device=device, + ) def g2p(self, text: str) -> str: return self.pipe(text)[0]["generated_text"] diff --git a/pythainlp/transliterate/tltk.py b/pythainlp/transliterate/tltk.py index 12da6c25c..6f84d8613 100644 --- a/pythainlp/transliterate/tltk.py +++ b/pythainlp/transliterate/tltk.py @@ -1,16 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + try: from tltk.nlp import g2p, th2ipa, th2roman except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) def romanize(text: str) -> str: - """ - Transliterating thai text to the Latin alphabet using tltk. + """Transliterating thai text to the Latin alphabet using tltk. :param str text: Thai text to be romanized :return: A string of Thai words rendered in the Latin alphabet. diff --git a/pythainlp/transliterate/umt5_thaig2p.py b/pythainlp/transliterate/umt5_thaig2p.py index 2b30d3b39..4976fe547 100644 --- a/pythainlp/transliterate/umt5_thaig2p.py +++ b/pythainlp/transliterate/umt5_thaig2p.py @@ -1,24 +1,27 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -umt5-thai-g2p-v2-0.5k +"""umt5-thai-g2p-v2-0.5k huggingface: https://huggingface.co/B-K/umt5-thai-g2p-v2-0.5k """ # Use a pipeline as a high-level helper +from __future__ import annotations + from transformers import pipeline class Umt5ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self, device: str = "cpu"): - self.pipe = pipeline("text2text-generation", model="B-K/umt5-thai-g2p-v2-0.5k", device=device) + self.pipe = pipeline( + "text2text-generation", + model="B-K/umt5-thai-g2p-v2-0.5k", + device=device, + ) def g2p(self, text: str) -> str: return self.pipe(text)[0]["generated_text"] diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 2c206bbe7..5f5a9edfa 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -1,25 +1,21 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Word-to-Phoneme (Thai W2P) +"""Thai Word-to-Phoneme (Thai W2P) GitHub : https://github.com/wannaphong/Thai_W2P """ -from typing import Union +from __future__ import annotations import numpy as np from pythainlp.corpus import download, get_corpus_path _GRAPHEMES = list( - "พจใงต้ืฮแาฐฒฤๅูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" - + " ณิฑชฉซทรฏฬํัฃวก่ป์ผฆบี๊ธญฌษะไ๋นโภ?" + "พจใงต้ืฮแาฐฒฤๅูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" + " ณิฑชฉซทรฏฬํัฃวก่ป์ผฆบี๊ธญฌษะไ๋นโภ?" ) _PHONEMES = list( - "-พจใงต้ืฮแาฐฒฤูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" - + " ณิฑชฉซทรํฬฏ–ัฃวก่ปผ์ฆบี๊ธฌญะไษ๋นโภ?" + "-พจใงต้ืฮแาฐฒฤูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" + " ณิฑชฉซทรํฬฏ–ัฃวก่ปผ์ฆบี๊ธฌญะไษ๋นโภ?" ) _MODEL_NAME = "thai_w2p" @@ -50,7 +46,7 @@ def _load_vocab(): return g2idx, idx2g, p2idx, idx2p -class Thai_W2P(): +class Thai_W2P: def __init__(self): super().__init__() self.graphemes = hp.graphemes @@ -133,7 +129,7 @@ def _encode(self, word: str) -> np.ndarray: return x - def _short_word(self, word: str) -> Union[str, None]: + def _short_word(self, word: str) -> str | None: self.word = word if self.word.endswith("."): self.word = self.word.replace(".", "") @@ -197,8 +193,7 @@ def __call__(self, word: str) -> str: def pronunciate(text: str) -> str: - """ - Convert a Thai word to its pronunciation in Thai letters. + """Convert a Thai word to its pronunciation in Thai letters. Input should be one single word. diff --git a/pythainlp/transliterate/wunsen.py b/pythainlp/transliterate/wunsen.py index 31d8b9eb8..ae503cd6d 100644 --- a/pythainlp/transliterate/wunsen.py +++ b/pythainlp/transliterate/wunsen.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text +"""Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text to Thai text By Wunsen @@ -11,12 +9,14 @@ * `GitHub \ `_ """ + +from __future__ import annotations + from wunsen import ThapSap class WunsenTransliterate: - """ - Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text + """Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text to Thai text by Wunsen @@ -36,20 +36,19 @@ def transliterate( self, text: str, lang: str, - jp_input: str = None, - zh_sandhi: bool = None, - system: str = None, + jp_input: str | None = None, + zh_sandhi: bool | None = None, + system: str | None = None, ): - """ - Use Wunsen for transliteration + """Use Wunsen for transliteration :param str text: text to be transliterated to Thai text. :param str lang: source language - :param str jp_input: Japanese input method (for Japanese only) - :param bool zh_sandhi: Mandarin third tone sandhi option - (for Mandarin only) - :param str system: transliteration system (for Japanese and - Mandarin only) + :param str | None jp_input: Japanese input method (for Japanese only). Default is None. + :param bool | None zh_sandhi: Mandarin third tone sandhi option + (for Mandarin only). Default is None. + :param str | None system: transliteration system (for Japanese and + Mandarin only). Default is None. :return: Thai text :rtype: str @@ -86,9 +85,7 @@ def transliterate( # output: 'โอฮาโย' wt.transliterate( - "ohayou", - lang="jp", - jp_input="Hepburn-no diacritic" + "ohayou", lang="jp", jp_input="Hepburn-no diacritic" ) # output: 'โอฮาโย' diff --git a/pythainlp/ulmfit/__init__.py b/pythainlp/ulmfit/__init__.py index a7128781b..ce9757727 100644 --- a/pythainlp/ulmfit/__init__.py +++ b/pythainlp/ulmfit/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Universal Language Model Fine-tuning for Text Classification (ULMFiT). +"""Universal Language Model Fine-tuning for Text Classification (ULMFiT). Code by Charin Polpanumas https://github.com/cstorm125/thai2fit/ diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 1494dcf1f..72d314b7a 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -1,18 +1,19 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Universal Language Model Fine-tuning for Text Classification (ULMFiT). """ -Universal Language Model Fine-tuning for Text Classification (ULMFiT). -""" + +from __future__ import annotations + import collections -from typing import Callable, Collection +from collections.abc import Callable, Collection import numpy as np import torch from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer from pythainlp.ulmfit.preprocess import ( fix_html, lowercase_all, @@ -67,11 +68,10 @@ def process_thai( text: str, pre_rules: Collection = pre_rules_th_sparse, - tok_func: Callable = THAI2FIT_TOKENIZER.word_tokenize, + tok_func: Callable | None = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: - """ - Process Thai texts for models (with sparse features as default) + """Process Thai texts for models (with sparse features as default) :param str text: text to be cleaned :param list[func] pre_rules: rules to apply before tokenization. @@ -132,6 +132,9 @@ def process_thai( """ res = text + if tok_func is None: + tok_func = thai2fit_tokenizer().word_tokenize + for rule in pre_rules: res = rule(res) res = tok_func(res) @@ -142,8 +145,7 @@ def process_thai( def document_vector(text: str, learn, data, agg: str = "mean"): - """ - This function vectorizes Thai input text into a 400 dimension vector using + """This function vectorizes Thai input text into a 400 dimension vector using :class:`fastai` language model and data bunch. :meth: `document_vector` get document vector using fastai language model @@ -182,8 +184,7 @@ def document_vector(text: str, learn, data, agg: str = "mean"): `_ """ - - s = THAI2FIT_TOKENIZER.word_tokenize(text) + s = thai2fit_tokenizer().word_tokenize(text) t = torch.tensor(data.vocab.numericalize(s), requires_grad=False).to( device ) @@ -200,8 +201,7 @@ def document_vector(text: str, learn, data, agg: str = "mean"): def merge_wgts(em_sz, wgts, itos_pre, itos_new): - """ - This function is to insert new vocab into an existing model named `wgts` + """This function is to insert new vocab into an existing model named `wgts` and update the model's weights for new vocab with the average embedding. :meth: `merge_wgts` insert pretrained weights and vocab into a new set @@ -219,7 +219,7 @@ def merge_wgts(em_sz, wgts, itos_pre, itos_new): from pythainlp.ulmfit import merge_wgts import torch - wgts = {'0.encoder.weight': torch.randn(5,3)} + wgts = {"0.encoder.weight": torch.randn(5, 3)} itos_pre = ["แมว", "คน", "หนู"] itos_new = ["ปลา", "เต่า", "นก"] em_sz = 3 diff --git a/pythainlp/ulmfit/preprocess.py b/pythainlp/ulmfit/preprocess.py index f9b89ef38..99f21c22c 100644 --- a/pythainlp/ulmfit/preprocess.py +++ b/pythainlp/ulmfit/preprocess.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Preprocessing for ULMFiT """ -Preprocessing for ULMFiT -""" + +from __future__ import annotations + import html import re -from typing import Collection, List +from collections.abc import Collection import emoji @@ -19,8 +20,7 @@ def replace_url(text: str) -> str: - """ - Replace URL in `text` with TK_URL + """Replace URL in `text` with TK_URL :param str text: text to replace URL in @@ -38,8 +38,7 @@ def replace_url(text: str) -> str: def fix_html(text: str) -> str: - """ - Replace HTML strings in `test`. (codes from `fastai`) + """Replace HTML strings in `test`. (codes from `fastai`) :param str text: text to replace HTML strings in @@ -83,8 +82,7 @@ def spec_add_spaces(text: str) -> str: def replace_rep_after(text: str) -> str: - """ - Replace repetitions at the character level in `text` after the repeated character. + """Replace repetitions at the character level in `text` after the repeated character. This is to prevent cases such as 'น้อยยยยยยยย' becomes 'น้อ xxrep 8 ย' ; instead it will retain the word as 'น้อย xxrep 8' @@ -105,16 +103,15 @@ def replace_rep_after(text: str) -> str: def _replace_rep(m): c, cc = m.groups() - return f"{c}{_TK_REP}{len(cc)+1} " + return f"{c}{_TK_REP}{len(cc) + 1} " re_rep = re.compile(r"(\S)(\1{3,})") return re_rep.sub(_replace_rep, text) -def replace_wrep_post(toks: Collection[str]) -> List[str]: - """ - Replace repetitive words after tokenization; +def replace_wrep_post(toks: Collection[str]) -> list[str]: + """Replace repetitive words after tokenization; fastai `replace_wrep` does not work well with Thai. :param list[str] toks: list of tokens @@ -148,13 +145,12 @@ def replace_wrep_post(toks: Collection[str]) -> List[str]: def rm_useless_newlines(text: str) -> str: - "Remove multiple newlines in `text`." - + """Remove multiple newlines in `text`.""" return re.sub(r"[\n]{2,}", " ", text) def rm_brackets(text: str) -> str: - "Remove all empty brackets and artifacts within brackets from `text`." + """Remove all empty brackets and artifacts within brackets from `text`.""" # remove empty brackets new_line = re.sub(r"\(\)", "", text) new_line = re.sub(r"\{\}", "", new_line) @@ -186,9 +182,8 @@ def rm_brackets(text: str) -> str: return new_line -def ungroup_emoji(toks: Collection[str]) -> List[str]: - """ - Ungroup Zero Width Joiner (ZVJ) Emojis +def ungroup_emoji(toks: Collection[str]) -> list[str]: + """Ungroup Zero Width Joiner (ZVJ) Emojis See https://emojipedia.org/emoji-zwj-sequence/ """ @@ -201,17 +196,15 @@ def ungroup_emoji(toks: Collection[str]) -> List[str]: return res -def lowercase_all(toks: Collection[str]) -> List[str]: - """ - Lowercase all English words; +def lowercase_all(toks: Collection[str]) -> list[str]: + """Lowercase all English words; English words in Thai texts don't usually have nuances of capitalization. """ return [tok.lower() for tok in toks] def replace_rep_nonum(text: str) -> str: - """ - Replace repetitions at the character level in `text` after the repetition. + """Replace repetitions at the character level in `text` after the repetition. This is done to prevent such case as 'น้อยยยยยยยย' becoming 'น้อ xxrep ย'; instead it will retain the word as 'น้อย xxrep ' @@ -239,9 +232,8 @@ def _replace_rep(m): return re_rep.sub(_replace_rep, text) -def replace_wrep_post_nonum(toks: Collection[str]) -> List[str]: - """ - Replace reptitive words post tokenization; +def replace_wrep_post_nonum(toks: Collection[str]) -> list[str]: + """Replace reptitive words post tokenization; fastai `replace_wrep` does not work well with Thai. :param list[str] toks: list of tokens @@ -274,9 +266,8 @@ def replace_wrep_post_nonum(toks: Collection[str]) -> List[str]: return res[1:] -def remove_space(toks: Collection[str]) -> List[str]: - """ - Do not include space for bag-of-word models. +def remove_space(toks: Collection[str]) -> list[str]: + """Do not include space for bag-of-word models. :param list[str] toks: list of tokens diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index a732891e7..2ce042773 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Tokenzier classes for ULMFiT """ -Tokenzier classes for ULMFiT -""" -from typing import Collection, List +from __future__ import annotations + +from collections.abc import Collection -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class BaseTokenizer: @@ -17,7 +17,7 @@ class BaseTokenizer: def __init__(self, lang: str): self.lang = lang - def tokenizer(self, t: str) -> List[str]: + def tokenizer(self, t: str) -> list[str]: return t.split(" ") def add_special_cases(self, toks: Collection[str]): @@ -25,8 +25,7 @@ def add_special_cases(self, toks: Collection[str]): class ThaiTokenizer(BaseTokenizer): - """ - Wrapper around a frozen newmm tokenizer to make it a + """Wrapper around a frozen newmm tokenizer to make it a :class:`fastai.BaseTokenizer`. (see: https://docs.fast.ai/text.transform#BaseTokenizer) """ @@ -35,9 +34,8 @@ def __init__(self, lang: str = "th"): self.lang = lang @staticmethod - def tokenizer(text: str) -> List[str]: - """ - This function tokenizes text using *newmm* engine and the dictionary + def tokenizer(text: str) -> list[str]: + """This function tokenizes text using *newmm* engine and the dictionary specifically for `ulmfit` related functions (see: `Dictionary file (.txt) \ `_). @@ -65,7 +63,7 @@ def tokenizer(text: str) -> List[str]: ' ', 'ภาวนามยปัญญา'] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + return thai2fit_tokenizer().word_tokenize(text) def add_special_cases(self, toks): pass diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index dca216969..11454c160 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Utility functions, like date conversion and digit conversion +"""Utility functions, like date conversion and digit conversion """ __all__ = [ @@ -102,6 +100,7 @@ from pythainlp.util.keywords import find_keyword, rank from pythainlp.util.lcs import longest_common_subsequence from pythainlp.util.normalize import ( + expand_maiyamok, maiyamok, normalize, remove_dangling, @@ -110,7 +109,6 @@ remove_tonemark, remove_zw, reorder_vowels, - expand_maiyamok, ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa @@ -123,14 +121,15 @@ remove_trailing_repeat_consonants, ) from pythainlp.util.strftime import thai_strftime + from pythainlp.util.thai import ( + analyze_thai_text, count_thai_chars, countthai, display_thai_char, isthai, isthaichar, thai_word_tone_detector, - analyze_thai_text, ) from pythainlp.util.thai_lunar_date import th_zodiac, to_lunar_date from pythainlp.util.thaiwordcheck import is_native_thai @@ -138,6 +137,7 @@ from pythainlp.util.trie import Trie, dict_trie from pythainlp.util.wordtonum import text_to_num, thaiword_to_num, words_to_num +# DO NOT REORDER these imports. # sound_syllable and pronounce have to be imported last, # to prevent circular import issues. # Other imports should be above this line, sorted. @@ -150,6 +150,6 @@ from pythainlp.util.pronounce import ( rhyme, spelling, - tone_to_spelling, thai_consonant_to_spelling, + tone_to_spelling, ) diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index a46b876c7..f52cffc0e 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Thai abbreviation tools """ -Thai abbreviation tools -""" -from typing import List, Tuple, Union +from __future__ import annotations -def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[float, None]]]: - """ - This function converts Thai text (with abbreviation) to full text. + +def abbreviation_to_full_text( + text: str, top_k: int = 2 +) -> list[tuple[str, float | None]]: + """This function converts Thai text (with abbreviation) to full text. This function uses KhamYo for handles abbreviations. See more `KhamYo `_. @@ -29,7 +29,7 @@ def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[ abbreviation_to_full_text(text) # output: [ - # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), + # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), # ('โรงแรมของเราน่าอยู่', tensor(0.2438)) # ] """ @@ -39,7 +39,7 @@ def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[ raise ImportError( """ This function needs to use khamyo. - You can install by pip install khamyo or + You can install by pip install khamyo or pip install pythainlp[abbreviation]. """ ) diff --git a/pythainlp/util/collate.py b/pythainlp/util/collate.py index 0c28426cc..6f6feb5e9 100644 --- a/pythainlp/util/collate.py +++ b/pythainlp/util/collate.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai collation (sorted according to Thai dictionary order) +"""Thai collation (sorted according to Thai dictionary order) Simple implementation using regular expressions """ + +from __future__ import annotations + import re -from typing import Iterable, List +from collections.abc import Iterable _RE_TONE = re.compile(r"[็-์]") _RE_LV_C = re.compile(r"([เ-ไ])([ก-ฮ])") @@ -22,9 +23,8 @@ def _thkey(word: str) -> str: return cv + tone -def collate(data: Iterable, reverse: bool = False) -> List[str]: - """ - This function sorts strings (almost) according to Thai dictionary. +def collate(data: Iterable, reverse: bool = False) -> list[str]: + """This function sorts strings (almost) according to Thai dictionary. Important notes: this implementation ignores tone marks and symbols diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index 6e101b607..70ce91508 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai date/time conversion. +"""Thai date/time conversion. Note: It does not take into account the change of new year's day in Thailand """ @@ -12,6 +10,7 @@ # AD คือ ค.ศ. # AH ปีฮิจเราะห์ศักราชเป็นปีพุทธศักราช จะต้องบวกด้วย 1122 # ไม่ได้รองรับปี พ.ศ. ก่อนการเปลี่ยนวันขึ้นปีใหม่ของประเทศไทย +from __future__ import annotations __all__ = [ "convert_years", @@ -25,7 +24,6 @@ import re from datetime import datetime, timedelta -from typing import Union try: from zoneinfo import ZoneInfo @@ -84,17 +82,20 @@ ["กันยายน", "กันยา", "ก.ย.", "09", "9"], ["ตุลาคม", "ตุลา", "ต.ค.", "10"], ["พฤศจิกายน", "พฤศจิกา", "พ.ย.", "11"], - ["ธันวาคม", "ธันวา", "ธ.ค.", "12"] + ["ธันวาคม", "ธันวา", "ธ.ค.", "12"], ] -thai_full_month_lists_regex = "(" + '|'.join( - ['|'.join(i) for i in thai_full_month_lists] -) + ")" +thai_full_month_lists_regex = ( + "(" + "|".join(["|".join(i) for i in thai_full_month_lists]) + ")" +) year_all_regex = r"(\d\d\d\d|\d\d)" -dates_list = "(" + '|'.join( - [str(i) for i in range(32, 0, -1)] + [ - "0" + str(i) for i in range(1, 10) - ] -) + ")" +dates_list = ( + "(" + + "|".join( + [str(i) for i in range(32, 0, -1)] + + ["0" + str(i) for i in range(1, 10)] + ) + + ")" +) _DAY = { "วันนี้": 0, @@ -120,8 +121,7 @@ def convert_years(year: str, src="be", target="ad") -> str: - """ - Convert years + """Convert years :param int year: Year :param str src: The source year @@ -145,7 +145,7 @@ def convert_years(year: str, src="be", target="ad") -> str: # พ.ศ. - 543  = ค.ศ. if target == "ad": output_year = str(int(year) - 543) - # พ.ศ. - 2324 = ร.ศ.  + # พ.ศ. - 2324 = ร.ศ. elif target == "re": output_year = str(int(year) - 2324) # พ.ศ. - 1122 = ฮ.ศ. @@ -199,17 +199,16 @@ def thai_strptime( text: str, fmt: str, year: str = "be", - add_year: int = None, - tzinfo=ZoneInfo("Asia/Bangkok") + add_year: int | None = None, + tzinfo=ZoneInfo("Asia/Bangkok"), ): - """ - Thai strptime + """Thai strptime :param str text: text :param str fmt: string containing date and time directives :param str year: year of the text \ (ad is Anno Domini and be is Buddhist Era) - :param int add_year: add to year when converting to ad + :param int | None add_year: add to year when converting to ad. Default is None. :param object tzinfo: tzinfo (default is Asia/Bangkok) :return: The year that is converted to datetime.datetime :rtype: datetime.datetime @@ -264,27 +263,28 @@ def thai_strptime( if "%f" in fmt: fmt = fmt.replace("%f", r"(\d+)") keys = [ - i.strip().strip('-').strip(':').strip('.') - for i in _old.split("%") if i != '' + i.strip().strip("-").strip(":").strip(".") + for i in _old.split("%") + if i != "" ] y = re.findall(fmt, text) - data = {i: ''.join(list(j)) for i, j in zip(keys, y[0])} + data = {i: "".join(list(j)) for i, j in zip(keys, y[0])} H = 0 M = 0 S = 0 f = 0 - d = data['d'] - m = _find_month(data['B']) - y = data['Y'] + d = data["d"] + m = _find_month(data["B"]) + y = data["Y"] if "H" in keys: - H = data['H'] + H = data["H"] if "M" in keys: - M = data['M'] + M = data["M"] if "S" in keys: - S = data['S'] + S = data["S"] if "f" in keys: - f = data['f'] + f = data["f"] if int(y) < 100 and year == "be": if add_year is None: y = str(2500 + int(y)) @@ -305,13 +305,12 @@ def thai_strptime( minute=int(M), second=int(S), microsecond=int(f), - tzinfo=tzinfo + tzinfo=tzinfo, ) def now_reign_year() -> int: - """ - Return the reign year of the 10th King of Chakri dynasty. + """Return the reign year of the 10th King of Chakri dynasty. :return: reign year of the 10th King of Chakri dynasty. :rtype: int @@ -332,8 +331,7 @@ def now_reign_year() -> int: def reign_year_to_ad(reign_year: int, reign: int) -> int: - """ - Convert reign year to AD. + """Convert reign year to AD. Return AD year according to the reign year for the 7th to 10th King of Chakri dynasty, Thailand. @@ -369,11 +367,8 @@ def reign_year_to_ad(reign_year: int, reign: int) -> int: return ad -def thaiword_to_date( - text: str, date: datetime = None -) -> Union[datetime, None]: - """ - Convert Thai relative date to :class:`datetime.datetime`. +def thaiword_to_date(text: str, date: datetime = None) -> datetime | None: + """Convert Thai relative date to :class:`datetime.datetime`. :param str text: Thai text containing relative date :param datetime.datetime date: date (default is datetime.datetime.now()) diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index 9fa2294c4..370968972 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Convert digits """ -Convert digits -""" + +from __future__ import annotations _arabic_thai = { "0": "๐", @@ -64,8 +64,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: - """ - This function converts Thai digits (i.e. ๑, ๓, ๑๐) to Arabic digits + """This function converts Thai digits (i.e. ๑, ๓, ๑๐) to Arabic digits (i.e. 1, 3, 10). :param str text: Text with Thai digits such as '๑', '๒', '๓' @@ -78,7 +77,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: from pythainlp.util import thai_digit_to_arabic_digit - text = 'เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท' + text = "เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท" thai_digit_to_arabic_digit(text) # output: เป็นจำนวน 123,400.25 บาท @@ -90,8 +89,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: def arabic_digit_to_thai_digit(text: str) -> str: - """ - This function converts Arabic digits (i.e. 1, 3, 10) to Thai digits + """This function converts Arabic digits (i.e. 1, 3, 10) to Thai digits (i.e. ๑, ๓, ๑๐). :param str text: Text with Arabic digits such as '1', '2', '3' @@ -104,7 +102,7 @@ def arabic_digit_to_thai_digit(text: str) -> str: from pythainlp.util import arabic_digit_to_thai_digit - text = 'เป็นจำนวน 123,400.25 บาท' + text = "เป็นจำนวน 123,400.25 บาท" arabic_digit_to_thai_digit(text) # output: เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท @@ -117,8 +115,7 @@ def arabic_digit_to_thai_digit(text: str) -> str: def digit_to_text(text: str) -> str: - """ - :param str text: Text with digits such as '1', '2', '๓', '๔' + """:param str text: Text with digits such as '1', '2', '๓', '๔' :return: Text with digits spelled out in Thai """ if not text or not isinstance(text, str): @@ -132,8 +129,7 @@ def digit_to_text(text: str) -> str: def text_to_arabic_digit(text: str) -> str: - """ - This function converts spelled out digits in Thai to Arabic digits. + """This function converts spelled out digits in Thai to Arabic digits. :param text: A digit spelled out in Thai :return: An Arabic digit such as '1', '2', '3' if the text is @@ -170,8 +166,7 @@ def text_to_arabic_digit(text: str) -> str: def text_to_thai_digit(text: str) -> str: - """ - This function converts spelled out digits in Thai to Thai digits. + """This function converts spelled out digits in Thai to Thai digits. :param text: A digit spelled out in Thai :return: A Thai digit such as '๑', '๒', '๓' if the text is digit diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index 7be117158..985a965da 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Convert emojis """ -Convert emojis -""" + +from __future__ import annotations import re @@ -1833,8 +1834,7 @@ def emoji_to_thai(text: str, delimiters=(_delimiter, _delimiter)) -> str: - """ - This function converts emojis to their Thai meanings + """This function converts emojis to their Thai meanings :param str text: Text with emojis :return: Text with emojis converted to their Thai meanings @@ -1855,7 +1855,6 @@ def emoji_to_thai(text: str, delimiters=(_delimiter, _delimiter)) -> str: emoji_to_thai("🇹🇭 นี่คือธงประเทศไทย") # output: :ธง_ไทย: นี่คือธงประเทศไทย """ - return _emoji_regex.sub( lambda match: delimiters[0] + _emoji_th[match.group(0)] diff --git a/pythainlp/util/encoding.py b/pythainlp/util/encoding.py index a741fc99e..0727c3f74 100644 --- a/pythainlp/util/encoding.py +++ b/pythainlp/util/encoding.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -def tis620_to_utf8(text: str)->str: - """ - Convert TIS-620 to UTF-8 +from __future__ import annotations + + +def tis620_to_utf8(text: str) -> str: + """Convert TIS-620 to UTF-8 :param str text: TIS-620 encoded text :return: UTF-8 encoded text @@ -22,8 +24,7 @@ def tis620_to_utf8(text: str)->str: def to_idna(text: str) -> str: - """ - Encode text with IDNA, as used in Internationalized Domain Name (IDN). + """Encode text with IDNA, as used in Internationalized Domain Name (IDN). :param str text: Thai text :return: IDNA-encoded text diff --git a/pythainlp/util/keyboard.py b/pythainlp/util/keyboard.py index f4e824b98..6ea667070 100644 --- a/pythainlp/util/keyboard.py +++ b/pythainlp/util/keyboard.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Functions related to keyboard layout. """ -Functions related to keyboard layout. -""" + +from __future__ import annotations EN_TH_KEYB_PAIRS = { "Z": "(", @@ -121,8 +121,7 @@ def eng_to_thai(text: str) -> str: - """ - Corrects the given text that was incorrectly typed using English-US + """Corrects the given text that was incorrectly typed using English-US Qwerty keyboard layout to the originally intended keyboard layout that is the Thai Kedmanee keyboard. @@ -144,8 +143,7 @@ def eng_to_thai(text: str) -> str: def thai_to_eng(text: str) -> str: - """ - Corrects the given text that was incorrectly typed using Thai Kedmanee + """Corrects the given text that was incorrectly typed using Thai Kedmanee keyboard layout to the originally intended keyboard layout that is the English-US Qwerty keyboard. @@ -167,8 +165,7 @@ def thai_to_eng(text: str) -> str: def thai_keyboard_dist(c1: str, c2: str, shift_dist: float = 0.0) -> float: - """ - Calculate Euclidean distance between two Thai characters + """Calculate Euclidean distance between two Thai characters according to their location on a Thai keyboard layout. A modified TIS 820-2531 standard keyboard layout, which is developed diff --git a/pythainlp/util/keywords.py b/pythainlp/util/keywords.py index 13da2db1b..fb9b10a33 100644 --- a/pythainlp/util/keywords.py +++ b/pythainlp/util/keywords.py @@ -1,18 +1,17 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from collections import Counter -from typing import Dict, List from pythainlp.corpus import thai_stopwords _STOPWORDS = thai_stopwords() -def rank(words: List[str], exclude_stopwords: bool = False) -> Counter: - """ - Count word frequencies given a list of Thai words with an option +def rank(words: list[str], exclude_stopwords: bool = False) -> Counter: + """Count word frequencies given a list of Thai words with an option to exclude stopwords. :param list words: a list of words @@ -72,9 +71,8 @@ def rank(words: List[str], exclude_stopwords: bool = False) -> Counter: return Counter(words) -def find_keyword(word_list: List[str], min_len: int = 3) -> Dict[str, int]: - """ - This function counts the frequencies of words in the list +def find_keyword(word_list: list[str], min_len: int = 3) -> dict[str, int]: + """This function counts the frequencies of words in the list where stopword is excluded and returns a frequency dictionary. :param list word_list: a list of words diff --git a/pythainlp/util/lcs.py b/pythainlp/util/lcs.py index 6dac741f1..e127b68a2 100644 --- a/pythainlp/util/lcs.py +++ b/pythainlp/util/lcs.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + def longest_common_subsequence(str1: str, str2: str) -> str: - """ - Find the longest common subsequence between two strings. + """Find the longest common subsequence between two strings. :param str str1: The first string. :param str str2: The second string. @@ -48,7 +48,6 @@ def longest_common_subsequence(str1: str, str2: str) -> str: i = m j = n while i > 0 and j > 0: - # If current character in str1 and str2 are same, then # current character is part of LCS if str1[i - 1] == str2[j - 1]: diff --git a/pythainlp/util/morse.py b/pythainlp/util/morse.py index 6717e22d8..73b8e1cd0 100644 --- a/pythainlp/util/morse.py +++ b/pythainlp/util/morse.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations THAI_MORSE_CODE = { "ก": "--.", @@ -134,8 +134,7 @@ def morse_encode(text: str, lang: str = "th") -> str: - """ - Convert text to Morse code (support Thai and English) + """Convert text to Morse code (support Thai and English) :param str text: Text :param str lang: Language Code (*th* is Thai and *en* is English) @@ -146,6 +145,7 @@ def morse_encode(text: str, lang: str = "th") -> str: :: from pythainlp.util.morse import morse_encode + print(morse_encode("แมว", lang="th")) # output: .-.- -- .-- @@ -165,8 +165,7 @@ def morse_encode(text: str, lang: str = "th") -> str: def morse_decode(morse_text: str, lang: str = "th") -> str: - """ - Simple Convert Morse code to text + """Simple Convert Morse code to text Thai still have some wrong character problem that\ can fix by spell corrector. diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index 975d8188b..6251ee495 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Text normalization """ -Text normalization -""" + +from __future__ import annotations import re -from typing import List, Union from pythainlp import thai_above_vowels as above_v from pythainlp import thai_below_vowels as below_v @@ -57,8 +56,7 @@ def _last_char(matchobj): # to be used with _RE_NOREPEAT_TONEMARKS def remove_dangling(text: str) -> str: - """ - Remove Thai non-base characters at the beginning of text. + """Remove Thai non-base characters at the beginning of text. This is a common "typo", especially for input field in a form, as these non-base characters can be visually hidden from user @@ -85,8 +83,7 @@ def remove_dangling(text: str) -> str: def remove_dup_spaces(text: str) -> str: - """ - Remove duplicate spaces. Replace multiple spaces with one space. + """Remove duplicate spaces. Replace multiple spaces with one space. Multiple newline characters and empty lines will be replaced with one newline character. @@ -111,8 +108,7 @@ def remove_dup_spaces(text: str) -> str: def remove_tonemark(text: str) -> str: - """ - Remove all Thai tone marks from the text. + """Remove all Thai tone marks from the text. Thai script has four tone marks indicating four tones as follows: @@ -144,8 +140,7 @@ def remove_tonemark(text: str) -> str: def remove_zw(text: str) -> str: - """ - Remove zero-width characters. + """Remove zero-width characters. These non-visible characters may cause unexpected result from the user's point of view. Removing them can make string matching more robust. @@ -167,8 +162,7 @@ def remove_zw(text: str) -> str: def reorder_vowels(text: str) -> str: - """ - Reorder vowels and tone marks to the standard logical order/spelling. + """Reorder vowels and tone marks to the standard logical order/spelling. Characters in input text will be reordered/transformed, according to these rules: @@ -189,8 +183,7 @@ def reorder_vowels(text: str) -> str: def remove_repeat_vowels(text: str) -> str: - """ - Remove repeating vowels, tone marks, and signs. + """Remove repeating vowels, tone marks, and signs. This function will call reorder_vowels() first, to make sure that double Sara E will be converted to Sara Ae and not be removed. @@ -210,8 +203,7 @@ def remove_repeat_vowels(text: str) -> str: def normalize(text: str) -> str: - """ - Normalize and clean Thai text with normalizing rules as follows: + """Normalize and clean Thai text with normalizing rules as follows: * Remove zero-width spaces * Remove duplicate spaces @@ -251,9 +243,8 @@ def normalize(text: str) -> str: return text -def expand_maiyamok(sent: Union[str, List[str]]) -> List[str]: - """ - Expand Maiyamok. +def expand_maiyamok(sent: str | list[str]) -> list[str]: + """Expand Maiyamok. Maiyamok (ๆ) (Unicode U+0E46) is a Thai character indicating word repetition. This function preprocesses Thai text by replacing @@ -311,9 +302,8 @@ def expand_maiyamok(sent: Union[str, List[str]]) -> List[str]: return output_toks[::-1] -def maiyamok(sent: Union[str, List[str]]) -> List[str]: - """ - Expand Maiyamok. +def maiyamok(sent: str | list[str]) -> list[str]: + """Expand Maiyamok. Deprecated. Use expand_maiyamok() instead. diff --git a/pythainlp/util/numtoword.py b/pythainlp/util/numtoword.py index 7d1f43ad5..6c7c2b6fa 100644 --- a/pythainlp/util/numtoword.py +++ b/pythainlp/util/numtoword.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert number value to Thai read out +"""Convert number value to Thai read out Adapted from http://justmindthought.blogspot.com/2012/12/code-php.html https://suksit.com/post/writing-bahttext-in-php/ """ +from __future__ import annotations + __all__ = ["bahttext", "num_to_thaiword"] _VALUES = [ @@ -29,8 +29,7 @@ def bahttext(number: float) -> str: - """ - This function converts a number to Thai text and adds + """This function converts a number to Thai text and adds a suffix "บาท" (Baht). The precision will be fixed at two decimal places (0.00) to fits "สตางค์" (Satang) unit. @@ -61,7 +60,7 @@ def bahttext(number: float) -> str: elif number == 0: ret = "ศูนย์บาทถ้วน" else: - num_int, num_dec = "{:.2f}".format(number).split(".") + num_int, num_dec = f"{number:.2f}".split(".") num_int = int(num_int) num_dec = int(num_dec) @@ -79,8 +78,7 @@ def bahttext(number: float) -> str: def num_to_thaiword(number: int) -> str: - """ - This function converts number to Thai text + """This function converts number to Thai text :param int number: an integer number to be converted to Thai text :return: text representing the number in Thai @@ -97,7 +95,6 @@ def num_to_thaiword(number: int) -> str: num_to_thaiword(11) # output: สิบเอ็ด """ - output = "" number_temp = number if number is None: diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index c473b2cd2..2fbc57bbe 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -1,11 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Phonemes util """ -Phonemes util -""" + +from __future__ import annotations + import unicodedata +from functools import lru_cache from pythainlp.tokenize import Tokenizer from pythainlp.util.trie import Trie @@ -88,8 +90,7 @@ def nectec_to_ipa(pronunciation: str) -> str: - """ - Convert NECTEC system to IPA system + """Convert NECTEC system to IPA system :param str pronunciation: NECTEC phoneme :return: IPA that is converted @@ -106,10 +107,10 @@ def nectec_to_ipa(pronunciation: str) -> str: References ---------- - Pornpimon Palingoon, Sumonmas Thatphithakkul. Chapter 4 Speech processing \ and Speech corpus. In: Handbook of Thai Electronic Corpus. \ 1st ed. p. 122–56. + """ parts = pronunciation.split("-") ipa = [] @@ -192,13 +193,17 @@ def nectec_to_ipa(pronunciation: str) -> str: } dict_ipa_rtgs_final = {"w": "o"} -trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) -ipa_cut = Tokenizer(custom_dict=trie, engine="newmm") + + +@lru_cache +def _ipa_cut(): + """Lazy load IPA tokenizer with cache""" + trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) + return Tokenizer(custom_dict=trie, engine="newmm") def ipa_to_rtgs(ipa: str) -> str: - """ - Convert IPA system to The Royal Thai General System of Transcription (RTGS) + """Convert IPA system to The Royal Thai General System of Transcription (RTGS) Docs: https://en.wikipedia.org/wiki/Help:IPA/Thai @@ -216,6 +221,7 @@ def ipa_to_rtgs(ipa: str) -> str: """ rtgs_parts = [] + ipa_cut = _ipa_cut() ipa_parts = ipa_cut.word_tokenize(ipa) for i, ipa_part in enumerate(ipa_parts): @@ -237,8 +243,7 @@ def ipa_to_rtgs(ipa: str) -> str: def remove_tone_ipa(ipa: str) -> str: - """ - Remove Thai Tones from IPA system + """Remove Thai Tones from IPA system :param str ipa: IPA phoneme :return: IPA phoneme with tones removed diff --git a/pythainlp/util/pronounce.py b/pythainlp/util/pronounce.py index 4c69d67f4..41f14914c 100644 --- a/pythainlp/util/pronounce.py +++ b/pythainlp/util/pronounce.py @@ -1,24 +1,22 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations + import re +from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_words from pythainlp.khavee import KhaveeVerifier -from pythainlp.tokenize import syllable_tokenize -from pythainlp.tokenize import Tokenizer -from pythainlp import thai_consonants, thai_tonemarks +from pythainlp.tokenize import Tokenizer, syllable_tokenize from pythainlp.util import remove_tonemark kv = KhaveeVerifier() all_thai_words_dict = None -def rhyme(word: str) -> List[str]: - """ - Find Thai rhyme +def rhyme(word: str) -> list[str]: + """Find Thai rhyme :param str word: A Thai word :return: All list Thai rhyme words @@ -44,10 +42,12 @@ def rhyme(word: str) -> List[str]: return sorted(list_sumpus) -thai_vowel = ''.join(( - "อะ,อา,อิ,อี,อึ,อื,อุ,อู,เอะ,เอ,แอะ,แอ,เอียะ,เอีย,เอือะ,เอือ,อัวะ,อัว,โอะ,", - "โอ,เอาะ,ออ,เออะ,เออ,อำ,ใอ,ไอ,เอา,ฤ,ฤๅ,ฦ,ฦๅ" -)).split(",") +thai_vowel = "".join( + ( + "อะ,อา,อิ,อี,อึ,อื,อุ,อู,เอะ,เอ,แอะ,แอ,เอียะ,เอีย,เอือะ,เอือ,อัวะ,อัว,โอะ,", + "โอ,เอาะ,ออ,เออะ,เออ,อำ,ใอ,ไอ,เอา,ฤ,ฤๅ,ฦ,ฦๅ", + ) +).split(",") thai_vowel_all = [ ("([ก-ฮ])ะ", "\\1อะ"), ("([ก-ฮ])า", "\\1อา"), @@ -83,8 +83,7 @@ def rhyme(word: str) -> List[str]: def thai_consonant_to_spelling(c: str) -> str: - """ - Thai consonants to spelling + """Thai consonants to spelling :param str c: A Thai consonant :return: spelling @@ -104,8 +103,7 @@ def thai_consonant_to_spelling(c: str) -> str: def tone_to_spelling(t: str) -> str: - """ - Thai tonemarks to spelling + """Thai tonemarks to spelling :param str t: A Thai tonemarks :return: spelling @@ -116,7 +114,7 @@ def tone_to_spelling(t: str) -> str: from pythainlp.util import tone_to_spelling - print(tone_to_spelling("่")) # ไม้เอก + print(tone_to_spelling("่")) # ไม้เอก # output: ไม้เอก """ if t == "่": @@ -130,9 +128,8 @@ def tone_to_spelling(t: str) -> str: return t -def spelling(word: str) -> List[str]: - """ - Thai word to spelling +def spelling(word: str) -> list[str]: + """Thai word to spelling This funnction support Thai root word only. @@ -154,8 +151,7 @@ def spelling(word: str) -> List[str]: if not word or not isinstance(word, str): return [] thai_vowel_tokenizer = Tokenizer( - custom_dict=thai_vowel + list(thai_consonants), - engine="longest" + custom_dict=thai_vowel + list(thai_consonants), engine="longest" ) word_pre = remove_tonemark(word).replace("็", "") tone = [tone_to_spelling(i) for i in word if i in thai_tonemarks] @@ -169,8 +165,9 @@ def spelling(word: str) -> List[str]: break list_word_output = thai_vowel_tokenizer.word_tokenize(word_output) output = [ - i for i in [thai_consonant_to_spelling(i) for i in list_word_output] - if '์' not in i + i + for i in [thai_consonant_to_spelling(i) for i in list_word_output] + if "์" not in i ] if word_pre == word: return output + [word] diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index 62e55aa2c..ad6349435 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Removement of repeated consonants at the end of words """ -Removement of repeated consonants at the end of words -""" -from typing import Iterable, List, Tuple + +from __future__ import annotations + +from collections.abc import Iterable from pythainlp import thai_consonants as consonants from pythainlp.corpus import thai_words -from pythainlp.util.trie import Trie # used by remove_trailing_repeat_consonants() # contains all words that has repeating consonants at the end @@ -25,8 +25,7 @@ def remove_trailing_repeat_consonants( custom_dict: Iterable[str] = [], has_dictionary_updated: bool = True, ) -> str: - """ - Remove repeating consonants at the last of the sentence. + """Remove repeating consonants at the last of the sentence. This function will remove the repeating consonants before a whitespace, new line or at the last @@ -41,7 +40,7 @@ def remove_trailing_repeat_consonants( :param str text: input text :param Trie dictionary: Trie dictionary to check the last word. If None, pythainlp.corpus.thai_words() will be used - :param bool has_dictionary_updated: If the dictionary is updated + :param bool has_dictionary_updated: If the dictionary is updated or the first time using in the kernel, set this true. If not, set this false to save time. :return: text without repeating Thai consonants @@ -101,8 +100,7 @@ def remove_trailing_repeat_consonants( def _remove_repeat_trailing_consonants_from_segment(segment: str) -> str: - """ - Remove repeating consonants at the last of the segment. + """Remove repeating consonants at the last of the segment. This function process only at the last of the given text. Details is same as remove_repeat_consonants(). @@ -153,8 +151,7 @@ def _remove_repeat_trailing_consonants_from_segment(segment: str) -> str: def _remove_all_last_consonants(text: str, dup: str) -> str: - """ - Reduce repeating characters at the end of the text. + """Reduce repeating characters at the end of the text. This function will remove the repeating characters at the last. The text just before the repeating characters will be returned. @@ -172,8 +169,7 @@ def _remove_all_last_consonants(text: str, dup: str) -> str: def _update_consonant_repeaters(custom_dict: Iterable[str]) -> None: - """ - Update dictionary of all words that has + """Update dictionary of all words that has repeating consonants at the end from the dictionary. Search all words in the dictionary that has more than 1 consonants @@ -196,8 +192,7 @@ def _update_consonant_repeaters(custom_dict: Iterable[str]) -> None: def _is_last_consonant_repeater(word: str) -> bool: - """ - Check if the word has repeating consonants at the end. + """Check if the word has repeating consonants at the end. This function checks if the word has more than 1 repeating consonants at the end. @@ -212,10 +207,9 @@ def _is_last_consonant_repeater(word: str) -> bool: def _find_longest_consonant_repeaters_match( - segment_head: str, repeaters: List[str] -) -> Tuple[str, int]: - """ - Find the longest word that matches the segment. + segment_head: str, repeaters: list[str] +) -> tuple[str, int]: + """Find the longest word that matches the segment. Find the longest word that matches the last of the segment from the given repeaters list. diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index 24b005e56..c1850829f 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -1,9 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp import ( thai_above_vowels, @@ -48,7 +49,13 @@ for i in thai_below_vowels: dict_vowel[i] = "อ" + i -_cut = Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") + +@lru_cache +def _cut(): + """Lazy load vowel tokenizer with cache""" + return Tokenizer( + list(dict_vowel.keys()) + list(thai_consonants), engine="mm" + ) def _clean(w): @@ -77,9 +84,8 @@ def _clean(w): return w -def spell_syllable(text: str) -> List[str]: - """ - Spell out syllables in Thai word distribution form. +def spell_syllable(text: str) -> list[str]: + """Spell out syllables in Thai word distribution form. :param str s: Thai syllables only :return: List of spelled out syllables @@ -93,7 +99,7 @@ def spell_syllable(text: str) -> List[str]: print(spell_syllable("แมว")) # output: ['มอ', 'วอ', 'แอ', 'แมว'] """ - tokens = _cut.word_tokenize(_clean(text)) + tokens = _cut().word_tokenize(_clean(text)) c_only = [tok + "อ" for tok in tokens if tok in set(thai_consonants)] v_only = [dict_vowel[tok] for tok in tokens if tok in set(dict_vowel)] @@ -102,9 +108,8 @@ def spell_syllable(text: str) -> List[str]: return c_only + v_only + t_only + [text] -def spell_word(text: str) -> List[str]: - """ - Spell out words in Thai word distribution form. +def spell_word(text: str) -> list[str]: + """Spell out words in Thai word distribution form. :param str w: Thai words only :return: List of spelled out words diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index e96299b44..21a18af24 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Thai date/time formatting. """ -Thai date/time formatting. -""" + +from __future__ import annotations import warnings from datetime import datetime @@ -30,13 +30,12 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: - """ - Standard datetime.strftime() with normalization and exception handling. + """Standard datetime.strftime() with normalization and exception handling. """ str_ = "" try: str_ = dt_obj.strftime(f"%{fmt_char}") - if not str_ or str_ == "%{}".format(fmt_char): + if not str_ or str_ == f"%{fmt_char}": # Normalize outputs for unsupported directives # in different platforms: # "%Q" may result "", "%Q", or "Q", make it all "Q" @@ -57,8 +56,7 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: - """ - Conversion support for thai_strftime(). + """Conversion support for thai_strftime(). The fmt_char should be in _NEED_L10N when calling this function. """ @@ -114,21 +112,13 @@ def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: ).zfill(2) elif fmt_char == "v": # BSD extension, ' 6-Oct-1976' - str_ = "{:>2}-{}-{}".format( - dt_obj.day, - thai_abbr_months[dt_obj.month - 1], - str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4), - ) + str_ = f"{dt_obj.day:>2}-{thai_abbr_months[dt_obj.month - 1]}-{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" elif fmt_char == "X": # Locale’s appropriate time representation. str_ = dt_obj.strftime("%H:%M:%S") elif fmt_char == "x": # Locale’s appropriate date representation. - str_ = "{}/{}/{}".format( - str(dt_obj.day).zfill(2), - str(dt_obj.month).zfill(2), - str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4), - ) + str_ = f"{str(dt_obj.day).zfill(2)}/{str(dt_obj.month).zfill(2)}/{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" elif fmt_char == "Y": # Year with century str_ = (str(dt_obj.year + _BE_AD_DIFFERENCE)).zfill(4) @@ -159,8 +149,7 @@ def thai_strftime( fmt: str = "%-d %b %y", thaidigit: bool = False, ) -> str: - """ - Convert :class:`datetime.datetime` into Thai date and time format. + """Convert :class:`datetime.datetime` into Thai date and time format. The formatting directives are similar to :func:`datatime.strrftime`. diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index acb78e769..e193e1a87 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Syllable tools """ -Syllable tools -""" + +from __future__ import annotations import re @@ -61,8 +61,7 @@ def sound_syllable(syllable: str) -> str: - """ - Sound syllable classification + """Sound syllable classification This function is sound syllable classification. The syllable is a live syllable or dead syllable. @@ -157,8 +156,7 @@ def sound_syllable(syllable: str) -> str: def syllable_open_close_detector(syllable: str) -> str: - """ - Open/close Thai syllables detector + """Open/close Thai syllables detector This function is used for finding Thai syllables that are open or closed sound. @@ -189,8 +187,7 @@ def syllable_open_close_detector(syllable: str) -> str: def syllable_length(syllable: str) -> str: - """ - Thai syllable length + """Thai syllable length This function is used for finding syllable's length. (long or short) @@ -241,8 +238,7 @@ def _check_sonorant_syllable(syllable: str) -> bool: def tone_detector(syllable: str) -> str: - """ - Thai tone detector for syllables + """Thai tone detector for syllables Return tone of a syllable. diff --git a/pythainlp/util/thai.py b/pythainlp/util/thai.py index d673d1beb..a8f7c8550 100644 --- a/pythainlp/util/thai.py +++ b/pythainlp/util/thai.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Check if it is Thai text """ -Check if it is Thai text -""" + +from __future__ import annotations import string -from typing import Tuple from collections import defaultdict from pythainlp import ( @@ -205,7 +204,6 @@ def display_thai_char(ch: str) -> str: display_thai_char("้") # output: "_้" """ - if ( ch in thai_above_vowels or ch in thai_tonemarks @@ -217,9 +215,8 @@ def display_thai_char(ch: str) -> str: return ch -def thai_word_tone_detector(word: str) -> Tuple[str, str]: - """ - Thai tone detector for word. +def thai_word_tone_detector(word: str) -> tuple[str, str]: + """Thai tone detector for word. It uses pythainlp.transliterate.pronunciate for converting word to\ pronunciation. @@ -248,8 +245,7 @@ def thai_word_tone_detector(word: str) -> Tuple[str, str]: def count_thai_chars(text: str) -> dict: - """ - Count Thai characters by type + """Count Thai characters by type This function will give you numbers of Thai characters by type\ (consonants, vowels, lead_vowels, follow_vowels, above_vowels,\ @@ -319,8 +315,7 @@ def count_thai_chars(text: str) -> dict: def analyze_thai_text(text: str) -> dict: - """ - Analyzes a string of Thai text and returns a dictionaries, + """Analyzes a string of Thai text and returns a dictionaries, where each values represents a single classified character from the text. The function processes the text character by character and maps each Thai @@ -337,6 +332,7 @@ def analyze_thai_text(text: str) -> dict: >>> analyze_thai_text("เล่น") {'สระ เอ': 1, 'ล': 1, 'ไม้เอก': 1, 'น': 1} + """ results = defaultdict(int) @@ -345,9 +341,9 @@ def analyze_thai_text(text: str) -> dict: # Check if the character is in our mapping if char in THAI_CHAR_NAMES: name = THAI_CHAR_NAMES[char] - results[name]+=1 + results[name] += 1 else: # If the character is not a known Thai character, classify it as character - results[char]+=1 + results[char] += 1 return dict(results) diff --git a/pythainlp/util/thai_lunar_date.py b/pythainlp/util/thai_lunar_date.py index b416d9443..e89002686 100644 --- a/pythainlp/util/thai_lunar_date.py +++ b/pythainlp/util/thai_lunar_date.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -This file is a port from +"""This file is a port from > https://gist.github.com/touchiep/99f4f5bb349d6b983ef78697630ab78e """ +from __future__ import annotations + from datetime import date, timedelta -from typing import Dict, List, Tuple, Union -_YEAR_DEV: Dict[int, float] = { +_YEAR_DEV: dict[int, float] = { 0: 0, 1901: 0.122733000004352, 1906: 1.91890000045229e-02, @@ -190,7 +189,7 @@ _DAYS_384 = [29, 30, 29, 30, 29, 30, 29, 30, 30, 29, 30, 29, 30, 29, 30] # Zodiac names in Thai, English, and Numeric representations -_ZODIAC: Dict[int, List[Union[str, int]]] = { +_ZODIAC: dict[int, list[str | int]] = { 1: [ "ชวด", "ฉลู", @@ -223,7 +222,7 @@ } -def _calculate_f_year_f_dev(year: int) -> Tuple[int, float]: +def _calculate_f_year_f_dev(year: int) -> tuple[int, float]: if year in _YEAR_DEV: return year, _YEAR_DEV[year] @@ -281,8 +280,7 @@ def last_day_in_year(year: int) -> int: def athikasurathin(year: int) -> bool: - """ - Check if a year is a leap year in the Thai lunar calendar + """Check if a year is a leap year in the Thai lunar calendar """ # Check divisibility by 400 (divisible by 400 is always a leap year) if year % 400 == 0: @@ -308,9 +306,8 @@ def number_day_in_year(year: int) -> int: return 365 -def th_zodiac(year: int, output_type: int = 1) -> Union[str, int]: - """ - Thai Zodiac Year Name +def th_zodiac(year: int, output_type: int = 1) -> str | int: + """Thai Zodiac Year Name Converts a Gregorian year to its corresponding Zodiac name. :param int year: The Gregorian year. AD (Anno Domini) @@ -331,8 +328,7 @@ def th_zodiac(year: int, output_type: int = 1) -> Union[str, int]: def to_lunar_date(input_date: date) -> str: - """ - Convert the solar date to Thai Lunar Date + """Convert the solar date to Thai Lunar Date :param date input_date: date of the day. :return: Thai text lunar date diff --git a/pythainlp/util/thaiwordcheck.py b/pythainlp/util/thaiwordcheck.py index cc4bed0ab..106dfbdf8 100644 --- a/pythainlp/util/thaiwordcheck.py +++ b/pythainlp/util/thaiwordcheck.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp.tools import warn_deprecation diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index ff54cb91a..b8d1a7aaf 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Spell out time as Thai words. +"""Spell out time as Thai words. Convert time string or time object to Thai words. """ + +from __future__ import annotations + from datetime import datetime, time -from typing import Union +from functools import lru_cache from pythainlp.tokenize import Tokenizer from pythainlp.util.numtoword import num_to_thaiword @@ -43,9 +44,14 @@ "นาฬิกา": 0, "ครึ่ง": 30, } -_THAI_TIME_CUT = Tokenizer( - custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" -) + + +@lru_cache +def _thai_time_cut(): + """Lazy load Thai time tokenizer with cache""" + return Tokenizer(custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm") + + _THAI_TIME_AFFIX = [ "โมงเช้า", "บ่ายโมง", @@ -116,7 +122,7 @@ def _format( m: int, s: int, fmt: str = "24h", - precision: Union[str, None] = None, + precision: str | None = None, ) -> str: text = "" if fmt == "6h": @@ -148,12 +154,11 @@ def _format( def time_to_thaiword( - time_data: Union[time, datetime, str], + time_data: time | datetime | str, fmt: str = "24h", - precision: Union[str, None] = None, + precision: str | None = None, ) -> str: - """ - Spell out time as Thai words. + """Spell out time as Thai words. :param str time_data: time input, can be a datetime.time object \ or a datetime.datetime object \ @@ -229,8 +234,7 @@ def time_to_thaiword( def thaiword_to_time(text: str, padding: bool = True) -> str: - """ - Convert Thai time in words into time (H:M). + """Convert Thai time in words into time (H:M). :param str text: Thai time in words :param bool padding: Zero pad the hour if True @@ -266,10 +270,10 @@ def thaiword_to_time(text: str, padding: bool = True) -> str: _LIST_THAI_TIME = _time.split("|") del _time - hour = _THAI_TIME_CUT.word_tokenize(_LIST_THAI_TIME[0]) + hour = _thai_time_cut().word_tokenize(_LIST_THAI_TIME[0]) minute = _LIST_THAI_TIME[1] if len(minute) > 1: - minute = _THAI_TIME_CUT.word_tokenize(minute) + minute = _thai_time_cut().word_tokenize(minute) else: minute = 0 text = "" diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index d08aa07c1..e3e518dc7 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Trie data structure. +"""Trie data structure. Designed to be used for tokenizer's dictionary, but can be for other purposes. """ -from typing import Iterable, Iterator, List, Union + +from __future__ import annotations + +from collections.abc import Iterable, Iterator class Trie(Iterable[str]): @@ -26,8 +27,7 @@ def __init__(self, words: Iterable[str]): self.add(word) def add(self, word: str) -> None: - """ - Add a word to the trie. + """Add a word to the trie. Spaces in front of and following the word will be removed. :param str text: a word @@ -44,8 +44,7 @@ def add(self, word: str) -> None: cur.end = True def remove(self, word: str) -> None: - """ - Remove a word from the trie. + """Remove a word from the trie. If the word is not found, do nothing. :param str text: a word @@ -69,9 +68,8 @@ def remove(self, word: str) -> None: break del parent.children[ch] # remove from parent dict - def prefixes(self, text: str) -> List[str]: - """ - List all possible words from first sequence of characters in a word. + def prefixes(self, text: str) -> list[str]: + """List all possible words from first sequence of characters in a word. :param str text: a word :return: a list of possible words @@ -98,9 +96,8 @@ def __len__(self) -> int: return len(self.words) -def dict_trie(dict_source: Union[str, Iterable[str], Trie]) -> Trie: - """ - Create a dictionary trie from a file or an iterable. +def dict_trie(dict_source: str | Iterable[str] | Trie) -> Trie: + """Create a dictionary trie from a file or an iterable. :param str|Iterable[str]|pythainlp.util.Trie dict_source: a path to dictionary file or a list of words or a pythainlp.util.Trie object @@ -111,7 +108,7 @@ def dict_trie(dict_source: Union[str, Iterable[str], Trie]) -> Trie: if isinstance(dict_source, str) and len(dict_source) > 0: # dict_source is a path to dictionary text file - with open(dict_source, "r", encoding="utf8") as f: + with open(dict_source, encoding="utf8") as f: _vocabs = f.read().splitlines() trie = Trie(_vocabs) elif isinstance(dict_source, Iterable) and not isinstance( diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 2e08d3b6a..c213b6ec1 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -1,15 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert number in words to a computable number value +"""Convert number in words to a computable number value First version of the code adapted from Korakot Chaovavanich's notebook https://colab.research.google.com/drive/148WNIeclf0kOU6QxKd6pcfwpSs8l-VKD#scrollTo=EuVDd0nNuI8Q """ + +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp.corpus import thai_words from pythainlp.tokenize import Tokenizer @@ -44,10 +45,13 @@ "แสน": 100000, # "ล้าน" was excluded as a special case } -_valid_tokens = ( - set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} -) -_tokenizer = Tokenizer(custom_dict=_valid_tokens) +_valid_tokens = set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} + + +@lru_cache +def _tokenizer(): + """Lazy load Thai numeral tokenizer with cache""" + return Tokenizer(custom_dict=_valid_tokens) def _check_is_thainum(word: str): @@ -60,16 +64,19 @@ def _check_is_thainum(word: str): return (False, None) -_dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] -_dict_words += list(_digits.keys()) -_dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] - -_tokenizer_thaiwords = Tokenizer(_dict_words) +@lru_cache +def _tokenizer_thaiwords(): + """Lazy load Thai words tokenizer with cache""" + _dict_words = [ + i for i in list(thai_words()) if not _check_is_thainum(i)[0] + ] + _dict_words += list(_digits.keys()) + _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] + return Tokenizer(_dict_words) def thaiword_to_num(word: str) -> int: - """ - Converts the spelled-out numerals in Thai scripts into an actual integer. + """Converts the spelled-out numerals in Thai scripts into an actual integer. :param str word: Spelled-out numerals in Thai scripts :return: Corresponding integer value of the input @@ -96,7 +103,7 @@ def thaiword_to_num(word: str) -> int: if not _re_thai_numerals.fullmatch(word): raise ValueError("The input string is not a valid Thai numeral") - tokens = _tokenizer.word_tokenize(word) + tokens = _tokenizer().word_tokenize(word) accumulated = 0 next_digit = 1 @@ -134,8 +141,7 @@ def _decimal_unit(words: list) -> float: def words_to_num(words: list) -> float: - """ - Thai Words to float + """Thai Words to float :param str text: Thai words :return: float of words @@ -165,9 +171,8 @@ def words_to_num(words: list) -> float: return num -def text_to_num(text: str) -> List[str]: - """ - Thai text to list of Thai words with floating point numbers +def text_to_num(text: str) -> list[str]: + """Thai text to list of Thai words with floating point numbers :param str text: Thai text with the spelled-out numerals :return: list of Thai words with float values of the input @@ -185,7 +190,7 @@ def text_to_num(text: str) -> List[str]: # output: ['10021889', 'บาท'] """ - _temp = _tokenizer_thaiwords.word_tokenize(text) + _temp = _tokenizer_thaiwords().word_tokenize(text) thainum = [] last_index = -1 list_word_new = [] diff --git a/pythainlp/wangchanberta/__init__.py b/pythainlp/wangchanberta/__init__.py index d3d565fe5..88d8e1957 100644 --- a/pythainlp/wangchanberta/__init__.py +++ b/pythainlp/wangchanberta/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index c6a071ea4..d07537f32 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import re import warnings -from typing import List, Tuple, Union from transformers import ( CamembertTokenizer, @@ -25,8 +25,7 @@ class ThaiNameTagger: def __init__( self, dataset_name: str = "thainer", grouped_entities: bool = True ): - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -56,9 +55,8 @@ def _clear_tag(self, tag): def get_ner( self, text: str, pos: bool = False, tag: bool = False - ) -> Union[List[Tuple[str, str]], str]: - """ - This function tags named entities in text in IOB format. + ) -> list[tuple[str, str]] | str: + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -136,8 +134,7 @@ class NamedEntityRecognition: def __init__( self, model: str = "pythainlp/thainer-corpus-v2-base-model" ) -> None: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -165,9 +162,8 @@ def _fix_span_error(self, words, ner): def get_ner( self, text: str, pos: bool = False, tag: bool = False - ) -> Union[List[Tuple[str, str]], str]: - """ - This function tags named entities in text in IOB format. + ) -> list[tuple[str, str]] | str: + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -225,9 +221,8 @@ def get_ner( return ner_tag -def segment(text: str) -> List[str]: - """ - Subword tokenize. SentencePiece from wangchanberta model. +def segment(text: str) -> list[str]: + """Subword tokenize. SentencePiece from wangchanberta model. :param str text: text to be tokenized :return: list of subwords diff --git a/pythainlp/word_vector/__init__.py b/pythainlp/word_vector/__init__.py index c8b0f9a6d..00c8423ae 100644 --- a/pythainlp/word_vector/__init__.py +++ b/pythainlp/word_vector/__init__.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -thai2fit - Thai word vector. +"""thai2fit - Thai word vector. Initial code from https://github.com/cstorm125/thai2fit """ + __all__ = [ "WordVector", ] diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 777c38df2..9bfd19e0a 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from gensim.models import KeyedVectors from gensim.models.keyedvectors import Word2VecKeyedVectors from numpy import ndarray, zeros from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER, word_tokenize +from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize WV_DIM = 300 # word vector dimension @@ -20,8 +19,7 @@ class WordVector: - """ - Word Vector class + """Word Vector class :param str model_name: model name @@ -33,8 +31,7 @@ class WordVector: """ def __init__(self, model_name: str = "thai2fit_wv") -> None: - """ - Word Vector class + """Word Vector class :param str model_name: model name @@ -47,8 +44,7 @@ def __init__(self, model_name: str = "thai2fit_wv") -> None: self.load_wordvector(model_name) def load_wordvector(self, model_name: str): - """ - Load word vector model. + """Load word vector model. :param str model_name: model name """ @@ -61,22 +57,20 @@ def load_wordvector(self, model_name: str): self.WV_DIM = self.model.vector_size if self.model_name == "thai2fit_wv": - self.tokenize = THAI2FIT_TOKENIZER.word_tokenize + self.tokenize = thai2fit_tokenizer().word_tokenize else: self.tokenize = word_tokenize def get_model(self) -> Word2VecKeyedVectors: - """ - Get word vector model. + """Get word vector model. :return: `gensim` word2vec model :rtype: gensim.models.keyedvectors.Word2VecKeyedVectors """ return self.model - def doesnt_match(self, words: List[str]) -> str: - """ - This function returns one word that is mostly unrelated to other words + def doesnt_match(self, words: list[str]) -> str: + """This function returns one word that is mostly unrelated to other words in the list. We use the function :func:`doesnt_match` from :mod:`gensim`. @@ -96,7 +90,7 @@ def doesnt_match(self, words: List[str]) -> str: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> words = ['อาหารเช้า', 'อาหารเที่ยง', 'อาหารเย็น', 'พริกไทย'] + >>> words = ["อาหารเช้า", "อาหารเที่ยง", "อาหารเย็น", "พริกไทย"] >>> wv.doesnt_match(words) พริกไทย @@ -106,17 +100,16 @@ def doesnt_match(self, words: List[str]) -> str: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> words = ['ดีไซน์เนอร์', 'พนักงานเงินเดือน', 'หมอ', 'เรือ'] + >>> words = ["ดีไซน์เนอร์", "พนักงานเงินเดือน", "หมอ", "เรือ"] >>> wv.doesnt_match(words) เรือ """ return self.model.doesnt_match(words) def most_similar_cosmul( - self, positive: List[str], negative: List[str] - ) -> List[Tuple[str, float]]: - """ - This function finds the top-10 words that are most similar with respect + self, positive: list[str], negative: list[str] + ) -> list[tuple[str, float]]: + """This function finds the top-10 words that are most similar with respect to two lists of words labeled as positive and negative. The top-10 most similar words are obtained using multiplication combination objective from Omer Levy and Yoav Goldberg @@ -147,7 +140,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['แม่น้ำ'] + >>> list_positive = ["แม่น้ำ"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ลำน้ำ', 0.8206598162651062), ('ทะเลสาบ', 0.775945782661438), @@ -162,7 +155,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['นายก', 'รัฐมนตรี', 'ประเทศ'] + >>> list_positive = ["นายก", "รัฐมนตรี", "ประเทศ"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('รองนายกรัฐมนตรี', 0.2730445861816406), @@ -180,7 +173,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['ประเทศ', 'ไทย', 'จีน', 'ญี่ปุ่น'] + >>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ประเทศจีน', 0.22022421658039093), ('เกาหลี', 0.2196873426437378), @@ -191,8 +184,8 @@ def most_similar_cosmul( ('อังกฤษ', 0.19610872864723206), ('ฮ่องกง', 0.1928885132074356), ('ฝรั่งเศส', 0.18383873999118805), ('พม่า', 0.18369348347187042)] >>> - >>> list_positive = ['ประเทศ', 'ไทย', 'จีน', 'ญี่ปุ่น'] - >>> list_negative = ['อเมริกา'] + >>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"] + >>> list_negative = ["อเมริกา"] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ประเทศไทย', 0.3278159201145172), ('เกาหลี', 0.3201899230480194), ('ประเทศจีน', 0.31755179166793823), ('พม่า', 0.30845439434051514), @@ -207,7 +200,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['เมนูอาหารไทย'] + >>> list_positive = ["เมนูอาหารไทย"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) KeyError: "word 'เมนูอาหารไทย' not in vocabulary" @@ -217,8 +210,7 @@ def most_similar_cosmul( ) def similarity(self, word1: str, word2: str) -> float: - """ - This function computes cosine similarity between two words. + """This function computes cosine similarity between two words. :param str word1: first word to be compared with :param str word2: second word to be compared with @@ -239,7 +231,7 @@ def similarity(self, word1: str, word2: str) -> float: >>> from pythainlp.word_vector import WordVector >>> wv = WordVector() - >>> wv.similarity('รถไฟ', 'รถไฟฟ้า') + >>> wv.similarity("รถไฟ", "รถไฟฟ้า") 0.43387136 @@ -249,15 +241,14 @@ def similarity(self, word1: str, word2: str) -> float: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> wv.similarity('เสือดาว', 'รถไฟฟ้า') + >>> wv.similarity("เสือดาว", "รถไฟฟ้า") 0.04300258 """ return self.model.similarity(word1, word2) def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: - """ - This function converts a Thai sentence into vector. + """This function converts a Thai sentence into vector. Specifically, it first tokenizes that text and map each tokenized word with the word vectors from the model. Then, word vectors are aggregated into one vector of 300 dimension @@ -282,7 +273,7 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> sentence = 'อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735' + >>> sentence = "อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735" >>> wv.sentence_vectorizer(sentence, use_mean=True) array([[-0.00421414, -0.08881307, 0.05081136, -0.05632929, -0.06607185, 0.03059357, -0.113882 , -0.00074836, 0.05035743, diff --git a/pythainlp/wsd/__init__.py b/pythainlp/wsd/__init__.py index f2933bb1a..1fce7736d 100644 --- a/pythainlp/wsd/__init__.py +++ b/pythainlp/wsd/__init__.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Thai Word Sense Disambiguation (WSD) """ -Thai Word Sense Disambiguation (WSD) -""" + __all__ = ["get_sense"] from pythainlp.wsd.core import get_sense diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index add2b3ad7..04e5da796 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple, Union +from __future__ import annotations from pythainlp.corpus import thai_wsd_dict from pythainlp.tokenize import Tokenizer @@ -53,11 +52,10 @@ def get_sense( device: str = "cpu", custom_dict: dict = dict(), custom_tokenizer: Tokenizer = _word_cut, -) -> List[Tuple[str, float]]: - """ - Get word sense from the sentence. +) -> list[tuple[str, float]]: + """Get word sense from the sentence. This function will get definition and distance from context in sentence. - + :param str sentence: Thai sentence :param str word: Thai word :param str device: device for running model on. @@ -67,19 +65,19 @@ def get_sense( :return: a list of definitions and distances (1 - cos_sim) or \ an empty list (if word is not in the dictionary) :rtype: List[Tuple[str, float]] - + We get the ideas from `Context-Aware Semantic Similarity Measurement for \ Unsupervised Word Sense Disambiguation \ `_ to build get_sense function. Use Thai dictionary from wiktionary. See `thai_dict `_. - + Use sentence transformers model from \ `sentence-transformers/paraphrase-multilingual-mpnet-base-v2 \ `_ \ for unsupervised word sense disambiguation. - + :Example: :: diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 44904e295..000000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -PyYAML>=5.4.1 -numpy>=1.22 -pyicu>=2.3 -python-crfsuite>=0.9.7 -requests>=2.31 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c0245293d..000000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[bumpversion] -current_version = 5.2.0 -commit = True -tag = True -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}-{release}{build} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - beta - prod - -[bumpversion:part:build] - -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - -[bumpversion:file:pythainlp/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[metadata] -description_file = README.md - -[coverage:run] -source = pythainlp diff --git a/setup.py b/setup.py deleted file mode 100644 index 86dc13857..000000000 --- a/setup.py +++ /dev/null @@ -1,221 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -""" -Setup script for PyThaiNLP. - -https://github.com/PyThaiNLP/pythainlp -""" - -from setuptools import find_packages, setup - -PYYAML = "PyYAML>=5.4.1" -PANDAS = "pandas>=0.24" -NUMPY = "numpy>=1.22" - -LONG_DESC = """ -![PyThaiNLP Logo](https://avatars0.githubusercontent.com/u/32934255?s=200&v=4) - -PyThaiNLP is a Python library for Thai natural language processing. -The library provides functions like word tokenization, part-of-speech tagging, -transliteration, soundex generation, spell checking, and -date and time parsing/formatting. - -Website: [pythainlp.github.io](https://pythainlp.org/) - -# Install - -For stable version: - -```sh -pip install pythainlp -``` - -For development version: - -```sh -pip install --upgrade --pre pythainlp -``` - -Some functionalities, like named-entity recognition, require extra packages. -See https://github.com/PyThaiNLP/pythainlp for installation options. -""" - -requirements = [ - "backports.zoneinfo; python_version<'3.9'", - "requests>=2.31", - PYYAML, - PANDAS, - NUMPY, - "tzdata; sys_platform == 'win32'", -] - -extras = { - "abbreviation": ["khamyo>=0.2.0"], - "attacut": ["attacut>=1.0.6"], - "benchmarks": [PYYAML, NUMPY, PANDAS], - "coreference_resolution": [ - "fastcoref>=2.1.5", - "spacy>=3.0", - ], - "dependency_parsing": [ - "spacy_thai>=0.7.1", - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - ], - "el": ["multiel>=0.5"], - "esupar": [ - "esupar>=1.3.8", - "numpy", - "transformers>=4.22.1", - ], - "generate": ["fastai<2.0"], - "icu": ["pyicu>=2.3"], - "ipa": ["epitran>=1.1"], - "ml": [NUMPY, "torch>=1.0.0"], - "mt5": ["sentencepiece>=0.1.91", "transformers>=4.6.0"], - "nlpo3": ["nlpo3>=1.3.1"], - "onnx": [NUMPY, "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"], - "oskut": ["oskut>=1.3"], - "sefr_cut": ["sefr_cut>=1.1"], - "spacy_thai": ["spacy_thai>=0.7.1"], - "spell": ["phunspell>=0.1.6", "symspellpy>=6.7.6"], - "ssg": ["ssg>=0.0.8"], - "textaugment": ["bpemb", "gensim>=4.0.0"], - "thai_nner": ["thai_nner"], - "thai2fit": ["emoji>=0.5.1", "gensim>=4.0.0", NUMPY], - "thai2rom": [NUMPY, "torch>=1.0.0"], - "budoux": ["budoux>=0.7.0"], - "translate": [ - 'fairseq>=0.10.0,<0.13;python_version<"3.11"', - 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', - "sacremoses>=0.0.41", - "sentencepiece>=0.1.91", - "torch>=1.0.0", - "transformers>=4.6.0", - "word2word>=1.0.0" - ], - "transformers_ud": [ - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - ], - "wangchanberta": ["sentencepiece>=0.1.91", "transformers>=4.6.0"], - "wangchanglm": [ - PANDAS, - "sentencepiece>=0.1.91", - "transformers>=4.6.0", - ], - "word_approximation": ["panphon>=0.20.0"], - "wordnet": ["nltk>=3.3"], - "wsd": ["sentence-transformers>=2.2.2"], - "wtp": ["transformers>=4.6.0", "wtpsplit>=1.0.1"], - "wunsen": ["wunsen>=0.0.1"], - # Compact dependencies, this one matches requirements.txt - "compact": [ - PYYAML, - "nlpo3>=1.3.1", - NUMPY, - "pyicu>=2.3", - "python-crfsuite>=0.9.7", - ], - # Full dependencies - "full": [ - PYYAML, - "attacut>=1.0.4", - "bpemb>=0.3.2", - "emoji>=0.5.1", - "epitran>=1.1", - 'fairseq>=0.10.0,<0.13;python_version<"3.11"', - 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', - "fastai<2.0", - "fastcoref>=2.1.5", - "gensim>=4.0.0", - "khamyo>=0.2.0", - "nlpo3>=1.3.1", - "nltk>=3.3", - NUMPY, - "onnxruntime>=1.10.0", - "oskut>=1.3", - PANDAS, - "panphon>=0.20.0", - "phunspell>=0.1.6", - "pyicu>=2.3", - "sacremoses>=0.0.41", - "sefr_cut>=1.1", - "sentencepiece>=0.1.91", - "sentence-transformers>=2.2.2", - "spacy>=3.0", - "spacy_thai>=0.7.1", - "ssg>=0.0.8", - "symspellpy>=6.7.6", - "thai_nner", - "torch>=1.0.0", - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - "wtpsplit>=1.0.1", - "wunsen>=0.0.3", - "word2word>=1.0.0", - "budoux>=0.7.0", - ], -} - -setup( - name="pythainlp", - version="5.2.0", - description="Thai Natural Language Processing library", - long_description=LONG_DESC, - long_description_content_type="text/markdown", - author="PyThaiNLP", - author_email="wannaphong@pythainlp.org", - url="https://github.com/PyThaiNLP/pythainlp", - packages=find_packages(exclude=["tests", "tests.*"]), - test_suite="tests", - python_requires=">=3.7", - package_data={ - "pythainlp": [ - "corpus/*", - ], - }, - include_package_data=True, - install_requires=requirements, - extras_require=extras, - license="Apache-2.0", - zip_safe=False, - keywords=[ - "pythainlp", - "NLP", - "natural language processing", - "text analytics", - "text processing", - "localization", - "computational linguistics", - "ThaiNLP", - "Thai NLP", - "Thai language", - ], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: Thai", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Text Processing", - "Topic :: Text Processing :: General", - "Topic :: Text Processing :: Linguistic", - ], - entry_points={ - "console_scripts": [ - "thainlp = pythainlp.__main__:main", - ], - }, - project_urls={ - "Documentation": "https://pythainlp.org/docs/5.2/", - "Tutorials": "https://pythainlp.org/tutorials/", - "Source Code": "https://github.com/PyThaiNLP/pythainlp", - "Bug Tracker": "https://github.com/PyThaiNLP/pythainlp/issues", - }, -) - -# TODO: Check extras and decide whether or not additional data, like model files, should be downloaded diff --git a/tests/__init__.py b/tests/__init__.py index 97ba77207..e325bf4ed 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. +"""Unit test. Each file in tests/ is for each main package. """ diff --git a/tests/compact/__init__.py b/tests/compact/__init__.py index 3aa77b4a2..63bba7f50 100644 --- a/tests/compact/__init__.py +++ b/tests/compact/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. Compact version. +"""Unit test. Compact version. Test functions that require "compact" dependencies (see setup.py). """ diff --git a/tests/compact/test_cli.py b/tests/compact/test_cli.py index 6c3a2dbe5..38b4c6c70 100644 --- a/tests/compact/test_cli.py +++ b/tests/compact/test_cli.py @@ -2,13 +2,12 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.cli module. (compact) +"""Unit tests for pythainlp.cli module. (compact) """ import unittest -from pythainlp import __main__, cli +from pythainlp import cli from pythainlp.cli.misspell import App as MisspellApp diff --git a/tests/compact/testc_util.py b/tests/compact/testc_util.py index 007db4c15..1f6d06366 100644 --- a/tests/compact/testc_util.py +++ b/tests/compact/testc_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import unittest diff --git a/tests/core/__init__.py b/tests/core/__init__.py index 3aac3cb6c..ea3afeac0 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. +"""Unit test. Each file in tests/ is for each main package. """ diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index 29494d63e..f541b08ef 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -6,8 +6,6 @@ import os import unittest -from requests import Response - from pythainlp.corpus import ( countries, download, diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index bd5562eb3..b1cbb0956 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -6,8 +6,8 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, Tokenizer, + display_cell_tokenize, etcc, longest, multi_cut, @@ -18,8 +18,8 @@ tcc, tcc_p, word_detokenize, + word_dict_trie, word_tokenize, - display_cell_tokenize, ) from pythainlp.util import dict_trie @@ -261,7 +261,7 @@ def test_numeric_data_format(self): class TokenizeTestCase(unittest.TestCase): def test_Tokenizer(self): - _tokenizer = Tokenizer(DEFAULT_WORD_DICT_TRIE) + _tokenizer = Tokenizer(word_dict_trie()) self.assertEqual(_tokenizer.word_tokenize(""), []) _tokenizer.set_tokenize_engine("longest") self.assertEqual(_tokenizer.word_tokenize(None), []) @@ -411,7 +411,6 @@ def test_longest(self): def test_longest_custom_dict(self): """Test switching the custom dict on longest segment function""" - self.assertEqual( word_tokenize("ทดสอบ ทดสอบ", engine="longest"), ["ทดสอบ", " ", "ทดสอบ"], diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 19558ba92..97f48b437 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import os @@ -16,6 +15,7 @@ from pythainlp.corpus.common import _THAI_WORDS_FILENAME from pythainlp.util import ( Trie, + analyze_thai_text, arabic_digit_to_thai_digit, bahttext, collate, @@ -46,6 +46,7 @@ remove_trailing_repeat_consonants, remove_zw, sound_syllable, + spelling, syllable_length, syllable_open_close_detector, text_to_arabic_digit, @@ -66,8 +67,6 @@ to_lunar_date, tone_detector, words_to_num, - spelling, - analyze_thai_text, ) from pythainlp.util.morse import morse_decode, morse_encode diff --git a/tests/extra/__init__.py b/tests/extra/__init__.py index f8b6bcf16..4073d5b1c 100644 --- a/tests/extra/__init__.py +++ b/tests/extra/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. Extra version. +"""Unit test. Extra version. Test functions that require dependencies beyond "compact" (see setup.py). """ diff --git a/tests/extra/testx_cli.py b/tests/extra/testx_cli.py index fe6c8e56d..730dff76c 100644 --- a/tests/extra/testx_cli.py +++ b/tests/extra/testx_cli.py @@ -5,7 +5,7 @@ import unittest -from pythainlp import __main__, cli +from pythainlp import cli from pythainlp.cli.benchmark import App as BenchmarkApp from pythainlp.cli.data import App as DataApp from pythainlp.cli.tokenize import App as TokenizeApp diff --git a/tests/extra/testx_coref.py b/tests/extra/testx_coref.py index bb2e4d09b..749b9a773 100644 --- a/tests/extra/testx_coref.py +++ b/tests/extra/testx_coref.py @@ -5,8 +5,6 @@ import unittest -from pythainlp.coref import coreference_resolution - class CorefTestCaseX(unittest.TestCase): def test_coreference_resolution(self): diff --git a/tests/extra/testx_spell.py b/tests/extra/testx_spell.py index 644def075..ea8f65273 100644 --- a/tests/extra/testx_spell.py +++ b/tests/extra/testx_spell.py @@ -8,10 +8,10 @@ from pythainlp.spell import ( correct, correct_sent, + get_words_spell_suggestion, spell, spell_sent, symspellpy, - get_words_spell_suggestion, ) from ..core.test_spell import SENT_TOKS diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index 321c7c747..d2170e0dd 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -8,7 +8,6 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, attacut, deepcut, nercut, @@ -19,6 +18,7 @@ ssg, subword_tokenize, tltk, + word_dict_trie, word_tokenize, ) @@ -275,12 +275,12 @@ def test_word_tokenize_deepcut(self): def test_deepcut(self): self.assertEqual(deepcut.segment(None), []) self.assertEqual(deepcut.segment(""), []) - self.assertIsNotNone(deepcut.segment("ทดสอบ", DEFAULT_WORD_DICT_TRIE)) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) self.assertIsNotNone( word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=DEFAULT_WORD_DICT_TRIE + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() ) ) diff --git a/tests/extra/testx_util.py b/tests/extra/testx_util.py index be81edd08..9bc2d857e 100644 --- a/tests/extra/testx_util.py +++ b/tests/extra/testx_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import unittest diff --git a/tox.ini b/tox.ini index ae24e75d4..59ee3c9dd 100644 --- a/tox.ini +++ b/tox.ini @@ -8,10 +8,10 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/pythainlp changedir = tests commands = discover deps = discover -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following lines: +; If you want to install the package with dependencies for testing: ; deps = -; -r{toxinidir}/requirements.txt +; .[compact] +; discover [testenv:flake8] basepython = python From 7dd940df9d4426918855157b8f351d41e0e99d7f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 05:31:07 +0000 Subject: [PATCH 07/13] Add documentation for profanity detection functions to docs/api/util.rst - Added censor_profanity documentation - Added contains_profanity documentation - Added find_profanity documentation - Functions documented in alphabetical order with descriptions Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- docs/api/util.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/api/util.rst b/docs/api/util.rst index ef38bcd44..a5339cedb 100644 --- a/docs/api/util.rst +++ b/docs/api/util.rst @@ -27,16 +27,26 @@ Modules The `bahttext` function specializes in converting numerical values into Thai Baht text, an essential feature for rendering financial data or monetary amounts in a user-friendly Thai format. -.. autofunction:: convert_years +.. autofunction:: censor_profanity :noindex: - The `convert_years` function is designed to facilitate the conversion of Western calendar years into Thai Buddhist Era (BE) years. This is significant for presenting dates and years in a Thai context. + The `censor_profanity` function replaces profanity words in Thai text with a replacement character (default: "*"). Users can provide custom profanity words in addition to the built-in list for content moderation and filtering. .. autofunction:: collate :noindex: The `collate` function is a versatile tool for sorting Thai text in a locale-specific manner. It ensures that text data is sorted correctly, taking into account the Thai language's unique characteristics. +.. autofunction:: contains_profanity + :noindex: + + The `contains_profanity` function checks if Thai text contains profanity words. It returns True if profanity is detected and False otherwise. Users can provide custom profanity words for enhanced content moderation. + +.. autofunction:: convert_years + :noindex: + + The `convert_years` function is designed to facilitate the conversion of Western calendar years into Thai Buddhist Era (BE) years. This is significant for presenting dates and years in a Thai context. + .. autofunction:: count_thai_chars :noindex: @@ -77,6 +87,11 @@ Modules The `find_keyword` function is a powerful utility for identifying keywords and key phrases in text data. It is a fundamental component for text analysis and information extraction tasks. +.. autofunction:: find_profanity + :noindex: + + The `find_profanity` function identifies and returns a list of all profanity words found in Thai text. Users can provide custom profanity words to enhance detection capabilities for content moderation. + .. autofunction:: ipa_to_rtgs :noindex: From 0906103e0d9114a849638437266a97ae13537b91 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 11:37:50 +0000 Subject: [PATCH 08/13] Update tests/core/test_profanity.py --- tests/core/test_profanity.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/core/test_profanity.py b/tests/core/test_profanity.py index e013a6c5d..b0622e1d3 100644 --- a/tests/core/test_profanity.py +++ b/tests/core/test_profanity.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 From b8ab5f864aa7cc750ef3c8697fbb7830ef8c5427 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 11:38:06 +0000 Subject: [PATCH 09/13] Update pythainlp/corpus/profanity_th.txt --- pythainlp/corpus/profanity_th.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pythainlp/corpus/profanity_th.txt b/pythainlp/corpus/profanity_th.txt index d812820e3..ddb35559d 100644 --- a/pythainlp/corpus/profanity_th.txt +++ b/pythainlp/corpus/profanity_th.txt @@ -1,7 +1,10 @@ -# Thai profanity words -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: OTHER +# SPDX-FileType: TEXT # SPDX-License-Identifier: Apache-2.0 # +# Thai profanity words +# # This list contains common Thai profanity words for filtering/detection purposes # Words are stored without tone marks or variations for broader matching ควย From 448ffb375c1d0e427a71716bec8c5b260d78381d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 11:38:43 +0000 Subject: [PATCH 10/13] Update pythainlp/util/profanity.py --- pythainlp/util/profanity.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index 58e6e77b1..77c35a34c 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ From a8d48c4ff5715c9a39c8d52195155a24ff06a533 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 11:52:10 +0000 Subject: [PATCH 11/13] Remove unused imports, remove trailing whitespaces --- pythainlp/util/profanity.py | 47 +++++++++++++++++------------------- tests/core/test_profanity.py | 13 +++++----- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index 77c35a34c..a6b7bcadd 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -4,17 +4,14 @@ """ Profanity detection for Thai language """ -from typing import Union from pythainlp.corpus.common import thai_profanity_words, thai_words from pythainlp.tokenize import word_tokenize -from pythainlp.util.trie import Trie, dict_trie +from pythainlp.util.trie import dict_trie def contains_profanity( - text: str, - custom_words: set[str] = None, - engine: str = "newmm" + text: str, custom_words: set[str] = None, engine: str = "newmm" ) -> bool: """ Check if the given text contains profanity words. @@ -42,30 +39,28 @@ def contains_profanity( """ if not text: return False - + profanity_set = set(thai_profanity_words()) if custom_words: profanity_set.update(custom_words) - + # Create custom dictionary that merges thai_words and profanity_set # for better tokenization custom_dict_set = set(thai_words()) custom_dict_set.update(profanity_set) custom_dict = dict_trie(dict_source=custom_dict_set) - + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) - + for token in tokens: if token in profanity_set: return True - + return False def find_profanity( - text: str, - custom_words: set[str] = None, - engine: str = "newmm" + text: str, custom_words: set[str] = None, engine: str = "newmm" ) -> list[str]: """ Find all profanity words in the given text. @@ -93,24 +88,24 @@ def find_profanity( """ if not text: return [] - + profanity_set = set(thai_profanity_words()) if custom_words: profanity_set.update(custom_words) - + # Create custom dictionary that merges thai_words and profanity_set # for better tokenization custom_dict_set = set(thai_words()) custom_dict_set.update(profanity_set) custom_dict = dict_trie(dict_source=custom_dict_set) - + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) - + found_profanity = [] for token in tokens: if token in profanity_set: found_profanity.append(token) - + return found_profanity @@ -118,7 +113,7 @@ def censor_profanity( text: str, replacement: str = "*", custom_words: set[str] = None, - engine: str = "newmm" + engine: str = "newmm", ) -> str: """ Replace profanity words in the text with a replacement character. @@ -147,24 +142,26 @@ def censor_profanity( """ if not text: return text - + profanity_set = set(thai_profanity_words()) if custom_words: profanity_set.update(custom_words) - + # Create custom dictionary that merges thai_words and profanity_set # for better tokenization custom_dict_set = set(thai_words()) custom_dict_set.update(profanity_set) custom_dict = dict_trie(dict_source=custom_dict_set) - - tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine, keep_whitespace=True) - + + tokens = word_tokenize( + text, custom_dict=custom_dict, engine=engine, keep_whitespace=True + ) + censored_tokens = [] for token in tokens: if token in profanity_set: censored_tokens.append(replacement * len(token)) else: censored_tokens.append(token) - + return "".join(censored_tokens) diff --git a/tests/core/test_profanity.py b/tests/core/test_profanity.py index b0622e1d3..6b1bde686 100644 --- a/tests/core/test_profanity.py +++ b/tests/core/test_profanity.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - """ Unit tests for profanity detection functions """ @@ -46,7 +45,7 @@ def test_find_profanity_with_profanity(self): result = find_profanity("ควย") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) - + result = find_profanity("สัส") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) @@ -63,7 +62,7 @@ def test_censor_profanity_with_profanity(self): result = censor_profanity("ควย") self.assertNotEqual(result, "ควย") self.assertIn("*", result) - + result = censor_profanity("สัส") self.assertNotEqual(result, "สัส") self.assertIn("*", result) @@ -78,10 +77,10 @@ def test_contains_profanity_with_custom_words(self): """Test detection with custom profanity words""" # Clean text shouldn't be detected self.assertFalse(contains_profanity("สวัสดีครับ", custom_words={"คำใหม่"})) - + # Custom word should be detected self.assertTrue(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) - + # Mix of default and custom words self.assertTrue(contains_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"})) @@ -90,7 +89,7 @@ def test_find_profanity_with_custom_words(self): # Should find custom words result = find_profanity("คำใหม่", custom_words={"คำใหม่"}) self.assertIn("คำใหม่", result) - + # Should find both default and custom words result = find_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) self.assertGreater(len(result), 1) @@ -101,7 +100,7 @@ def test_censor_profanity_with_custom_words(self): result = censor_profanity("คำใหม่", custom_words={"คำใหม่"}) self.assertNotEqual(result, "คำใหม่") self.assertIn("*", result) - + # Should censor both default and custom words result = censor_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) self.assertNotIn("ควย", result) From f4404909b08714bb52657cb3fc45924294972dc1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 13:02:41 +0000 Subject: [PATCH 12/13] Fix type violation: change custom_words type from set[str] to set[str] | None - Fixed type hint for custom_words parameter in contains_profanity() - Fixed type hint for custom_words parameter in find_profanity() - Fixed type hint for custom_words parameter in censor_profanity() - All 11 tests pass Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/util/profanity.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index a6b7bcadd..e27691ca1 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -11,7 +11,7 @@ def contains_profanity( - text: str, custom_words: set[str] = None, engine: str = "newmm" + text: str, custom_words: set[str] | None = None, engine: str = "newmm" ) -> bool: """ Check if the given text contains profanity words. @@ -60,7 +60,7 @@ def contains_profanity( def find_profanity( - text: str, custom_words: set[str] = None, engine: str = "newmm" + text: str, custom_words: set[str] | None = None, engine: str = "newmm" ) -> list[str]: """ Find all profanity words in the given text. @@ -112,7 +112,7 @@ def find_profanity( def censor_profanity( text: str, replacement: str = "*", - custom_words: set[str] = None, + custom_words: set[str] | None = None, engine: str = "newmm", ) -> str: """ From 5d82c22205aafe0655ee21387e321e4713bf821d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 13:20:52 +0000 Subject: [PATCH 13/13] Update pythainlp/util/profanity.py --- pythainlp/util/profanity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index e27691ca1..883116c57 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -4,6 +4,7 @@ """ Profanity detection for Thai language """ +from __future__ import annotations from pythainlp.corpus.common import thai_profanity_words, thai_words from pythainlp.tokenize import word_tokenize