From b6afe06f90618972e203607bb50cf8b64a73adbc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 22:21:43 +0000 Subject: [PATCH 1/3] Initial plan From 720b580d634ad24c06533ba1c5510eeccdb01c5d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 22:29:42 +0000 Subject: [PATCH 2/3] Modernize Python code to Python 3.9: remove coding headers, add future annotations, update type hints Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 2 +- pythainlp/__init__.py | 127 +- pythainlp/__main__.py | 1 - pythainlp/ancient/__init__.py | 1 - pythainlp/ancient/aksonhan.py | 2 +- pythainlp/ancient/currency.py | 19 +- pythainlp/augment/__init__.py | 1 - pythainlp/augment/lm/__init__.py | 1 - pythainlp/augment/lm/fasttext.py | 10 +- pythainlp/augment/lm/phayathaibert.py | 5 +- pythainlp/augment/lm/wangchanberta.py | 6 +- pythainlp/augment/word2vec/__init__.py | 1 - pythainlp/augment/word2vec/bpemb_wv.py | 7 +- pythainlp/augment/word2vec/core.py | 8 +- pythainlp/augment/word2vec/ltw2v.py | 7 +- pythainlp/augment/word2vec/thai2fit.py | 7 +- pythainlp/augment/wordnet.py | 9 +- pythainlp/benchmarks/__init__.py | 1 - pythainlp/benchmarks/word_tokenization.py | 19 +- pythainlp/chat/__init__.py | 1 - pythainlp/chat/core.py | 5 +- pythainlp/classify/__init__.py | 1 - pythainlp/classify/param_free.py | 27 +- pythainlp/cli/__init__.py | 18 +- pythainlp/cli/benchmark.py | 11 +- pythainlp/cli/data.py | 1 - pythainlp/cli/misspell.py | 3 +- pythainlp/cli/soundex.py | 1 - pythainlp/cli/tag.py | 1 - pythainlp/cli/tokenize.py | 1 - pythainlp/coref/__init__.py | 2 +- pythainlp/coref/_fastcoref.py | 5 +- pythainlp/coref/core.py | 5 +- pythainlp/coref/han_coref.py | 3 +- pythainlp/corpus/__init__.py | 31 +- pythainlp/corpus/common.py | 48 +- pythainlp/corpus/core.py | 71 +- pythainlp/corpus/icu.py | 6 +- pythainlp/corpus/oscar.py | 10 +- pythainlp/corpus/th_en_translit.py | 5 +- pythainlp/corpus/tnc.py | 14 +- pythainlp/corpus/ttc.py | 6 +- pythainlp/corpus/util.py | 17 +- pythainlp/corpus/volubilis.py | 6 +- pythainlp/corpus/wikipedia.py | 10 +- pythainlp/corpus/wordnet.py | 32 +- pythainlp/el/__init__.py | 1 - pythainlp/el/_multiel.py | 1 - pythainlp/el/core.py | 28 +- pythainlp/generate/__init__.py | 1 - pythainlp/generate/core.py | 10 +- pythainlp/generate/thai2fit.py | 6 +- pythainlp/generate/wangchanglm.py | 111 +- pythainlp/khavee/__init__.py | 15 +- pythainlp/khavee/core.py | 1378 ++++++++--------- pythainlp/lm/__init__.py | 11 +- pythainlp/lm/text_util.py | 21 +- pythainlp/morpheme/__init__.py | 7 +- pythainlp/morpheme/thaiwordcheck.py | 4 +- pythainlp/morpheme/word_formation.py | 7 +- pythainlp/parse/__init__.py | 2 +- pythainlp/parse/core.py | 8 +- pythainlp/parse/esupar_engine.py | 8 +- pythainlp/parse/spacy_thai_engine.py | 8 +- pythainlp/parse/transformers_ud.py | 9 +- pythainlp/parse/ud_goeswith.py | 8 +- pythainlp/phayathaibert/__init__.py | 1 - pythainlp/phayathaibert/core.py | 18 +- pythainlp/soundex/__init__.py | 1 - pythainlp/soundex/core.py | 4 +- pythainlp/soundex/lk82.py | 4 +- pythainlp/soundex/metasound.py | 3 +- pythainlp/soundex/prayut_and_somchaip.py | 4 +- pythainlp/soundex/sound.py | 33 +- pythainlp/soundex/udom83.py | 4 +- pythainlp/spell/__init__.py | 5 +- pythainlp/spell/core.py | 26 +- pythainlp/spell/phunspell.py | 10 +- pythainlp/spell/pn.py | 46 +- pythainlp/spell/symspellpy.py | 15 +- pythainlp/spell/tltk.py | 11 +- .../spell/wanchanberta_thai_grammarly.py | 87 +- pythainlp/spell/words_spelling_correction.py | 97 +- pythainlp/summarize/__init__.py | 1 - pythainlp/summarize/core.py | 15 +- pythainlp/summarize/freq.py | 9 +- pythainlp/summarize/keybert.py | 34 +- pythainlp/summarize/mt5.py | 6 +- pythainlp/tag/__init__.py | 1 - pythainlp/tag/_tag_perceptron.py | 30 +- pythainlp/tag/blackboard.py | 9 +- pythainlp/tag/chunk.py | 7 +- pythainlp/tag/crfchunk.py | 7 +- pythainlp/tag/locations.py | 7 +- pythainlp/tag/named_entity.py | 27 +- pythainlp/tag/orchid.py | 10 +- pythainlp/tag/perceptron.py | 6 +- pythainlp/tag/pos_tag.py | 30 +- pythainlp/tag/thai_nner.py | 5 +- pythainlp/tag/thainer.py | 8 +- pythainlp/tag/tltk.py | 13 +- pythainlp/tag/unigram.py | 11 +- pythainlp/tag/wangchanberta_onnx.py | 6 +- pythainlp/tokenize/__init__.py | 7 +- pythainlp/tokenize/_utils.py | 13 +- pythainlp/tokenize/attacut.py | 10 +- pythainlp/tokenize/budoux.py | 6 +- pythainlp/tokenize/core.py | 37 +- pythainlp/tokenize/crfcut.py | 20 +- pythainlp/tokenize/deepcut.py | 7 +- pythainlp/tokenize/etcc.py | 13 +- pythainlp/tokenize/han_solo.py | 6 +- pythainlp/tokenize/longest.py | 19 +- pythainlp/tokenize/multi_cut.py | 19 +- pythainlp/tokenize/nercut.py | 8 +- pythainlp/tokenize/newmm.py | 12 +- pythainlp/tokenize/nlpo3.py | 6 +- pythainlp/tokenize/oskut.py | 6 +- pythainlp/tokenize/pyicu.py | 10 +- pythainlp/tokenize/sefr_cut.py | 6 +- pythainlp/tokenize/ssg.py | 5 +- pythainlp/tokenize/tcc.py | 13 +- pythainlp/tokenize/tcc_p.py | 13 +- pythainlp/tokenize/thai2fit.py | 3 +- pythainlp/tokenize/thaisumcut.py | 10 +- pythainlp/tokenize/tltk.py | 13 +- pythainlp/tokenize/wtsplit.py | 8 +- pythainlp/tools/__init__.py | 1 - pythainlp/tools/core.py | 4 +- pythainlp/tools/misspell.py | 5 +- pythainlp/tools/path.py | 6 +- pythainlp/translate/__init__.py | 1 - pythainlp/translate/core.py | 19 +- pythainlp/translate/en_th.py | 12 +- pythainlp/translate/small100.py | 19 +- pythainlp/translate/th_fr.py | 4 +- pythainlp/translate/tokenization_small100.py | 121 +- pythainlp/translate/word2word_translate.py | 138 +- pythainlp/translate/zh_th.py | 4 +- pythainlp/transliterate/__init__.py | 1 - pythainlp/transliterate/core.py | 6 +- pythainlp/transliterate/ipa.py | 8 +- pythainlp/transliterate/iso_11940.py | 4 +- pythainlp/transliterate/lookup.py | 7 +- pythainlp/transliterate/pyicu.py | 4 +- pythainlp/transliterate/royin.py | 78 +- pythainlp/transliterate/spoonerism.py | 3 +- pythainlp/transliterate/thai2rom.py | 7 +- pythainlp/transliterate/thai2rom_onnx.py | 4 +- pythainlp/transliterate/thaig2p.py | 9 +- pythainlp/transliterate/thaig2p_v2.py | 9 +- pythainlp/transliterate/tltk.py | 7 +- pythainlp/transliterate/umt5_thaig2p.py | 11 +- pythainlp/transliterate/w2p.py | 13 +- pythainlp/transliterate/wunsen.py | 8 +- pythainlp/ulmfit/__init__.py | 1 - pythainlp/ulmfit/core.py | 10 +- pythainlp/ulmfit/preprocess.py | 18 +- pythainlp/ulmfit/tokenizer.py | 9 +- pythainlp/util/__init__.py | 37 +- pythainlp/util/abbreviation.py | 10 +- pythainlp/util/collate.py | 8 +- pythainlp/util/date.py | 55 +- pythainlp/util/digitconv.py | 7 +- pythainlp/util/emojiconv.py | 2 + pythainlp/util/encoding.py | 5 +- pythainlp/util/keyboard.py | 3 +- pythainlp/util/keywords.py | 8 +- pythainlp/util/lcs.py | 4 +- pythainlp/util/morse.py | 3 +- pythainlp/util/normalize.py | 8 +- pythainlp/util/numtoword.py | 5 +- pythainlp/util/phoneme.py | 7 +- pythainlp/util/pronounce.py | 33 +- .../util/remove_trailing_repeat_consonants.py | 11 +- pythainlp/util/spell_words.py | 15 +- pythainlp/util/strftime.py | 17 +- pythainlp/util/syllable.py | 3 +- pythainlp/util/thai.py | 10 +- pythainlp/util/thai_lunar_date.py | 12 +- pythainlp/util/thaiwordcheck.py | 3 +- pythainlp/util/time.py | 18 +- pythainlp/util/trie.py | 12 +- pythainlp/util/wordtonum.py | 19 +- pythainlp/wangchanberta/__init__.py | 1 - pythainlp/wangchanberta/core.py | 10 +- pythainlp/word_vector/__init__.py | 2 +- pythainlp/word_vector/core.py | 31 +- pythainlp/wsd/__init__.py | 2 +- pythainlp/wsd/core.py | 5 +- setup.py | 1 - 191 files changed, 2128 insertions(+), 1890 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d84632ba3..b01c8fd28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.ruff] line-length = 79 indent-width = 4 -target-version = "py310" +target-version = "py39" [tool.ruff.format] quote-style = "double" diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index 7e0a61544..bd43ddd25 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -1,64 +1,63 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -__version__ = "5.2.0" - -thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars - -thai_vowels = ( - "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37" - + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47" -) # 20 -thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 -thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4 -thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 -thai_below_vowels = "\u0e38\u0e39" # 2 - -thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4 - -# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: -# These signs can be part of a word -thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars - -# Any Thai character that can be part of a word -thai_letters = "".join( - [thai_consonants, thai_vowels, thai_tonemarks, thai_signs] -) # 74 - -# Fongman, Angkhankhu, Khomut: -# These characters are section markers -thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars - -thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10 -thai_symbols = "\u0e3f" # Thai Bath ฿ - -# All Thai characters that are presented in Unicode -thai_characters = "".join( - [thai_letters, thai_punctuations, thai_digits, thai_symbols] -) -# Thai pangram by Sungsit Sawaiwan -# CC BY-SA License -# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html -thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ -๏ จับฅอคนบั่นต้อง อาญา -ขุดฆ่าโคตรฃัตติยา ซ่านม้วย -ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ -สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ -๏ กตัญญูไป่พร้อม ปฐมฌาน -เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ -ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ -โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ -๑๒ กรกฎาคม ๒๕๕๘""" - -from pythainlp.soundex import soundex -from pythainlp.spell import correct, spell -from pythainlp.tag import pos_tag -from pythainlp.tokenize import ( - Tokenizer, - sent_tokenize, - subword_tokenize, - word_tokenize, -) -from pythainlp.transliterate import romanize, transliterate -from pythainlp.util import collate, thai_strftime +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +__version__ = "5.2.0" + +thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars + +thai_vowels = ( + "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37" + + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47" +) # 20 +thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 +thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4 +thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 +thai_below_vowels = "\u0e38\u0e39" # 2 + +thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4 + +# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: +# These signs can be part of a word +thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars + +# Any Thai character that can be part of a word +thai_letters = "".join( + [thai_consonants, thai_vowels, thai_tonemarks, thai_signs] +) # 74 + +# Fongman, Angkhankhu, Khomut: +# These characters are section markers +thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars + +thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10 +thai_symbols = "\u0e3f" # Thai Bath ฿ + +# All Thai characters that are presented in Unicode +thai_characters = "".join( + [thai_letters, thai_punctuations, thai_digits, thai_symbols] +) +# Thai pangram by Sungsit Sawaiwan +# CC BY-SA License +# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html +thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ +๏ จับฅอคนบั่นต้อง อาญา +ขุดฆ่าโคตรฃัตติยา ซ่านม้วย +ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ +สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ +๏ กตัญญูไป่พร้อม ปฐมฌาน +เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ +ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ +โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ +๑๒ กรกฎาคม ๒๕๕๘""" + +from pythainlp.soundex import soundex +from pythainlp.spell import correct, spell +from pythainlp.tag import pos_tag +from pythainlp.tokenize import ( + Tokenizer, + sent_tokenize, + subword_tokenize, + word_tokenize, +) +from pythainlp.transliterate import romanize, transliterate +from pythainlp.util import collate, thai_strftime diff --git a/pythainlp/__main__.py b/pythainlp/__main__.py index cb9d53a45..91a280366 100644 --- a/pythainlp/__main__.py +++ b/pythainlp/__main__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py index 4a30d29c6..f8d7b41ae 100644 --- a/pythainlp/ancient/__init__.py +++ b/pythainlp/ancient/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 8769993fe..15e41e5f3 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_orst_words diff --git a/pythainlp/ancient/currency.py b/pythainlp/ancient/currency.py index bdf685e96..0725d780d 100644 --- a/pythainlp/ancient/currency.py +++ b/pythainlp/ancient/currency.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + def convert_currency(value: float, from_unit: str) -> dict: """ @@ -44,14 +45,14 @@ def convert_currency(value: float, from_unit: str) -> dict: # } """ conversion_factors_to_att = { - 'เบี้ย': 1, - 'อัฐ': 100, # 1 อัฐ = 100 เบี้ย - 'ไพ': 2 * 100, # 1 ไพ = 2 อัฐ - 'เฟื้อง': 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ - 'สลึง': 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง - 'บาท': 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง - 'ตำลึง': 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท - 'ชั่ง': 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง + "เบี้ย": 1, + "อัฐ": 100, # 1 อัฐ = 100 เบี้ย + "ไพ": 2 * 100, # 1 ไพ = 2 อัฐ + "เฟื้อง": 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ + "สลึง": 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง + "บาท": 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง + "ตำลึง": 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท + "ชั่ง": 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง } if from_unit not in conversion_factors_to_att: diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py index c82c17995..8935333b6 100644 --- a/pythainlp/augment/__init__.py +++ b/pythainlp/augment/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py index 1707936a4..f4a69dee5 100644 --- a/pythainlp/augment/lm/__init__.py +++ b/pythainlp/augment/lm/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index a91af4b52..6bf31b237 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import itertools -from typing import List, Tuple from gensim.models.fasttext import FastText as FastText_gensim from gensim.models.keyedvectors import KeyedVectors @@ -30,7 +30,7 @@ def __init__(self, model_path: str): self.model = FastText_gensim.load(model_path) self.dict_wv = list(self.model.key_to_index.keys()) - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: """ Thai text tokenization for fastText @@ -41,7 +41,7 @@ def tokenize(self, text: str) -> List[str]: """ return word_tokenize(text, engine="icu") - def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: + def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: """ :param str sent: text of sentence :param float p: probability @@ -61,7 +61,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: + ) -> list[tuple[str]]: """ Text Augment from fastText diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index fa7e9a8d3..73bdddf5a 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import random import re -from typing import List from pythainlp.phayathaibert.core import ThaiTextProcessor @@ -57,7 +56,7 @@ def generate( def augment( self, text: str, num_augs: int = 3, sample: bool = False - ) -> List[str]: + ) -> list[str]: """ Text augmentation from PhayaThaiBERT diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index 962e5f7cf..64702d4fe 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - -from typing import List +from __future__ import annotations from transformers import ( CamembertTokenizer, @@ -52,7 +50,7 @@ def generate(self, sentence: str, num_replace_tokens: int = 3): masked_text = self.input_text return self.sent2 - def augment(self, sentence: str, num_replace_tokens: int = 3) -> List[str]: + def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]: """ Text augmentation from WangchanBERTa diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py index 08786229f..8a51b2593 100644 --- a/pythainlp/augment/word2vec/__init__.py +++ b/pythainlp/augment/word2vec/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index 635553f40..8d0a1a54c 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug @@ -22,7 +21,7 @@ def __init__(self, lang: str = "th", vs: int = 100000, dim: int = 300): self.model = self.bpemb_temp.emb self.load_w2v() - def tokenizer(self, text: str) -> List[str]: + def tokenizer(self, text: str) -> list[str]: """ :param str text: Thai text :rtype: List[str] @@ -39,7 +38,7 @@ def load_w2v(self): def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: + ) -> list[tuple[str]]: """ Text Augment using word2vec from BPEmb diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py index 345bf9097..72559ae43 100644 --- a/pythainlp/augment/word2vec/core.py +++ b/pythainlp/augment/word2vec/core.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import itertools -from typing import List, Tuple class Word2VecAug: @@ -28,7 +28,7 @@ def __init__( self.model = model self.dict_wv = list(self.model.key_to_index.keys()) - def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: + def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: """ :param str sent: text of sentence :param float p: probability @@ -48,7 +48,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: + ) -> list[tuple[str]]: """ :param str sentence: text of sentence :param int n_sent: maximum number of synonymous sentences diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 3ae90e24b..fb73b76e6 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path @@ -21,7 +20,7 @@ def __init__(self): self.ltw2v_wv = get_corpus_path("ltw2v") self.load_w2v() - def tokenizer(self, text: str) -> List[str]: + def tokenizer(self, text: str) -> list[str]: """ :param str text: Thai text :rtype: List[str] @@ -36,7 +35,7 @@ def load_w2v(self): # insert substitute def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: + ) -> list[tuple[str]]: """ Text Augment using word2vec from Thai2Fit diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 16aae20c2..5d0326852 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path @@ -21,7 +20,7 @@ def __init__(self): self.thai2fit_wv = get_corpus_path("thai2fit_wv") self.load_w2v() - def tokenizer(self, text: str) -> List[str]: + def tokenizer(self, text: str) -> list[str]: """ :param str text: Thai text :rtype: List[str] @@ -37,7 +36,7 @@ def load_w2v(self): def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> List[Tuple[str]]: + ) -> list[tuple[str]]: """ Text Augment using word2vec from Thai2Fit diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index ea2ab3d70..074e4a60c 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -1,10 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l """ + +from __future__ import annotations + __all__ = [ "WordNetAug", "postype2wordnet", @@ -12,7 +14,6 @@ import itertools from collections import OrderedDict -from typing import List from nltk.corpus import wordnet as wn @@ -126,7 +127,7 @@ def __init__(self): def find_synonyms( self, word: str, pos: str = None, postag_corpus: str = "orchid" - ) -> List[str]: + ) -> list[str]: """ Find synonyms using wordnet @@ -162,7 +163,7 @@ def augment( max_syn_sent: int = 6, postag: bool = True, postag_corpus: str = "orchid", - ) -> List[List[str]]: + ) -> list[list[str]]: """ Text Augment using wordnet diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index 66a012ffe..cc25d2705 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 35bd0ae6c..ce9991cfc 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import re import sys -from typing import List, Tuple import numpy as np import pandas as pd @@ -18,13 +17,13 @@ ) # regex for removing repeated separators, i.e. |||| -MULTIPLE_SEPS_RX = re.compile("{sep}+".format(sep=re.escape(SEPARATOR))) +MULTIPLE_SEPS_RX = re.compile(f"{re.escape(SEPARATOR)}+") # regex for removing tags, i.e. , TAG_RX = re.compile(r"<\/?[A-Z]+>") # regex for removing trailing separators, i.e. a|dog| -> a|dog -TAILING_SEP_RX = re.compile("{sep}$".format(sep=re.escape(SEPARATOR))) +TAILING_SEP_RX = re.compile(f"{re.escape(SEPARATOR)}$") def _f1(precision: float, recall: float) -> float: @@ -68,7 +67,7 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: return dict(items) -def benchmark(ref_samples: List[str], samples: List[str]) -> pd.DataFrame: +def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: """ Performance benchmarking for samples. @@ -184,9 +183,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: correctly_tokenised_words = np.sum(tokenization_indicators) - tokenization_indicators = list( - map(str, tokenization_indicators) - ) + tokenization_indicators = list(map(str, tokenization_indicators)) return { "char_level": { @@ -256,9 +253,9 @@ def _find_word_boundaries(bin_reps) -> list: def _find_words_correctly_tokenised( - ref_boundaries: List[Tuple[int, int]], - predicted_boundaries: List[Tuple[int, int]], -) -> Tuple[int]: + ref_boundaries: list[tuple[int, int]], + predicted_boundaries: list[tuple[int, int]], +) -> tuple[int]: """ Find whether each word is correctly tokenized. diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py index 28698b252..2459ff1cd 100644 --- a/pythainlp/chat/__init__.py +++ b/pythainlp/chat/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 5f198ec84..3659b9b53 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import torch @@ -69,7 +70,7 @@ def chat(self, text: str) -> str: import torch chatbot = ChatBotModel() - chatbot.load_model(device="cpu",torch_dtype=torch.bfloat16) + chatbot.load_model(device="cpu", torch_dtype=torch.bfloat16) print(chatbot.chat("สวัสดี")) # output: ยินดีที่ได้รู้จัก diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py index af282e47f..ae305872b 100644 --- a/pythainlp/classify/__init__.py +++ b/pythainlp/classify/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index e2a7fbe57..b3ccac213 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import gzip import json -from typing import List, Tuple import numpy as np @@ -20,7 +19,11 @@ class GzipModel: :param str model_path: Path for loading model (if you saved the model) """ - def __init__(self, training_data: List[Tuple[str, str]] = None, model_path: str = None): + def __init__( + self, + training_data: list[tuple[str, str]] = None, + model_path: str = None, + ): if model_path is not None: self.load(model_path) else: @@ -47,7 +50,7 @@ def predict(self, x1: str, k: int = 1) -> str: from pythainlp.classify import GzipModel - training_data = [ + training_data = [ ("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"), ("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"), ("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"), @@ -56,7 +59,7 @@ def predict(self, x1: str, k: int = 1) -> str: ("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"), ("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"), ("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"), - ("นี่เป็นบทความหนึ่ง", "Neutral") + ("นี่เป็นบทความหนึ่ง", "Neutral"), ] model = GzipModel(training_data) print(model.predict("ฉันดีใจ", k=1)) @@ -85,13 +88,17 @@ def save(self, path: str): :param str path: path for save model """ with open(path, "w") as f: - json.dump({ - "training_data": self.training_data.tolist(), - "Cx2_list": self.Cx2_list - }, f, ensure_ascii=False) + json.dump( + { + "training_data": self.training_data.tolist(), + "Cx2_list": self.Cx2_list, + }, + f, + ensure_ascii=False, + ) def load(self, path: str): - with open(path, "r") as f: + with open(path) as f: data = json.load(f) self.Cx2_list = data["Cx2_list"] self.training_data = np.array(data["training_data"]) diff --git a/pythainlp/cli/__init__.py b/pythainlp/cli/__init__.py index 8d6649e6a..2bf309c52 100644 --- a/pythainlp/cli/__init__.py +++ b/pythainlp/cli/__init__.py @@ -1,19 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """Command line helpers.""" +from __future__ import annotations + import io import sys from argparse import ArgumentError, ArgumentParser -from pythainlp.cli import data, tokenize, soundex, tag, benchmark, misspell + +from pythainlp.cli import benchmark, data, misspell, soundex, tag, tokenize sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8") # a command should start with a verb when possible -COMMANDS = sorted(["data", "soundex", "tag", "tokenize", "benchmark", "misspell"]) +COMMANDS = sorted( + ["data", "soundex", "tag", "tokenize", "benchmark", "misspell"] +) CLI_NAME = "thainlp" @@ -37,6 +41,7 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None: parser.print_help() raise ArgumentError(None, "No command provided.") + if __name__ == "__main__": # Create a simple mapping from command name to the imported module COMMAND_MAP = { @@ -54,6 +59,9 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None: COMMAND_MAP[command].run() else: if len(sys.argv) < 2: - print(f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}", file=sys.stderr) + print( + f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}", + file=sys.stderr, + ) else: - print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr) \ No newline at end of file + print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr) diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index 68d59c10d..e342ff9fd 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -8,13 +7,12 @@ import json import os - from pythainlp import cli from pythainlp.tools import safe_print def _read_file(path): - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: lines = map(lambda r: r.strip(), f.readlines()) return list(lines) @@ -77,9 +75,9 @@ def __init__(self, name, argv): actual = _read_file(args.input_file) expected = _read_file(args.test_file) - assert len(actual) == len( - expected - ), "Input and test files do not have the same number of samples" + assert len(actual) == len(expected), ( + "Input and test files do not have the same number of samples" + ) safe_print( "Benchmarking %s against %s with %d samples in total" @@ -88,6 +86,7 @@ def __init__(self, name, argv): try: import yaml + from pythainlp.benchmarks import word_tokenization except ImportError: raise ImportError( diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index 0c5c704ed..37cba0b93 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/cli/misspell.py b/pythainlp/cli/misspell.py index 11077282a..66b4601e4 100644 --- a/pythainlp/cli/misspell.py +++ b/pythainlp/cli/misspell.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -54,7 +53,7 @@ def __init__(self, argv): if args.seed is not None: random.seed(args.seed) - with open(args.file, "r", encoding="utf-8") as f: + with open(args.file, encoding="utf-8") as f: lines = f.readlines() misspelled_lines = [ diff --git a/pythainlp/cli/soundex.py b/pythainlp/cli/soundex.py index 60b0a1808..d6f97f0db 100644 --- a/pythainlp/cli/soundex.py +++ b/pythainlp/cli/soundex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index 6d4632cd0..bd8cabd47 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 7a12914d2..156879b10 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py index 24c11cdcd..366fff8e9 100644 --- a/pythainlp/coref/__init__.py +++ b/pythainlp/coref/__init__.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ PyThaiNLP Coreference Resolution """ + __all__ = ["coreference_resolution"] from pythainlp.coref.core import coreference_resolution diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index 2b996d42a..32c537b3b 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations import spacy @@ -30,7 +29,7 @@ def _to_json(self, _predict): "clusters": _predict.get_clusters(as_strings=False), } - def predict(self, texts: List[str]) -> List[dict]: + def predict(self, texts: list[str]) -> list[dict]: return [ self._to_json(pred) for pred in self.model.predict(texts=texts) ] diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 6dcb3ec75..4ffe50f35 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations _MODEL = None def coreference_resolution( - texts: List[str], model_name: str = "han-coref-v1.0", device: str = "cpu" + texts: list[str], model_name: str = "han-coref-v1.0", device: str = "cpu" ): """ Coreference Resolution diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py index 45f0e3c62..4fd6d2f6e 100644 --- a/pythainlp/coref/han_coref.py +++ b/pythainlp/coref/han_coref.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import spacy from pythainlp.coref._fastcoref import FastCoref diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 424a25976..ad212dfa4 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,6 +8,8 @@ Including download manager. """ +from __future__ import annotations + __all__ = [ "corpus_db_path", "corpus_db_url", @@ -91,20 +92,6 @@ def corpus_db_path() -> str: return _CORPUS_DB_PATH -from pythainlp.corpus.core import ( - download, - get_corpus, - get_corpus_as_is, - get_corpus_db, - get_corpus_db_detail, - get_corpus_default_db, - get_corpus_path, - get_path_folder_corpus, - make_safe_directory_name, - get_hf_hub, - path_pythainlp_corpus, - remove, -) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.common import ( countries, find_synonyms, @@ -122,6 +109,20 @@ def corpus_db_path() -> str: thai_words, thai_wsd_dict, ) +from pythainlp.corpus.core import ( + download, + get_corpus, + get_corpus_as_is, + get_corpus_db, + get_corpus_db_detail, + get_corpus_default_db, + get_corpus_path, + get_hf_hub, + get_path_folder_corpus, + make_safe_directory_name, + path_pythainlp_corpus, + remove, +) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.icu import thai_icu_words from pythainlp.corpus.volubilis import thai_volubilis_words from pythainlp.corpus.wikipedia import thai_wikipedia_titles diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 6e23a7da4..285d02838 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,6 +6,8 @@ Common lists of words. """ +from __future__ import annotations + import ast __all__ = [ @@ -26,45 +27,44 @@ "thai_wsd_dict", ] -from typing import FrozenSet, List, Union from pythainlp.corpus import get_corpus, get_corpus_as_is, get_corpus_path from pythainlp.tools import warn_deprecation -_THAI_COUNTRIES: FrozenSet[str] = frozenset() +_THAI_COUNTRIES: frozenset[str] = frozenset() _THAI_COUNTRIES_FILENAME = "countries_th.txt" -_THAI_THAILAND_PROVINCES: FrozenSet[str] = frozenset() -_THAI_THAILAND_PROVINCES_DETAILS: List[dict] = [] +_THAI_THAILAND_PROVINCES: frozenset[str] = frozenset() +_THAI_THAILAND_PROVINCES_DETAILS: list[dict] = [] _THAI_THAILAND_PROVINCES_FILENAME = "thailand_provinces_th.csv" -_THAI_SYLLABLES: FrozenSet[str] = frozenset() +_THAI_SYLLABLES: frozenset[str] = frozenset() _THAI_SYLLABLES_FILENAME = "syllables_th.txt" -_THAI_WORDS: FrozenSet[str] = frozenset() +_THAI_WORDS: frozenset[str] = frozenset() _THAI_WORDS_FILENAME = "words_th.txt" -_THAI_STOPWORDS: FrozenSet[str] = frozenset() +_THAI_STOPWORDS: frozenset[str] = frozenset() _THAI_STOPWORDS_FILENAME = "stopwords_th.txt" -_THAI_NEGATIONS: FrozenSet[str] = frozenset() +_THAI_NEGATIONS: frozenset[str] = frozenset() _THAI_NEGATIONS_FILENAME = "negations_th.txt" -_THAI_FAMLIY_NAMES: FrozenSet[str] = frozenset() +_THAI_FAMLIY_NAMES: frozenset[str] = frozenset() _THAI_FAMLIY_NAMES_FILENAME = "family_names_th.txt" -_THAI_FEMALE_NAMES: FrozenSet[str] = frozenset() +_THAI_FEMALE_NAMES: frozenset[str] = frozenset() _THAI_FEMALE_NAMES_FILENAME = "person_names_female_th.txt" -_THAI_MALE_NAMES: FrozenSet[str] = frozenset() +_THAI_MALE_NAMES: frozenset[str] = frozenset() _THAI_MALE_NAMES_FILENAME = "person_names_male_th.txt" -_THAI_ORST_WORDS: FrozenSet[str] = frozenset() +_THAI_ORST_WORDS: frozenset[str] = frozenset() _THAI_DICT: dict[str, list] = {} _THAI_WSD_DICT: dict[str, list] = {} _THAI_SYNONYMS: dict[str, list] = {} -def countries() -> FrozenSet[str]: +def countries() -> frozenset[str]: """ Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", "แอลจีเรีย", and "ลาว". @@ -81,7 +81,7 @@ def countries() -> FrozenSet[str]: return _THAI_COUNTRIES -def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]: +def provinces(details: bool = False) -> frozenset[str] | list[dict]: """ Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". @@ -124,7 +124,7 @@ def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]: return _THAI_THAILAND_PROVINCES -def thai_syllables() -> FrozenSet[str]: +def thai_syllables() -> frozenset[str]: """ Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", "โมน", "โม่ง", "กา", "ก่า", and, "ก้า". @@ -142,7 +142,7 @@ def thai_syllables() -> FrozenSet[str]: return _THAI_SYLLABLES -def thai_words() -> FrozenSet[str]: +def thai_words() -> frozenset[str]: """ Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", and "พิษภัย". \n(See: `dev/pythainlp/corpus/words_th.txt\ @@ -158,7 +158,7 @@ def thai_words() -> FrozenSet[str]: return _THAI_WORDS -def thai_orst_words() -> FrozenSet[str]: +def thai_orst_words() -> frozenset[str]: """ Return a frozenset of Thai words from Royal Society of Thailand \n(See: `dev/pythainlp/corpus/thai_orst_words.txt\ @@ -174,7 +174,7 @@ def thai_orst_words() -> FrozenSet[str]: return _THAI_ORST_WORDS -def thai_stopwords() -> FrozenSet[str]: +def thai_stopwords() -> frozenset[str]: """ Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", "การ", and "ประการหนึ่ง". \n(See: `dev/pythainlp/corpus/stopwords_th.txt\ @@ -197,7 +197,7 @@ def thai_stopwords() -> FrozenSet[str]: return _THAI_STOPWORDS -def thai_negations() -> FrozenSet[str]: +def thai_negations() -> frozenset[str]: """ Return a frozenset of Thai negation words including "ไม่" and "แต่". \n(See: `dev/pythainlp/corpus/negations_th.txt\ @@ -213,7 +213,7 @@ def thai_negations() -> FrozenSet[str]: return _THAI_NEGATIONS -def thai_family_names() -> FrozenSet[str]: +def thai_family_names() -> frozenset[str]: """ Return a frozenset of Thai family names \n(See: `dev/pythainlp/corpus/family_names_th.txt\ @@ -229,7 +229,7 @@ def thai_family_names() -> FrozenSet[str]: return _THAI_FAMLIY_NAMES -def thai_female_names() -> FrozenSet[str]: +def thai_female_names() -> frozenset[str]: """ Return a frozenset of Thai female names \n(See: `dev/pythainlp/corpus/person_names_female_th.txt\ @@ -245,7 +245,7 @@ def thai_female_names() -> FrozenSet[str]: return _THAI_FEMALE_NAMES -def thai_male_names() -> FrozenSet[str]: +def thai_male_names() -> frozenset[str]: """ Return a frozenset of Thai male names \n(See: `dev/pythainlp/corpus/person_names_male_th.txt\ @@ -360,7 +360,7 @@ def thai_synonym() -> dict: return thai_synonyms() -def find_synonyms(word: str) -> List[str]: +def find_synonyms(word: str) -> list[str]: """ Find synonyms diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index d3372bfe2..ecaa33197 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,10 +5,11 @@ Corpus related functions. """ +from __future__ import annotations + import json import os import re -from typing import Union from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path @@ -45,7 +45,7 @@ def get_corpus_db_detail(name: str, version: str = "") -> dict: :return: details about corpus :rtype: dict """ - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: local_db = json.load(f) if not version: @@ -133,7 +133,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: """ path = path_pythainlp_corpus(filename) lines = [] - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = fh.read().splitlines() if not comments: @@ -173,13 +173,13 @@ def get_corpus_as_is(filename: str) -> list: """ path = path_pythainlp_corpus(filename) lines = [] - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = fh.read().splitlines() return lines -def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]: +def get_corpus_default_db(name: str, version: str = "") -> str | None: """ Get model path from default_db.json @@ -211,7 +211,7 @@ def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]: def get_corpus_path( name: str, version: str = "", force: bool = False -) -> Union[str, None]: +) -> str | None: """ Get corpus path. @@ -252,9 +252,8 @@ def get_corpus_path( print(get_corpus_path('wiki_lm_lstm')) # output: /root/pythainlp-data/thwiki_model_lstm.pth """ - from typing import Dict - CUSTOMIZE: Dict[str, str] = { + CUSTOMIZE: dict[str, str] = { # "the corpus name":"path" } if name in list(CUSTOMIZE): @@ -450,7 +449,7 @@ def download( # check if corpus is available if name in corpus_db: - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: local_db = json.load(f) corpus = corpus_db[name] @@ -525,9 +524,7 @@ def download( # This awkward behavior is for backward-compatibility with # database files generated previously using TinyDB if local_db["_default"]: - corpus_no = ( - max((int(no) for no in local_db["_default"])) + 1 - ) + corpus_no = max(int(no) for no in local_db["_default"]) + 1 else: corpus_no = 1 local_db["_default"][str(corpus_no)] = { @@ -588,7 +585,7 @@ def remove(name: str) -> bool: if _CHECK_MODE == "1": print("PyThaiNLP is read-only mode. It can't download.") return False - with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: + with open(corpus_db_path(), encoding="utf-8-sig") as f: db = json.load(f) data = [ corpus for corpus in db["_default"].values() if corpus["name"] == name @@ -617,7 +614,7 @@ def get_path_folder_corpus(name, version, *path): return os.path.join(get_corpus_path(name, version), *path) -def make_safe_directory_name(name:str) -> str: +def make_safe_directory_name(name: str) -> str: """ Make safe directory name @@ -626,17 +623,40 @@ def make_safe_directory_name(name:str) -> str: :rtype: str """ # Replace invalid characters with an underscore - safe_name = re.sub(r'[<>:"/\\|?*]', '_', name) + safe_name = re.sub(r'[<>:"/\\|?*]', "_", name) # Remove leading/trailing spaces or periods (especially important for Windows) - safe_name = safe_name.strip(' .') + safe_name = safe_name.strip(" .") # Prevent names that are reserved on Windows - reserved_names = ['CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'] + reserved_names = [ + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", + ] if safe_name.upper() in reserved_names: - safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict + safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict return safe_name -def get_hf_hub(repo_id:str, filename: str=None) -> str: +def get_hf_hub(repo_id: str, filename: str = None) -> str: """ HuggingFace Hub in :mod:`pythainlp` data directory. @@ -653,19 +673,16 @@ def get_hf_hub(repo_id:str, filename: str=None) -> str: Please installing the package via 'pip install huggingface-hub'. """) except Exception as e: - raise Exception(f"An unexpected error occurred: {e}") + raise RuntimeError(f"An unexpected error occurred: {e}") from e hf_root = get_full_data_path("hf_models") name_dir = make_safe_directory_name(repo_id) root_project = os.path.join(hf_root, name_dir) - if filename!=None: + if filename is not None: output_path = hf_hub_download( - repo_id=repo_id, - filename=filename, - local_dir=root_project + repo_id=repo_id, filename=filename, local_dir=root_project ) else: output_path = snapshot_download( - repo_id=repo_id, - local_dir=root_project + repo_id=repo_id, local_dir=root_project ) return output_path diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py index 3854c598f..ada98f81a 100644 --- a/pythainlp/corpus/icu.py +++ b/pythainlp/corpus/icu.py @@ -1,18 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Provides an optional word list from International Components for Unicode (ICU) dictionary. """ -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus _THAI_ICU_FILENAME = "icubrk_th.txt" -def thai_icu_words() -> FrozenSet[str]: +def thai_icu_words() -> frozenset[str]: """ Return a frozenset of words from the Thai dictionary for BreakIterator of the International Components for Unicode (ICU). diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index 6e5b5ba1d..0a0b540fc 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,17 +8,18 @@ https://web.facebook.com/groups/colab.thailand/permalink/1524070061101680/ """ +from __future__ import annotations + __all__ = ["word_freqs", "unigram_word_freqs"] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus_path _OSCAR_FILENAME = "oscar_icu" -def word_freqs() -> List[Tuple[str, int]]: +def word_freqs() -> list[tuple[str, int]]: """ Get word frequency from OSCAR Corpus (words tokenized using ICU) """ @@ -29,7 +29,7 @@ def word_freqs() -> List[Tuple[str, int]]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as f: + with open(path, encoding="utf-8-sig") as f: lines = list(f.readlines()) del lines[0] for line in lines: @@ -53,7 +53,7 @@ def unigram_word_freqs() -> dict[str, int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: lines = list(fh.readlines()) del lines[0] for i in lines: diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 8f8a61948..4a163a1f2 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,6 +9,8 @@ Zenodo. https://doi.org/10.5281/zenodo.6716672 """ +from __future__ import annotations + __all__ = [ "get_transliteration_dict", "TRANSLITERATE_EN", @@ -43,7 +44,7 @@ def get_transliteration_dict() -> defaultdict: lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} ) try: - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: # assume that the first row contains column names, so skip it. for line in f.readlines()[1:]: stripped = line.strip() diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 36a3e0311..7aef7b5c2 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project. # SPDX-License-Identifier: Apache-2.0 """ Thai National Corpus word frequency """ +from __future__ import annotations + __all__ = [ "bigram_word_freqs", "trigram_word_freqs", @@ -13,7 +14,6 @@ ] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus, get_corpus_path @@ -22,7 +22,7 @@ _TRIGRAM_CORPUS_NAME = "tnc_trigram_word_freqs" -def word_freqs() -> List[Tuple[str, int]]: +def word_freqs() -> list[tuple[str, int]]: """ Get word frequency from Thai National Corpus (TNC) \n(See: `dev/pythainlp/corpus/tnc_freq.txt\ @@ -54,7 +54,7 @@ def unigram_word_freqs() -> dict[str, int]: return freqs -def bigram_word_freqs() -> dict[Tuple[str, str], int]: +def bigram_word_freqs() -> dict[tuple[str, str], int]: """ Get bigram word frequency from Thai National Corpus (TNC) """ @@ -64,7 +64,7 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: for i in fh.readlines(): temp = i.strip().split(" ") freqs[(temp[0], temp[1])] = int(temp[-1]) @@ -72,7 +72,7 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]: return freqs -def trigram_word_freqs() -> dict[Tuple[str, str, str], int]: +def trigram_word_freqs() -> dict[tuple[str, str, str], int]: """ Get trigram word frequency from Thai National Corpus (TNC) """ @@ -82,7 +82,7 @@ def trigram_word_freqs() -> dict[Tuple[str, str, str], int]: return freqs path = str(path) - with open(path, "r", encoding="utf-8-sig") as fh: + with open(path, encoding="utf-8-sig") as fh: for i in fh.readlines(): temp = i.strip().split(" ") freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index c6e470f74..a32051259 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,17 +8,18 @@ https://www.facebook.com/photo.php?fbid=363640477387469&set=gm.434330506948445&type=3&permPage=1 """ +from __future__ import annotations + __all__ = ["word_freqs", "unigram_word_freqs"] from collections import defaultdict -from typing import List, Tuple from pythainlp.corpus import get_corpus _UNIGRAM_FILENAME = "ttc_freq.txt" -def word_freqs() -> List[Tuple[str, int]]: +def word_freqs() -> list[tuple[str, int]]: """ Get word frequency from Thai Textbook Corpus (TTC) \n(See: `dev/pythainlp/corpus/ttc_freq.txt\ diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index b33b3ef11..d4e86c156 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,15 +12,17 @@ `_ """ +from __future__ import annotations + from collections import Counter -from typing import Callable, Iterable, Iterator, List, Set, Tuple +from collections.abc import Callable, Iterable, Iterator from pythainlp.corpus import thai_words from pythainlp.tokenize import newmm from pythainlp.util import Trie -def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]: +def index_pairs(words: list[str]) -> Iterator[tuple[int, int]]: """ Return beginning and ending indexes of word pairs """ @@ -32,9 +33,9 @@ def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]: def find_badwords( - tokenize: Callable[[str], List[str]], + tokenize: Callable[[str], list[str]], training_data: Iterable[Iterable[str]], -) -> Set[str]: +) -> set[str]: """ Find words that do not work well with the `tokenize` function for the provided `training_data`. @@ -68,10 +69,10 @@ def find_badwords( def revise_wordset( - tokenize: Callable[[str], List[str]], + tokenize: Callable[[str], list[str]], orig_words: Iterable[str], training_data: Iterable[Iterable[str]], -) -> Set[str]: +) -> set[str]: """ Revise a set of words that could improve tokenization performance of a dictionary-based `tokenize` function. @@ -119,7 +120,7 @@ def revise_wordset( def revise_newmm_default_wordset( training_data: Iterable[Iterable[str]], -) -> Set[str]: +) -> set[str]: """ Revise a set of word that could improve tokenization performance of `pythainlp.tokenize.newmm`, a dictionary-based tokenizer and a default diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py index de65e1e1f..8f21a5dba 100644 --- a/pythainlp/corpus/volubilis.py +++ b/pythainlp/corpus/volubilis.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Provides an optional word list from the Volubilis dictionary. """ -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus @@ -13,7 +13,7 @@ _VOLUBILIS_FILENAME = "volubilis_words_th.txt" -def thai_volubilis_words() -> FrozenSet[str]: +def thai_volubilis_words() -> frozenset[str]: """ Return a frozenset of Thai words from the Volubilis dictionary diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py index 2c0ce7c2e..a036f4568 100644 --- a/pythainlp/corpus/wikipedia.py +++ b/pythainlp/corpus/wikipedia.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Provides an optional word list from Thai Wikipedia titles. """ -from typing import FrozenSet + +from __future__ import annotations from pythainlp.corpus.common import get_corpus @@ -13,7 +13,7 @@ _WIKIPEDIA_TITLES_FILENAME = "wikipedia_titles_th.txt" -def thai_wikipedia_titles() -> FrozenSet[str]: +def thai_wikipedia_titles() -> frozenset[str]: """ Return a frozenset of words from Thai Wikipedia titles corpus. They are mostly nouns and noun phrases, @@ -31,6 +31,8 @@ def thai_wikipedia_titles() -> FrozenSet[str]: """ global _WIKIPEDIA_TITLES if not _WIKIPEDIA_TITLES: - _WIKIPEDIA_TITLES = get_corpus(_WIKIPEDIA_TITLES_FILENAME, comments=False) + _WIKIPEDIA_TITLES = get_corpus( + _WIKIPEDIA_TITLES_FILENAME, comments=False + ) return _WIKIPEDIA_TITLES diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py index 658314b4a..6ea2a0097 100644 --- a/pythainlp/corpus/wordnet.py +++ b/pythainlp/corpus/wordnet.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,6 +10,9 @@ For more on usage, see NLTK Howto: https://www.nltk.org/howto/wordnet.html """ + +from __future__ import annotations + import nltk try: @@ -88,7 +90,7 @@ def synset(name_synsets): >>> from pythainlp.corpus.wordnet import synset >>> - >>> difficult = synset('difficult.a.01') + >>> difficult = synset("difficult.a.01") >>> difficult Synset('difficult.a.01') >>> @@ -252,13 +254,13 @@ def lemma(name_synsets): >>> from pythainlp.corpus.wordnet import lemma >>> - >>> lemma('practice.v.01.exercise') + >>> lemma("practice.v.01.exercise") Lemma('practice.v.01.exercise') >>> - >>> lemma('drill.v.03.exercise') + >>> lemma("drill.v.03.exercise") Lemma('drill.v.03.exercise') >>> - >>> lemma('exercise.n.01.exercise') + >>> lemma("exercise.n.01.exercise") Lemma('exercise.n.01.exercise') """ return wordnet.lemma(name_synsets) @@ -282,7 +284,7 @@ def lemma_from_key(key): >>> from pythainlp.corpus.wordnet import lemma, lemma_from_key >>> - >>> practice = lemma('practice.v.01.exercise') + >>> practice = lemma("practice.v.01.exercise") >>> practice.key() exercise%2:41:00:: >>> lemma_from_key(practice.key()) @@ -317,9 +319,9 @@ def path_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import path_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> path_similarity(entity, obj) 0.3333333333333333 @@ -355,9 +357,9 @@ def lch_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import lch_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> lch_similarity(entity, obj) 2.538973871058276 @@ -387,9 +389,9 @@ def wup_similarity(synsets1, synsets2): >>> from pythainlp.corpus.wordnet import wup_similarity, synset >>> - >>> entity = synset('entity.n.01') - >>> obj = synset('object.n.01') - >>> cat = synset('cat.n.01') + >>> entity = synset("entity.n.01") + >>> obj = synset("object.n.01") + >>> cat = synset("cat.n.01") >>> >>> wup_similarity(entity, obj) 0.5 diff --git a/pythainlp/el/__init__.py b/pythainlp/el/__init__.py index d076062b1..f6e8a6a28 100644 --- a/pythainlp/el/__init__.py +++ b/pythainlp/el/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/el/_multiel.py b/pythainlp/el/_multiel.py index babe2b292..2c73b64b0 100644 --- a/pythainlp/el/_multiel.py +++ b/pythainlp/el/_multiel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index dbb30d97e..b991d78e2 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -1,12 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations class EntityLinker: - def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata"): + def __init__( + self, + model_name: str = "bela", + device: str = "cuda", + tag: str = "wikidata", + ): """ EntityLinker @@ -21,24 +25,30 @@ def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata") self.device = device self.tag = tag if self.model_name not in ["bela"]: - raise NotImplementedError(f"EntityLinker doesn't support {model_name} model.") + raise NotImplementedError( + f"EntityLinker doesn't support {model_name} model." + ) if self.tag not in ["wikidata"]: - raise NotImplementedError(f"EntityLinker doesn't support {tag} tag.") + raise NotImplementedError( + f"EntityLinker doesn't support {tag} tag." + ) from pythainlp.el._multiel import MultiEL + self.model = MultiEL(model_name=self.model_name, device=self.device) - def get_el(self, list_text:Union[List[str], str])->Union[List[dict], str]: + + def get_el(self, list_text: list[str] | str) -> list[dict] | str: """ Get Entity Linking from Thai Text - + :param str Union[List[str], str]: list of Thai text or text :return: list of entity linking :rtype: Union[List[dict], str] - + :Example: :: from pythainlp.el import EntityLinker - + el = EntityLinker(device="cuda") print(el.get_el("จ๊อบเคยเป็นซีอีโอบริษัทแอปเปิล")) # output: [{'offsets': [11, 23], diff --git a/pythainlp/generate/__init__.py b/pythainlp/generate/__init__.py index ab68623f9..b3cb92c4a 100644 --- a/pythainlp/generate/__init__.py +++ b/pythainlp/generate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 14102f14c..5f3bb60fc 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,8 +8,9 @@ https://towardsdatascience.com/understanding-word-n-grams-and-n-gram-probability-in-natural-language-processing-9d9eef0fa058 """ +from __future__ import annotations + import random -from typing import List, Union from pythainlp.corpus.oscar import ( unigram_word_freqs as oscar_word_freqs_unigram, @@ -52,7 +52,7 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: + ) -> list[str] | str: """ :param str start_seq: word to begin sentence with :param int N: number of words @@ -148,7 +148,7 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: + ) -> list[str] | str: """ :param str start_seq: word to begin sentence with :param int N: number of words @@ -240,7 +240,7 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, duplicate: bool = False, - ) -> Union[List[str], str]: + ) -> list[str] | str: """ :param str start_seq: word to begin sentence with :param int N: number of words diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 445a063af..2498937f2 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,11 +8,12 @@ https://github.com/PyThaiNLP/tutorials/blob/master/source/notebooks/text_generation.ipynb """ +from __future__ import annotations + __all__ = ["gen_sentence"] import pickle import random -from typing import List, Union # fastai import fastai @@ -88,7 +88,7 @@ def gen_sentence( N: int = 4, prob: float = 0.001, output_str: bool = True, -) -> Union[List[str], str]: +) -> list[str] | str: """ Text generator using Thai2fit diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index cdb63efad..1e93b6413 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import re import torch @@ -9,34 +10,32 @@ class WangChanGLM: def __init__(self): - self.exclude_pattern = re.compile(r'[^ก-๙]+') + self.exclude_pattern = re.compile(r"[^ก-๙]+") self.stop_token = "\n" self.PROMPT_DICT = { "prompt_input": ( ": {input}\n: {instruction}\n: " ), - "prompt_no_input": ( - ": {instruction}\n: " - ), - "prompt_chatbot": ( - ": {human}\n: {bot}" - ), + "prompt_no_input": (": {instruction}\n: "), + "prompt_chatbot": (": {human}\n: {bot}"), } - def is_exclude(self, text:str)->bool: + + def is_exclude(self, text: str) -> bool: return bool(self.exclude_pattern.search(text)) + def load_model( self, - model_path:str="pythainlp/wangchanglm-7.5B-sft-en-sharded", - return_dict:bool=True, - load_in_8bit:bool=False, - device:str="cuda", + model_path: str = "pythainlp/wangchanglm-7.5B-sft-en-sharded", + return_dict: bool = True, + load_in_8bit: bool = False, + device: str = "cuda", torch_dtype=torch.float16, - offload_folder:str="./", - low_cpu_mem_usage:bool=True + offload_folder: str = "./", + low_cpu_mem_usage: bool = True, ): """ Load model - + :param str model_path: model path :param bool return_dict: return dict :param bool load_in_8bit: load model in 8bit @@ -47,6 +46,7 @@ def load_model( """ import pandas as pd from transformers import AutoModelForCausalLM, AutoTokenizer + self.device = device self.torch_dtype = torch_dtype self.model_path = model_path @@ -57,27 +57,30 @@ def load_model( device_map=device, torch_dtype=torch_dtype, offload_folder=offload_folder, - low_cpu_mem_usage=low_cpu_mem_usage + low_cpu_mem_usage=low_cpu_mem_usage, ) self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) - self.df = pd.DataFrame(self.tokenizer.vocab.items(), columns=['text', 'idx']) - self.df['is_exclude'] = self.df.text.map(self.is_exclude) + self.df = pd.DataFrame( + self.tokenizer.vocab.items(), columns=["text", "idx"] + ) + self.df["is_exclude"] = self.df.text.map(self.is_exclude) self.exclude_ids = self.df[self.df.is_exclude is True].idx.tolist() + def gen_instruct( self, - text:str, - max_new_tokens:int=512, - top_p:float=0.95, - temperature:float=0.9, - top_k:int=50, - no_repeat_ngram_size:int=2, - typical_p:float=1., - thai_only:bool=True, - skip_special_tokens:bool=True + text: str, + max_new_tokens: int = 512, + top_p: float = 0.95, + temperature: float = 0.9, + top_k: int = 50, + no_repeat_ngram_size: int = 2, + typical_p: float = 1.0, + thai_only: bool = True, + skip_special_tokens: bool = True, ): """ Generate Instruct - + :param str text: text :param int max_new_tokens: maximum number of new tokens :param float top_p: top p @@ -95,43 +98,47 @@ def gen_instruct( if thai_only: output_tokens = self.model.generate( input_ids=batch["input_ids"], - max_new_tokens=max_new_tokens, # 512 - begin_suppress_tokens = self.exclude_ids, + max_new_tokens=max_new_tokens, # 512 + begin_suppress_tokens=self.exclude_ids, no_repeat_ngram_size=no_repeat_ngram_size, - #oasst k50 + # oasst k50 top_k=top_k, - top_p=top_p, # 0.95 + top_p=top_p, # 0.95 typical_p=typical_p, - temperature=temperature, # 0.9 + temperature=temperature, # 0.9 ) else: output_tokens = self.model.generate( input_ids=batch["input_ids"], - max_new_tokens=max_new_tokens, # 512 + max_new_tokens=max_new_tokens, # 512 no_repeat_ngram_size=no_repeat_ngram_size, - #oasst k50 + # oasst k50 top_k=top_k, - top_p=top_p, # 0.95 + top_p=top_p, # 0.95 typical_p=typical_p, - temperature=temperature, # 0.9 + temperature=temperature, # 0.9 ) - return self.tokenizer.decode(output_tokens[0][len(batch["input_ids"][0]):], skip_special_tokens=skip_special_tokens) + return self.tokenizer.decode( + output_tokens[0][len(batch["input_ids"][0]) :], + skip_special_tokens=skip_special_tokens, + ) + def instruct_generate( self, instruct: str, context: str = None, max_new_tokens=512, - temperature: float =0.9, + temperature: float = 0.9, top_p: float = 0.95, - top_k:int=50, - no_repeat_ngram_size:int=2, - typical_p:float=1, - thai_only:bool=True, - skip_special_tokens:bool=True + top_k: int = 50, + no_repeat_ngram_size: int = 2, + typical_p: float = 1, + thai_only: bool = True, + skip_special_tokens: bool = True, ): """ Generate Instruct - + :param str instruct: Instruct :param str context: context :param int max_new_tokens: maximum number of new tokens @@ -153,7 +160,7 @@ def instruct_generate( model = WangChanGLM() - model.load_model(device="cpu",torch_dtype=torch.bfloat16) + model.load_model(device="cpu", torch_dtype=torch.bfloat16) print(model.instruct_generate(instruct="ขอวิธีลดน้ำหนัก")) # output: ลดน้ําหนักให้ได้ผล ต้องทําอย่างค่อยเป็นค่อยไป @@ -166,12 +173,12 @@ def instruct_generate( """ if context in (None, ""): - prompt = self.PROMPT_DICT['prompt_no_input'].format_map( - {'instruction': instruct, 'input': ''} + prompt = self.PROMPT_DICT["prompt_no_input"].format_map( + {"instruction": instruct, "input": ""} ) else: - prompt = self.PROMPT_DICT['prompt_input'].format_map( - {'instruction': instruct, 'input': context} + prompt = self.PROMPT_DICT["prompt_input"].format_map( + {"instruction": instruct, "input": context} ) result = self.gen_instruct( prompt, @@ -182,6 +189,6 @@ def instruct_generate( no_repeat_ngram_size=no_repeat_ngram_size, typical_p=typical_p, thai_only=thai_only, - skip_special_tokens=skip_special_tokens + skip_special_tokens=skip_special_tokens, ) return result diff --git a/pythainlp/khavee/__init__.py b/pythainlp/khavee/__init__.py index b8959d272..e4882747b 100644 --- a/pythainlp/khavee/__init__.py +++ b/pythainlp/khavee/__init__.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -__all__ = ["KhaveeVerifier"] - -from pythainlp.khavee.core import KhaveeVerifier +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +__all__ = ["KhaveeVerifier"] + +from pythainlp.khavee.core import KhaveeVerifier diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 81e05b662..c22a8212f 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -1,691 +1,687 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: C901 - -from typing import List, Union - -from pythainlp import thai_consonants -from pythainlp.tokenize import subword_tokenize -from pythainlp.util import remove_tonemark, sound_syllable - - -class KhaveeVerifier: - def __init__(self): - """ - KhaveeVerifier: Thai Poetry verifier - """ - - def _has_true_final_yl(self, word: str) -> bool: - """ - Check if ย or ล is a true final consonant - (not just part of the vowel sound with ไ/ใ) - - :param str word: Thai word - :return: True if ย or ล is a true final consonant - :rtype: bool - """ - if len(word) < 2: - return False - # Count consonants in the word - consonant_count = sum(1 for c in word if c in thai_consonants) - # If there are 2+ consonants and word ends with ย or ล, it's a true final - return consonant_count >= 2 and word[-1] in ["ย", "ล"] - - def check_sara(self, word: str) -> str: - """ - Check the vowels in the Thai word. - - :param str word: Thai word - :return: vowel name of the word - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_sara("เริง")) - # output: 'เออ' - """ - sara = [] - countoa = 0 - - # In case of การันย์ - if "์" in word[-1]: - word = word[:-2] - - # In case of สระเดี่ยว - for i in word: - if i in ("ะ", "ั"): - sara.append("อะ") - elif i == "ิ": - sara.append("อิ") - elif i == "ุ": - sara.append("อุ") - elif i == "ึ": - sara.append("อึ") - elif i == "ี": - sara.append("อี") - elif i == "ู": - sara.append("อู") - elif i == "ื": - sara.append("อือ") - elif i == "เ": - sara.append("เอ") - elif i == "แ": - sara.append("แอ") - elif i == "า": - sara.append("อา") - elif i == "โ": - sara.append("โอ") - elif i == "ำ": - sara.append("อำ") - elif i == "อ": - countoa += 1 - sara.append("ออ") - elif i == "ั" and "ว" in word: - sara.append("อัว") - elif i in ("ไ", "ใ"): - sara.append("ไอ") - elif i == "็": - sara.append("ออ") - elif "รร" in word: - if self.check_marttra(word) == "กม": - sara.append("อำ") - else: - sara.append("อะ") - - # In case of ออ - if countoa == 1 and "อ" in word[-1] and "เ" not in word: - sara.remove("ออ") - - # In case of เอ เอ - countA = 0 - for i in sara: - if i == "เอ": - countA = countA + 1 - if countA > 1: - sara.remove("เอ") - sara.remove("เอ") - sara.append("แ") - - # In case of สระประสม - if "เอ" in sara and "อะ" in sara: - sara.remove("เอ") - sara.remove("อะ") - sara.append("เอะ") - elif "แอ" in sara and "อะ" in sara: - sara.remove("แอ") - sara.remove("อะ") - sara.append("แอะ") - - if "เอะ" in sara and "ออ" in sara: - sara.remove("เอะ") - sara.remove("ออ") - sara.append("เออะ") - elif "เอ" in sara and "อิ" in sara: - sara.remove("เอ") - sara.remove("อิ") - sara.append("เออ") - elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: - sara.remove("เอ") - sara.remove("ออ") - sara.append("เออ") - elif "โอ" in sara and "อะ" in sara: - sara.remove("โอ") - sara.remove("อะ") - sara.append("โอะ") - elif "เอ" in sara and "อี" in sara: - sara.remove("เอ") - sara.remove("อี") - sara.append("เอีย") - elif "เอ" in sara and "อือ" in sara: - sara.remove("เอ") - sara.remove("อือ") - sara.append("อัว") - elif "เอ" in sara and "อา" in sara: - sara.remove("เอ") - sara.remove("อา") - sara.append("เอา") - elif "เ" in word and "า" in word and "ะ" in word: - sara = [] - sara.append("เอาะ") - - if "อือ" in sara and "เออ" in sara: - sara.remove("เออ") - sara.remove("อือ") - sara.append("เอือ") - elif "ออ" in sara and len(sara) > 1: - sara.remove("ออ") - elif "ว" in word and len(sara) == 0: - sara.append("อัว") - - if "ั" in word and self.check_marttra(word) == "กา": - sara = [] - sara.append("ไอ") - - # In case of อ - if word == "เออะ": - sara = [] - sara.append("เออะ") - elif word == "เออ": - sara = [] - sara.append("เออ") - elif word == "เอ": - sara = [] - sara.append("เอ") - elif word == "เอะ": - sara = [] - sara.append("เอะ") - elif word == "เอา": - sara = [] - sara.append("เอา") - elif word == "เอาะ": - sara = [] - sara.append("เอาะ") - - if "ฤา" in word or "ฦา" in word: - sara = [] - sara.append("อือ") - elif "ฤ" in word or "ฦ" in word: - sara = [] - sara.append("อึ") - - # In case of กน - if not sara and len(word) == 2: - if word[-1] != "ร": - sara.append("โอะ") - else: - sara.append("ออ") - elif not sara and len(word) == 3: - sara.append("ออ") - - # In case of บ่ - if word == "บ่": - sara = [] - sara.append("ออ") - - if "ํ" in word: - sara = [] - sara.append("อำ") - - if "เ" in word and "ื" in word and "อ" in word: - sara = [] - sara.append("เอือ") - - if not sara: - return "Can't find Sara in this word" - - return sara[0] - - def check_marttra(self, word: str) -> str: - """ - Check the Thai spelling Section in the Thai word. - - :param str word: Thai word - :return: name of spelling Section of the word. - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_marttra("สาว")) - # output: 'เกอว' - """ - # Handle consonant clusters ending with ร - # ตร, ทร → remove ร (treat as final ต/ท sound) - # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) - # But single syllable words like "กร" should keep ร - if len(word) >= 3 and word[-1] == "ร": - if word[-2] in ["ต", "ท"]: - word = word[:-1] - elif word[-2] in ["ก", "ข", "ค", "ฆ"]: - word = word[:-1] - - word = self.handle_karun_sound_silence(word) - word = remove_tonemark(word) - - # Check for ำ at the end (represents "am" sound, ends with m) - if word[-1] == "ำ": - return "กม" - - # Check for vowels and special patterns that indicate open syllables (กา) - # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel - if "ไ" in word or "ใ" in word: - if word[-1] not in ["ย", "ล"]: - return "กา" - elif not self._has_true_final_yl(word): - # ย/ล is part of the vowel sound, not a true final - return "กา" - # else: ย/ล is a true final, continue to consonant classification below - - if ( - ("ํ" in word and "า" in word) - ): - return "กา" - elif ( - word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] - or ("ี" in word and "ย" in word[-1]) - or ("ื" in word and "อ" in word[-1]) - ): - return "กา" - elif word[-1] in ["ง"]: - return "กง" - elif word[-1] in ["ม"]: - return "กม" - elif word[-1] in ["ย"]: - return "เกย" - elif word[-1] in ["ล"]: - return "เกย" - elif word[-1] in ["ว"]: - return "เกอว" - elif word[-1] in ["ก", "ข", "ค", "ฆ"]: - return "กก" - elif word[-1] in [ - "จ", - "ช", - "ซ", - "ฎ", - "ฏ", - "ฐ", - "ฑ", - "ฒ", - "ด", - "ต", - "ถ", - "ท", - "ธ", - "ศ", - "ษ", - "ส", - ]: - return "กด" - elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: - return "กน" - elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: - return "กบ" - else: - if "็" in word: - return "กา" - else: - return "Cant find Marttra in this word" - - def is_sumpus(self, word1: str, word2: str) -> bool: - """ - Check the rhyme between two words. - - :param str word1: Thai word - :param str word2: Thai word - :return: boolean - :rtype: bool - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.is_sumpus("สรร", "อัน")) - # output: True - - print(kv.is_sumpus("สรร", "แมว")) - # output: False - """ - marttra1 = self.check_marttra(word1) - marttra2 = self.check_marttra(word2) - sara1 = self.check_sara(word1) - sara2 = self.check_sara(word2) - if sara1 == "อะ" and marttra1 == "เกย": - sara1 = "ไอ" - marttra1 = "กา" - elif sara2 == "อะ" and marttra2 == "เกย": - sara2 = "ไอ" - marttra2 = "กา" - if sara1 == "อำ" and marttra1 == "กม": - sara1 = "อำ" - marttra1 = "กา" - elif sara2 == "อำ" and marttra2 == "กม": - sara2 = "อำ" - marttra2 = "กา" - return bool(marttra1 == marttra2 and sara1 == sara2) - - def check_karu_lahu(self, text): - if ( - self.check_marttra(text) != "กา" - or ( - self.check_marttra(text) == "กา" - and self.check_sara(text) - in [ - "อา", - "อี", - "อือ", - "อู", - "เอ", - "แอ", - "โอ", - "ออ", - "เออ", - "เอีย", - "เอือ", - "อัว", - ] - ) - or self.check_sara(text) in ["อำ", "ไอ", "เอา"] - ) and text not in ["บ่", "ณ", "ธ", "ก็"]: - return "karu" - else: - return "lahu" - - def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: - """ - Check the suitability of the poem according to Thai principles. - - :param str text: Thai poem - :param int k_type: type of Thai poem - :return: the check results of the suitability of the poem according to Thai principles. - :rtype: Union[List[str], str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ - มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: The poem is correct according to the principle. - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ - เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: [ - "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", - "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" - ] - """ - if k_type == 8: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 10: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 10 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append( - [ - sub_sent[1], - sub_sent[2], - sub_sent[3], - sub_sent[4], - ] - ) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 3: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form of klon 8." - elif k_type == 4: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 5: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 4 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 1: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form." - - else: - return "Something went wrong. Make sure you enter it in the correct form." - - def check_aek_too( - self, text: Union[List[str], str], dead_syllable_as_aek: bool = False - ) -> Union[List[bool], List[str], bool, str]: - """ - Checker of Thai tonal words - - :param Union[List[str], str] text: Thai word or list of Thai words - :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek - :return: the check result if the word is aek or too or False (not both) or list of check results if input is list - :rtype: Union[List[bool], List[str], bool, str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - # การเช็คคำเอกโท - print( - kv.check_aek_too("เอง"), - kv.check_aek_too("เอ่ง"), - kv.check_aek_too("เอ้ง"), - ) - # -> False, aek, too - print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน - # -> [False, 'aek', 'too'] - - - """ - if isinstance(text, list): - return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] - - if not isinstance(text, str): - raise TypeError("text must be str or iterable list[str]") - - word_characters = [*text] - if "่" in word_characters and "้" not in word_characters: - return "aek" - elif "้" in word_characters and "่" not in word_characters: - return "too" - if dead_syllable_as_aek and sound_syllable(text) == "dead": - return "aek" - else: - return False - - def handle_karun_sound_silence(self, word: str) -> str: - """ - Handle silent sounds in Thai words using '์' character (Karun) - by stripping all characters before the 'Karun' character that should be silenced - - :param str text: Thai word - :return: Thai word with silent words stripped - :rtype: str - """ - sound_silenced = word.endswith("์") - if not sound_silenced: - return word - # Remove ์ and the silent consonant before it - # การันต์ (์) marks the consonant immediately before it as silent - word = word[:-2] - return word +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: C901 +from __future__ import annotations + +from pythainlp import thai_consonants +from pythainlp.tokenize import subword_tokenize +from pythainlp.util import remove_tonemark, sound_syllable + + +class KhaveeVerifier: + def __init__(self): + """ + KhaveeVerifier: Thai Poetry verifier + """ + + def _has_true_final_yl(self, word: str) -> bool: + """ + Check if ย or ล is a true final consonant + (not just part of the vowel sound with ไ/ใ) + + :param str word: Thai word + :return: True if ย or ล is a true final consonant + :rtype: bool + """ + if len(word) < 2: + return False + # Count consonants in the word + consonant_count = sum(1 for c in word if c in thai_consonants) + # If there are 2+ consonants and word ends with ย or ล, it's a true final + return consonant_count >= 2 and word[-1] in ["ย", "ล"] + + def check_sara(self, word: str) -> str: + """ + Check the vowels in the Thai word. + + :param str word: Thai word + :return: vowel name of the word + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_sara("เริง")) + # output: 'เออ' + """ + sara = [] + countoa = 0 + + # In case of การันย์ + if "์" in word[-1]: + word = word[:-2] + + # In case of สระเดี่ยว + for i in word: + if i in ("ะ", "ั"): + sara.append("อะ") + elif i == "ิ": + sara.append("อิ") + elif i == "ุ": + sara.append("อุ") + elif i == "ึ": + sara.append("อึ") + elif i == "ี": + sara.append("อี") + elif i == "ู": + sara.append("อู") + elif i == "ื": + sara.append("อือ") + elif i == "เ": + sara.append("เอ") + elif i == "แ": + sara.append("แอ") + elif i == "า": + sara.append("อา") + elif i == "โ": + sara.append("โอ") + elif i == "ำ": + sara.append("อำ") + elif i == "อ": + countoa += 1 + sara.append("ออ") + elif i == "ั" and "ว" in word: + sara.append("อัว") + elif i in ("ไ", "ใ"): + sara.append("ไอ") + elif i == "็": + sara.append("ออ") + elif "รร" in word: + if self.check_marttra(word) == "กม": + sara.append("อำ") + else: + sara.append("อะ") + + # In case of ออ + if countoa == 1 and "อ" in word[-1] and "เ" not in word: + sara.remove("ออ") + + # In case of เอ เอ + countA = 0 + for i in sara: + if i == "เอ": + countA = countA + 1 + if countA > 1: + sara.remove("เอ") + sara.remove("เอ") + sara.append("แ") + + # In case of สระประสม + if "เอ" in sara and "อะ" in sara: + sara.remove("เอ") + sara.remove("อะ") + sara.append("เอะ") + elif "แอ" in sara and "อะ" in sara: + sara.remove("แอ") + sara.remove("อะ") + sara.append("แอะ") + + if "เอะ" in sara and "ออ" in sara: + sara.remove("เอะ") + sara.remove("ออ") + sara.append("เออะ") + elif "เอ" in sara and "อิ" in sara: + sara.remove("เอ") + sara.remove("อิ") + sara.append("เออ") + elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: + sara.remove("เอ") + sara.remove("ออ") + sara.append("เออ") + elif "โอ" in sara and "อะ" in sara: + sara.remove("โอ") + sara.remove("อะ") + sara.append("โอะ") + elif "เอ" in sara and "อี" in sara: + sara.remove("เอ") + sara.remove("อี") + sara.append("เอีย") + elif "เอ" in sara and "อือ" in sara: + sara.remove("เอ") + sara.remove("อือ") + sara.append("อัว") + elif "เอ" in sara and "อา" in sara: + sara.remove("เอ") + sara.remove("อา") + sara.append("เอา") + elif "เ" in word and "า" in word and "ะ" in word: + sara = [] + sara.append("เอาะ") + + if "อือ" in sara and "เออ" in sara: + sara.remove("เออ") + sara.remove("อือ") + sara.append("เอือ") + elif "ออ" in sara and len(sara) > 1: + sara.remove("ออ") + elif "ว" in word and len(sara) == 0: + sara.append("อัว") + + if "ั" in word and self.check_marttra(word) == "กา": + sara = [] + sara.append("ไอ") + + # In case of อ + if word == "เออะ": + sara = [] + sara.append("เออะ") + elif word == "เออ": + sara = [] + sara.append("เออ") + elif word == "เอ": + sara = [] + sara.append("เอ") + elif word == "เอะ": + sara = [] + sara.append("เอะ") + elif word == "เอา": + sara = [] + sara.append("เอา") + elif word == "เอาะ": + sara = [] + sara.append("เอาะ") + + if "ฤา" in word or "ฦา" in word: + sara = [] + sara.append("อือ") + elif "ฤ" in word or "ฦ" in word: + sara = [] + sara.append("อึ") + + # In case of กน + if not sara and len(word) == 2: + if word[-1] != "ร": + sara.append("โอะ") + else: + sara.append("ออ") + elif not sara and len(word) == 3: + sara.append("ออ") + + # In case of บ่ + if word == "บ่": + sara = [] + sara.append("ออ") + + if "ํ" in word: + sara = [] + sara.append("อำ") + + if "เ" in word and "ื" in word and "อ" in word: + sara = [] + sara.append("เอือ") + + if not sara: + return "Can't find Sara in this word" + + return sara[0] + + def check_marttra(self, word: str) -> str: + """ + Check the Thai spelling Section in the Thai word. + + :param str word: Thai word + :return: name of spelling Section of the word. + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_marttra("สาว")) + # output: 'เกอว' + """ + # Handle consonant clusters ending with ร + # ตร, ทร → remove ร (treat as final ต/ท sound) + # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) + # But single syllable words like "กร" should keep ร + if len(word) >= 3 and word[-1] == "ร": + if word[-2] in ["ต", "ท"]: + word = word[:-1] + elif word[-2] in ["ก", "ข", "ค", "ฆ"]: + word = word[:-1] + + word = self.handle_karun_sound_silence(word) + word = remove_tonemark(word) + + # Check for ำ at the end (represents "am" sound, ends with m) + if word[-1] == "ำ": + return "กม" + + # Check for vowels and special patterns that indicate open syllables (กา) + # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel + if "ไ" in word or "ใ" in word: + if word[-1] not in ["ย", "ล"]: + return "กา" + elif not self._has_true_final_yl(word): + # ย/ล is part of the vowel sound, not a true final + return "กา" + # else: ย/ล is a true final, continue to consonant classification below + + if "ํ" in word and "า" in word: + return "กา" + elif ( + word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] + or ("ี" in word and "ย" in word[-1]) + or ("ื" in word and "อ" in word[-1]) + ): + return "กา" + elif word[-1] in ["ง"]: + return "กง" + elif word[-1] in ["ม"]: + return "กม" + elif word[-1] in ["ย"]: + return "เกย" + elif word[-1] in ["ล"]: + return "เกย" + elif word[-1] in ["ว"]: + return "เกอว" + elif word[-1] in ["ก", "ข", "ค", "ฆ"]: + return "กก" + elif word[-1] in [ + "จ", + "ช", + "ซ", + "ฎ", + "ฏ", + "ฐ", + "ฑ", + "ฒ", + "ด", + "ต", + "ถ", + "ท", + "ธ", + "ศ", + "ษ", + "ส", + ]: + return "กด" + elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: + return "กน" + elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: + return "กบ" + else: + if "็" in word: + return "กา" + else: + return "Cant find Marttra in this word" + + def is_sumpus(self, word1: str, word2: str) -> bool: + """ + Check the rhyme between two words. + + :param str word1: Thai word + :param str word2: Thai word + :return: boolean + :rtype: bool + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.is_sumpus("สรร", "อัน")) + # output: True + + print(kv.is_sumpus("สรร", "แมว")) + # output: False + """ + marttra1 = self.check_marttra(word1) + marttra2 = self.check_marttra(word2) + sara1 = self.check_sara(word1) + sara2 = self.check_sara(word2) + if sara1 == "อะ" and marttra1 == "เกย": + sara1 = "ไอ" + marttra1 = "กา" + elif sara2 == "อะ" and marttra2 == "เกย": + sara2 = "ไอ" + marttra2 = "กา" + if sara1 == "อำ" and marttra1 == "กม": + sara1 = "อำ" + marttra1 = "กา" + elif sara2 == "อำ" and marttra2 == "กม": + sara2 = "อำ" + marttra2 = "กา" + return bool(marttra1 == marttra2 and sara1 == sara2) + + def check_karu_lahu(self, text): + if ( + self.check_marttra(text) != "กา" + or ( + self.check_marttra(text) == "กา" + and self.check_sara(text) + in [ + "อา", + "อี", + "อือ", + "อู", + "เอ", + "แอ", + "โอ", + "ออ", + "เออ", + "เอีย", + "เอือ", + "อัว", + ] + ) + or self.check_sara(text) in ["อำ", "ไอ", "เอา"] + ) and text not in ["บ่", "ณ", "ธ", "ก็"]: + return "karu" + else: + return "lahu" + + def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: + """ + Check the suitability of the poem according to Thai principles. + + :param str text: Thai poem + :param int k_type: type of Thai poem + :return: the check results of the suitability of the poem according to Thai principles. + :rtype: Union[List[str], str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ + มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: The poem is correct according to the principle. + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ + เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: [ + "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", + "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" + ] + """ + if k_type == 8: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 10: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 10 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append( + [ + sub_sent[1], + sub_sent[2], + sub_sent[3], + sub_sent[4], + ] + ) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 3: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form of klon 8." + elif k_type == 4: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 5: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 4 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 1: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form." + + else: + return "Something went wrong. Make sure you enter it in the correct form." + + def check_aek_too( + self, text: list[str] | str, dead_syllable_as_aek: bool = False + ) -> list[bool] | list[str] | bool | str: + """ + Checker of Thai tonal words + + :param Union[List[str], str] text: Thai word or list of Thai words + :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek + :return: the check result if the word is aek or too or False (not both) or list of check results if input is list + :rtype: Union[List[bool], List[str], bool, str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + # การเช็คคำเอกโท + print( + kv.check_aek_too("เอง"), + kv.check_aek_too("เอ่ง"), + kv.check_aek_too("เอ้ง"), + ) + # -> False, aek, too + print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน + # -> [False, 'aek', 'too'] + + + """ + if isinstance(text, list): + return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] + + if not isinstance(text, str): + raise TypeError("text must be str or iterable list[str]") + + word_characters = [*text] + if "่" in word_characters and "้" not in word_characters: + return "aek" + elif "้" in word_characters and "่" not in word_characters: + return "too" + if dead_syllable_as_aek and sound_syllable(text) == "dead": + return "aek" + else: + return False + + def handle_karun_sound_silence(self, word: str) -> str: + """ + Handle silent sounds in Thai words using '์' character (Karun) + by stripping all characters before the 'Karun' character that should be silenced + + :param str text: Thai word + :return: Thai word with silent words stripped + :rtype: str + """ + sound_silenced = word.endswith("์") + if not sound_silenced: + return word + # Remove ์ and the silent consonant before it + # การันต์ (์) marks the consonant immediately before it as silent + word = word[:-2] + return word diff --git a/pythainlp/lm/__init__.py b/pythainlp/lm/__init__.py index 60d8b503c..259f101d2 100644 --- a/pythainlp/lm/__init__.py +++ b/pythainlp/lm/__init__.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -__all__ = [ - "calculate_ngram_counts", - "remove_repeated_ngrams" -] +__all__ = ["calculate_ngram_counts", "remove_repeated_ngrams"] -from pythainlp.lm.text_util import calculate_ngram_counts, remove_repeated_ngrams +from pythainlp.lm.text_util import ( + calculate_ngram_counts, + remove_repeated_ngrams, +) diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 946fd8451..35e4b9167 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -1,16 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 # ruff: noqa: C901 - -from typing import List, Tuple, Dict +from __future__ import annotations def calculate_ngram_counts( - list_words: List[str], - n_min: int = 2, - n_max: int = 4) -> Dict[Tuple[str], int]: + list_words: list[str], n_min: int = 2, n_max: int = 4 +) -> dict[tuple[str], int]: """ Calculates the counts of n-grams in the list words for the specified range. @@ -26,13 +23,13 @@ def calculate_ngram_counts( for n in range(n_min, n_max + 1): for i in range(len(list_words) - n + 1): - ngram = tuple(list_words[i:i + n]) + ngram = tuple(list_words[i : i + n]) ngram_counts[ngram] = ngram_counts.get(ngram, 0) + 1 return ngram_counts -def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: +def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]: """ Remove repeated n-grams @@ -46,7 +43,7 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: from pythainlp.lm import remove_repeated_ngrams - remove_repeated_ngrams(['เอา', 'เอา', 'แบบ', 'ไหน'], n=1) + remove_repeated_ngrams(["เอา", "เอา", "แบบ", "ไหน"], n=1) # output: ['เอา', 'แบบ', 'ไหน'] """ if not string_list or n <= 0: @@ -58,12 +55,14 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: for i in range(len(string_list)): if i + n <= len(string_list): - ngram = tuple(string_list[i:i + n]) + ngram = tuple(string_list[i : i + n]) if ngram not in unique_ngrams: unique_ngrams.add(ngram) - if not output_list or output_list[-(n - 1):] != list(ngram[:-1]): + if not output_list or output_list[-(n - 1) :] != list( + ngram[:-1] + ): output_list.extend(ngram) else: output_list.append(ngram[-1]) diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py index d40baa777..3f8ed10dc 100644 --- a/pythainlp/morpheme/__init__.py +++ b/pythainlp/morpheme/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,9 +5,7 @@ """ PyThaiNLP morpheme """ -__all__ = [ - "nighit", - "is_native_thai" -] + +__all__ = ["nighit", "is_native_thai"] from pythainlp.morpheme.thaiwordcheck import is_native_thai from pythainlp.morpheme.word_formation import nighit diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py index 61c02baff..9fce45aaf 100644 --- a/pythainlp/morpheme/thaiwordcheck.py +++ b/pythainlp/morpheme/thaiwordcheck.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,6 +12,9 @@ http://www.trueplookpanya.com/learning/detail/30589-043067 - วารุณี บำรุงรส 2010. คำไทยแท้ https://www.gotoknow.org/posts/377619 """ + +from __future__ import annotations + import re _THANTHAKHAT_CHAR = "\u0e4c" # Thanthakhat (cancellation of sound) diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index ede6c248d..11b432d93 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp import thai_consonants @@ -31,7 +32,7 @@ def nighit(w1: str, w2: str) -> str: assert nighit("สํ","ปทา")=="สัมปทา" assert nighit("สํ","โยค")=="สังโยค" """ - if not str(w1).endswith('ํ') and len(w1) != 2: + if not str(w1).endswith("ํ") and len(w1) != 2: raise NotImplementedError(f"The function doesn't support {w1}.") list_w1 = list(w1) list_w2 = list(w2) @@ -56,4 +57,4 @@ def nighit(w1: str, w2: str) -> str: The function doesn't support {w1} and {w2}. """) newword.extend(list_w2) - return ''.join(newword) + return "".join(newword) diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py index d7aae6f1c..e4afb08e1 100644 --- a/pythainlp/parse/__init__.py +++ b/pythainlp/parse/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,7 @@ """ PyThaiNLP Parse """ + __all__ = ["dependency_parsing"] from pythainlp.parse.core import dependency_parsing diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index 2a64b4bd5..6421dd87a 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - -from typing import List, Union +from __future__ import annotations _tagger = None _tagger_name = "" @@ -11,10 +9,10 @@ def dependency_parsing( text: str, - model: Union[str, None] = None, + model: str | None = None, tag: str = "str", engine: str = "esupar", -) -> Union[List[List[str]], str]: +) -> list[list[str]] | str: """ Dependency Parsing diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 259cb00b7..06ae3a35e 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- """ esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages GitHub: https://github.com/KoichiYasuoka/esupar """ -from typing import List, Union + +from __future__ import annotations try: import esupar @@ -18,9 +18,7 @@ def __init__(self, model: str = "th") -> None: model = "th" self.nlp = esupar.load(model) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: _data = str(self.nlp(text)) if tag == "list": _temp = _data.splitlines() diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py index 6d0eef7ab..3f44a61a7 100644 --- a/pythainlp/parse/spacy_thai_engine.py +++ b/pythainlp/parse/spacy_thai_engine.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- """ spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies. GitHub: https://github.com/KoichiYasuoka/spacy-thai """ -from typing import List, Union + +from __future__ import annotations import spacy_thai @@ -14,9 +14,7 @@ class Parse: def __init__(self, model: str = "th") -> None: self.nlp = spacy_thai.load() - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: doc = self.nlp(text) _text = [] if tag == "list": diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 9d1ffca5b..2f4aec7b2 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ TransformersUD @@ -10,8 +9,10 @@ GitHub: https://github.com/KoichiYasuoka """ + +from __future__ import annotations + import os -from typing import List, Union import numpy import torch @@ -56,9 +57,7 @@ def __init__( model=t, tokenizer=self.tokenizer ) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: w = [ (t["start"], t["end"], t["entity_group"]) for t in self.deprel(text) diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index fc258d41d..904c64dd3 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ UDgoeswith @@ -10,7 +9,8 @@ GitHub: https://github.com/KoichiYasuoka """ -from typing import List, Union + +from __future__ import annotations import numpy as np import torch @@ -27,9 +27,7 @@ def __init__( self.tokenizer = AutoTokenizer.from_pretrained(model) self.model = AutoModelForTokenClassification.from_pretrained(model) - def __call__( - self, text: str, tag: str = "str" - ) -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str: w = self.tokenizer(text, return_offsets_mapping=True) v = w["input_ids"] x = [ diff --git a/pythainlp/phayathaibert/__init__.py b/pythainlp/phayathaibert/__init__.py index 2d50efabd..088f8d219 100644 --- a/pythainlp/phayathaibert/__init__.py +++ b/pythainlp/phayathaibert/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 5631b766c..a73bcdfe9 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import random import re import warnings -from typing import Callable, List, Tuple, Union +from collections.abc import Callable from transformers import ( CamembertTokenizer, @@ -139,7 +139,7 @@ def _replace_rep(m): re_rep = re.compile(r"(\S)(\1{3,})") return re_rep.sub(_replace_rep, text) - def replace_wrep_post(self, toks: List[str]) -> List[str]: + def replace_wrep_post(self, toks: list[str]) -> list[str]: """ Replace repetitive words post tokenization; fastai `replace_wrep` does not work well with Thai. @@ -166,7 +166,7 @@ def replace_wrep_post(self, toks: List[str]) -> List[str]: return res[1:] - def remove_space(self, toks: List[str]) -> List[str]: + def remove_space(self, toks: list[str]) -> list[str]: """ Do not include space for bag-of-word models. :param List[str] toks: list of tokens @@ -189,7 +189,7 @@ def remove_space(self, toks: List[str]) -> List[str]: def preprocess( self, text: str, - pre_rules: List[Callable] = [ + pre_rules: list[Callable] = [ rm_brackets, replace_newlines, rm_useless_spaces, @@ -253,7 +253,7 @@ def augment( text: str, num_augs: int = 3, sample: bool = False, - ) -> List[str]: + ) -> list[str]: """ Text augmentation from PhayaThaiBERT @@ -315,7 +315,7 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: def get_tag( self, sentence: str, strategy: str = "simple" - ) -> List[List[Tuple[str, str]]]: + ) -> list[list[tuple[str, str]]]: """ Marks sentences with part-of-speech (POS) tags. @@ -363,7 +363,7 @@ def get_ner( tag: bool = False, pos: bool = False, strategy: str = "simple", - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: + ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: """ This function tags named entities in text in IOB format. @@ -435,7 +435,7 @@ def get_ner( return sample_output -def segment(sentence: str) -> List[str]: +def segment(sentence: str) -> list[str]: """ Subword tokenize of PhayaThaiBERT, \ sentencepiece from WangchanBERTa model with vocabulary expansion. diff --git a/pythainlp/soundex/__init__.py b/pythainlp/soundex/__init__.py index ed924db9f..6ff802204 100644 --- a/pythainlp/soundex/__init__.py +++ b/pythainlp/soundex/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/soundex/core.py b/pythainlp/soundex/core.py index 8ecbcb1b7..e0435a92e 100644 --- a/pythainlp/soundex/core.py +++ b/pythainlp/soundex/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,6 +6,9 @@ Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ + +from __future__ import annotations + from pythainlp.soundex import DEFAULT_SOUNDEX_ENGINE from pythainlp.soundex.lk82 import lk82 from pythainlp.soundex.metasound import metasound diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py index fd3010e69..1885553a7 100644 --- a/pythainlp/soundex/lk82.py +++ b/pythainlp/soundex/lk82.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -15,6 +14,9 @@ by Korakot Chaovavanich https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8 """ + +from __future__ import annotations + import re from pythainlp.util import remove_tonemark diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py index 7729d520c..924f9b58d 100644 --- a/pythainlp/soundex/metasound.py +++ b/pythainlp/soundex/metasound.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -12,6 +11,8 @@ https://pdfs.semanticscholar.org/3983/963e87ddc6dfdbb291099aa3927a0e3e4ea6.pdf """ +from __future__ import annotations + _CONS_THANTHAKHAT = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ์" _THANTHAKHAT = "์" # \u0e4c _C1 = "กขฃคฆฅ" # sound K -> coded letter 1 diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py index 5ba0908d7..ae74895ae 100644 --- a/pythainlp/soundex/prayut_and_somchaip.py +++ b/pythainlp/soundex/prayut_and_somchaip.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,6 +8,9 @@ References: Prayut Suwanvisat, Somchai Prasitjutrakul.Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf """ + +from __future__ import annotations + from pythainlp import thai_characters _C0 = "AEIOUHWYอ" diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py index 355949d90..a376d0715 100644 --- a/pythainlp/soundex/sound.py +++ b/pythainlp/soundex/sound.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations import panphon import panphon.distance @@ -13,6 +12,7 @@ _ft = panphon.FeatureTable() _dst = panphon.distance.Distance() + def _clean_ipa(ipa: str) -> str: """ Clean IPA by removing tones and space between phonetic codes @@ -21,7 +21,18 @@ def _clean_ipa(ipa: str) -> str: :return: IPA with tones removed from the text :rtype: str """ - return ipa.replace("˩˩˦","").replace("˥˩","").replace("˨˩","").replace("˦˥","").replace("˧","").replace("˧","").replace(" .",".").replace(". ",".").strip() + return ( + ipa.replace("˩˩˦", "") + .replace("˥˩", "") + .replace("˨˩", "") + .replace("˦˥", "") + .replace("˧", "") + .replace("˧", "") + .replace(" .", ".") + .replace(". ", ".") + .strip() + ) + def word2audio(word: str) -> str: """ @@ -41,10 +52,13 @@ def word2audio(word: str) -> str: """ _word = word_tokenize(word) _phone = [pronunciate(w, engine="w2p") for w in _word] - _ipa = [_clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone] - return '.'.join(_ipa) + _ipa = [ + _clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone + ] + return ".".join(_ipa) -def audio_vector(word: str) -> List[List[int]]: + +def audio_vector(word: str) -> list[list[int]]: """ Convert audio to vector list @@ -62,7 +76,8 @@ def audio_vector(word: str) -> List[List[int]]: """ return _ft.word_to_vector_list(word2audio(word), numeric=True) -def word_approximation(word: str, list_word: List[str]) -> List[float]: + +def word_approximation(word: str, list_word: list[str]) -> list[float]: """ Thai Word Approximation @@ -81,5 +96,7 @@ def word_approximation(word: str, list_word: List[str]) -> List[float]: """ _word = word2audio(word) _list_word = [word2audio(w) for w in list_word] - _distance = [_dst.weighted_feature_edit_distance(_word, w) for w in _list_word] + _distance = [ + _dst.weighted_feature_edit_distance(_word, w) for w in _list_word + ] return _distance diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py index 7b401bb01..caf6dd45a 100644 --- a/pythainlp/soundex/udom83.py +++ b/pythainlp/soundex/udom83.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -16,6 +15,9 @@ by Korakot Chaovavanich https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8 """ + +from __future__ import annotations + import re from pythainlp import thai_consonants diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 54e9af445..611a06230 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -22,4 +21,6 @@ # these imports are placed here to avoid circular imports from pythainlp.spell.core import correct, correct_sent, spell, spell_sent -from pythainlp.spell.words_spelling_correction import get_words_spell_suggestion +from pythainlp.spell.words_spelling_correction import ( + get_words_spell_suggestion, +) diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index a9320e756..d99ce5154 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,18 +5,21 @@ Spell checking functions """ -from functools import lru_cache +from __future__ import annotations + import itertools -from typing import List +from functools import lru_cache from pythainlp.spell import DEFAULT_SPELL_CHECKER + @lru_cache def default_spell_checker(): """Lazy load default spell checker with cache""" return DEFAULT_SPELL_CHECKER() -def spell(word: str, engine: str = "pn") -> List[str]: + +def spell(word: str, engine: str = "pn") -> list[str]: """ Provides a list of possible correct spellings of the given word. The list of words are from the words in the dictionary @@ -42,13 +44,13 @@ def spell(word: str, engine: str = "pn") -> List[str]: from pythainlp.spell import spell - spell("เส้นตรบ", engine="pn") + spell("เส้นตรบ", engine="pn") # output: ['เส้นตรง'] spell("เส้นตรบ") # output: ['เส้นตรง'] - spell("เส้นตรบ", engine="tltk") + spell("เส้นตรบ", engine="tltk") # output: ['เส้นตรง'] spell("ครัช") @@ -125,7 +127,9 @@ def correct(word: str, engine: str = "pn") -> str: text_correct = SPELL_CHECKER(word) elif engine == "wanchanberta_thai_grammarly": - from pythainlp.spell.wanchanberta_thai_grammarly import correct as SPELL_CHECKER + from pythainlp.spell.wanchanberta_thai_grammarly import ( + correct as SPELL_CHECKER, + ) text_correct = SPELL_CHECKER(word) @@ -135,7 +139,7 @@ def correct(word: str, engine: str = "pn") -> str: return text_correct -def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: +def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: """ Provides a list of possible correct spellings of sentence @@ -152,7 +156,7 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: from pythainlp.spell import spell_sent - spell_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy') + spell_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # output: [['เด็ก', 'อินเทอร์เน็ต', 'แรง']] """ if engine == "symspellpy": @@ -173,7 +177,7 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]: return list_new -def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]: +def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]: """ Corrects and returns the spelling of the given sentence @@ -191,7 +195,7 @@ def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]: from pythainlp.spell import correct_sent - correct_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy') + correct_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # output: ['เด็ก', 'อินเทอร์เน็ต', 'แรง'] """ return spell_sent(list_words, engine=engine)[0] diff --git a/pythainlp/spell/phunspell.py b/pythainlp/spell/phunspell.py index 7a293816d..26cb90f1e 100644 --- a/pythainlp/spell/phunspell.py +++ b/pythainlp/spell/phunspell.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,17 +10,20 @@ * \ https://github.com/dvwright/phunspell """ -from typing import List + +from __future__ import annotations try: import phunspell except ImportError: - raise ImportError("Import Error; Install phunspell by pip install phunspell") + raise ImportError( + "Import Error; Install phunspell by pip install phunspell" + ) pspell = phunspell.Phunspell("th_TH") -def spell(text: str) -> List[str]: +def spell(text: str) -> list[str]: return list(pspell.suggest(text)) diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 4cdfab5d7..12d2acc3e 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Spell checker, using Peter Norvig algorithm. Spelling dictionary can be customized. @@ -6,19 +5,12 @@ Based on Peter Norvig's Python code from http://norvig.com/spell-correct.html """ + +from __future__ import annotations + from collections import Counter +from collections.abc import Callable, ItemsView, Iterable from string import digits -from typing import ( - Callable, - Dict, - ItemsView, - Iterable, - List, - Optional, - Set, - Tuple, - Union, -) from pythainlp import thai_digits, thai_letters from pythainlp.corpus import tnc @@ -39,7 +31,7 @@ def _is_thai_and_not_num(word: str) -> bool: def _keep( - word_freq: Tuple[str, int], + word_freq: tuple[str, int], min_freq: int, min_len: int, max_len: int, @@ -59,7 +51,7 @@ def _keep( return dict_filter(word) -def _edits1(word: str) -> Set[str]: +def _edits1(word: str) -> set[str]: """ Returns a set of words with an edit distance of 1 from the input word """ @@ -72,7 +64,7 @@ def _edits1(word: str) -> Set[str]: return set(deletes + transposes + replaces + inserts) -def _edits2(word: str) -> Set[str]: +def _edits2(word: str) -> set[str]: """ Returns a set of words with an edit distance of 2 from the input word """ @@ -80,14 +72,12 @@ def _edits2(word: str) -> Set[str]: def _convert_custom_dict( - custom_dict: Union[ - Dict[str, int], Iterable[str], Iterable[Tuple[str, int]] - ], + custom_dict: dict[str, int] | Iterable[str] | Iterable[tuple[str, int]], min_freq: int, min_len: int, max_len: int, - dict_filter: Optional[Callable[[str], bool]], -) -> List[Tuple[str, int]]: + dict_filter: Callable[[str], bool] | None, +) -> list[tuple[str, int]]: """ Converts a custom dictionary to a list of (str, int) tuples """ @@ -123,13 +113,13 @@ def _convert_custom_dict( class NorvigSpellChecker: def __init__( self, - custom_dict: Union[ - Dict[str, int], Iterable[str], Iterable[Tuple[str, int]] - ] = None, + custom_dict: dict[str, int] + | Iterable[str] + | Iterable[tuple[str, int]] = None, min_freq: int = 2, min_len: int = 2, max_len: int = 40, - dict_filter: Optional[Callable[[str], bool]] = _is_thai_and_not_num, + dict_filter: Callable[[str], bool] | None = _is_thai_and_not_num, ): """ Initializes Peter Norvig's spell checker object. @@ -191,7 +181,7 @@ def dictionary(self) -> ItemsView[str, int]: from pythainlp.spell import NorvigSpellChecker - dictionary= [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] + dictionary = [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] checker = NorvigSpellChecker(custom_dict=dictionary) checker.dictionary() @@ -199,7 +189,7 @@ def dictionary(self) -> ItemsView[str, int]: """ return self.__WORDS.items() - def known(self, words: Iterable[str]) -> List[str]: + def known(self, words: Iterable[str]) -> list[str]: """ Returns a list of given words found in the spelling dictionary @@ -220,7 +210,7 @@ def known(self, words: Iterable[str]) -> List[str]: checker.known(["เพยน", "เพล", "เพลง"]) # output: ['เพล', 'เพลง'] - checker.known(['ยกไ', 'ไฟล์ม']) + checker.known(["ยกไ", "ไฟล์ม"]) # output: [] checker.known([]) @@ -280,7 +270,7 @@ def freq(self, word: str) -> int: """ return self.__WORDS[word] - def spell(self, word: str) -> List[str]: + def spell(self, word: str) -> list[str]: """ Returns a list of all correctly-spelled words whose spelling is similar to the given word by edit distance metrics. diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index b3c5abb81..d75b502f3 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,12 +12,14 @@ https://github.com/mammothb/symspellpy """ -from typing import List +from __future__ import annotations try: from symspellpy import SymSpell, Verbosity except ImportError: - raise ImportError("Import Error; Install symspellpy by pip install symspellpy") + raise ImportError( + "Import Error; Install symspellpy by pip install symspellpy" + ) from pythainlp.corpus import get_corpus_path, path_pythainlp_corpus @@ -42,7 +43,7 @@ ) -def spell(text: str, max_edit_distance: int = 2) -> List[str]: +def spell(text: str, max_edit_distance: int = 2) -> list[str]: return [ str(i).split(",", maxsplit=1)[0] for i in list( @@ -58,8 +59,8 @@ def correct(text: str, max_edit_distance: int = 1) -> str: def spell_sent( - list_words: List[str], max_edit_distance: int = 2 -) -> List[List[str]]: + list_words: list[str], max_edit_distance: int = 2 +) -> list[list[str]]: temp = [ str(i).split(",", maxsplit=1)[0].split(" ") for i in list( @@ -77,7 +78,7 @@ def spell_sent( return list_new -def correct_sent(list_words: List[str], max_edit_distance=1) -> List[str]: +def correct_sent(list_words: list[str], max_edit_distance=1) -> list[str]: return [ i[0] for i in spell_sent(list_words, max_edit_distance=max_edit_distance) diff --git a/pythainlp/spell/tltk.py b/pythainlp/spell/tltk.py index dd120aa3d..d21381675 100644 --- a/pythainlp/spell/tltk.py +++ b/pythainlp/spell/tltk.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,12 +10,16 @@ * \ https://pypi.org/project/tltk/ """ + +from __future__ import annotations + try: from tltk.nlp import spell_candidates except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") -from typing import List + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) -def spell(text: str) -> List[str]: +def spell(text: str) -> list[str]: return spell_candidates(text) diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 2707467f8..6cf81f878 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,7 +10,9 @@ * GitHub: \ https://github.com/bookpanda/Two-stage-Thai-Misspelling-Correction-Based-on-Pre-trained-Language-Models """ -from typing import List + +from __future__ import annotations + import torch from transformers import ( AutoModelForMaskedLM, @@ -21,28 +22,41 @@ use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") -tokenizer = AutoTokenizer.from_pretrained("airesearch/wangchanberta-base-att-spm-uncased") +tokenizer = AutoTokenizer.from_pretrained( + "airesearch/wangchanberta-base-att-spm-uncased" +) + class BertModel(torch.nn.Module): def __init__(self): super().__init__() - self.bert = BertForTokenClassification.from_pretrained('bookpanda/wangchanberta-base-att-spm-uncased-tagging') + self.bert = BertForTokenClassification.from_pretrained( + "bookpanda/wangchanberta-base-att-spm-uncased-tagging" + ) def forward(self, input_id, mask, label): - output = self.bert(input_ids=input_id, attention_mask=mask, labels=label, return_dict=False) + output = self.bert( + input_ids=input_id, + attention_mask=mask, + labels=label, + return_dict=False, + ) return output + tagging_model = BertModel() if use_cuda: tagging_model = tagging_model.to(device=device) -ids_to_labels = {0: 'f', 1: 'i'} +ids_to_labels = {0: "f", 1: "i"} -def align_word_ids(texts: str) -> List[int]: - tokenized_inputs = tokenizer(texts, padding='max_length', max_length=512, truncation=True) + +def align_word_ids(texts: str) -> list[int]: + tokenized_inputs = tokenizer( + texts, padding="max_length", max_length=512, truncation=True + ) word_ids = tokenized_inputs.word_ids() label_ids = [] for word_idx in word_ids: - if word_idx is None: label_ids.append(-100) else: @@ -53,10 +67,17 @@ def align_word_ids(texts: str) -> List[int]: return label_ids + def evaluate_one_text(model, sentence): - text = tokenizer(sentence, padding='max_length', max_length = 512, truncation=True, return_tensors="pt") - mask = text['attention_mask'][0].unsqueeze(0).to(device) - input_id = text['input_ids'][0].unsqueeze(0).to(device) + text = tokenizer( + sentence, + padding="max_length", + max_length=512, + truncation=True, + return_tensors="pt", + ) + mask = text["attention_mask"][0].unsqueeze(0).to(device) + input_id = text["input_ids"][0].unsqueeze(0).to(device) label_ids = torch.Tensor(align_word_ids(sentence)).unsqueeze(0).to(device) logits = tagging_model(input_id, mask, None) @@ -67,39 +88,53 @@ def evaluate_one_text(model, sentence): return prediction_label -mlm_model = AutoModelForMaskedLM.from_pretrained("bookpanda/wangchanberta-base-att-spm-uncased-masking") +mlm_model = AutoModelForMaskedLM.from_pretrained( + "bookpanda/wangchanberta-base-att-spm-uncased-masking" +) if use_cuda: mlm_model = mlm_model.to(device=device) + def correct(text: str) -> str: ans = [] i_f = evaluate_one_text(tagging_model, text) a = tokenizer(text) i_f_len = len(i_f) for j in range(i_f_len): - if i_f[j] == 'i': - ph = a['input_ids'][j+1] - a['input_ids'][j+1] = 25004 - b = {'input_ids': torch.Tensor([a['input_ids']]).type(torch.int64).to(device), 'attention_mask': torch.Tensor([a['attention_mask']]).type(torch.int64).to(device)} + if i_f[j] == "i": + ph = a["input_ids"][j + 1] + a["input_ids"][j + 1] = 25004 + b = { + "input_ids": torch.Tensor([a["input_ids"]]) + .type(torch.int64) + .to(device), + "attention_mask": torch.Tensor([a["attention_mask"]]) + .type(torch.int64) + .to(device), + } token_logits = mlm_model(**b).logits - mask_token_index = torch.where(b["input_ids"] == tokenizer.mask_token_id)[1] + mask_token_index = torch.where( + b["input_ids"] == tokenizer.mask_token_id + )[1] mask_token_logits = token_logits[0, mask_token_index, :] - top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist() + top_5_tokens = ( + torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist() + ) ans.append((j, top_5_tokens[0])) - text = ''.join(tokenizer.convert_ids_to_tokens(a['input_ids'])) - a['input_ids'][j+1] = ph - for x,y in ans: - a['input_ids'][x+1] = y - final_output = tokenizer.convert_ids_to_tokens(a['input_ids']) + text = "".join(tokenizer.convert_ids_to_tokens(a["input_ids"])) + a["input_ids"][j + 1] = ph + for x, y in ans: + a["input_ids"][x + 1] = y + final_output = tokenizer.convert_ids_to_tokens(a["input_ids"]) if "" in final_output: final_output.remove("") if "" in final_output: final_output.remove("") if "" in final_output: final_output.remove("") - if final_output[0] == '▁': + if final_output[0] == "▁": final_output.pop(0) - final_output = ''.join(final_output) + final_output = "".join(final_output) final_output = final_output.replace("▁", " ") final_output = final_output.replace("", "") return final_output diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index 10f0d7991..fa6a27ed6 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -1,24 +1,34 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import os + from pythainlp.corpus import get_hf_hub -from typing import List, Union class FastTextEncoder: """ - A class to load pre-trained FastText-like word embeddings, - compute word and sentence vectors, and interact with an ONNX + A class to load pre-trained FastText-like word embeddings, + compute word and sentence vectors, and interact with an ONNX model for nearest neighbor suggestions. """ # --- Initialization and Data Loading --- - - def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_words=2000000, minn=5, maxn=5): + + def __init__( + self, + model_dir, + nn_model_path, + words_list, + bucket=2000000, + nb_words=2000000, + minn=5, + maxn=5, + ): """ - Initializes the FastTextEncoder, loading embeddings, vocabulary, + Initializes the FastTextEncoder, loading embeddings, vocabulary, nearest neighbor model, and suggestion words list. Args: @@ -31,15 +41,16 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word maxn (int): Maximum character length for subwords. """ try: - import numpy as np # reduce load + import numpy as np # reduce load import onnxruntime + self.np = np except ModuleNotFoundError: raise ModuleNotFoundError(""" Please installing the package via 'pip install numpy onnxruntime'. """) except Exception as e: - raise Exception(f"An unexpected error occurred: {e}") + raise RuntimeError(f"An unexpected error occurred: {e}") from e self.model_dir = model_dir self.nn_model_path = nn_model_path self.bucket = bucket @@ -55,10 +66,12 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word def _load_embeddings(self): """Loads embeddings matrix and vocabulary list.""" - input_matrix = self.np.load(os.path.join(self.model_dir, "embeddings.npy")) + input_matrix = self.np.load( + os.path.join(self.model_dir, "embeddings.npy") + ) words = [] vocab_path = os.path.join(self.model_dir, "vocabulary.txt") - with open(vocab_path, "r", encoding='utf-8') as f: + with open(vocab_path, encoding="utf-8") as f: for line in f.readlines(): words.append(line.rstrip()) return words, input_matrix @@ -72,7 +85,10 @@ def _load_onnx_session(self, onnx_path): """Loads the ONNX inference session.""" # Note: Using providers=["CPUExecutionProvider"] for platform independence import onnxruntime as rt - sess = rt.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) + + sess = rt.InferenceSession( + onnx_path, providers=["CPUExecutionProvider"] + ) return sess # --- Helper Methods for Encoding --- @@ -103,9 +119,11 @@ def _get_subwords(self, word): for ngram_start in range(0, len(_word)): for ngram_length in range(self.minn, self.maxn + 1): if ngram_start + ngram_length <= len(_word): - _candidate_subword = _word[ngram_start:ngram_start + ngram_length] + _candidate_subword = _word[ + ngram_start : ngram_start + ngram_length + ] # Only append if not already included (e.g., as the full word) - if _candidate_subword not in _subwords: + if _candidate_subword not in _subwords: _subwords.append(_candidate_subword) _subword_ids.append(self._get_hash(_candidate_subword)) @@ -115,20 +133,22 @@ def get_word_vector(self, word): """Computes the normalized vector for a single word.""" # subword_ids[1] contains the array of indices for the word and its subwords subword_ids = self._get_subwords(word)[1] - + # Check if the array of subword indices is empty if subword_ids.size == 0: # Return a 300-dimensional zero vector if no word/subword is found. return self.np.zeros(self.embedding_dim) # Compute the mean of the embeddings for all subword indices - vector = self.np.mean([self.embeddings[s] for s in subword_ids], axis=0) - + vector = self.np.mean( + [self.embeddings[s] for s in subword_ids], axis=0 + ) + # Normalize the vector norm = self.np.linalg.norm(vector) if norm > 0: vector /= norm - + return vector def _tokenize(self, sentence): @@ -136,11 +156,11 @@ def _tokenize(self, sentence): tokens = [] word = "" for c in sentence: - if c in [' ', '\n', '\r', '\t', '\v', '\f', '\0']: + if c in [" ", "\n", "\r", "\t", "\v", "\f", "\0"]: if word: tokens.append(word) word = "" - if c == '\n': + if c == "\n": tokens.append("") else: word += c @@ -156,7 +176,7 @@ def get_sentence_vector(self, line): # get_word_vector already handles normalization, so no need to do it again here vec = self.get_word_vector(t) vectors.append(vec) - + # If the sentence was empty and resulted in no vectors, return a zero vector if not vectors: return self.np.zeros(self.embedding_dim) @@ -167,15 +187,15 @@ def get_sentence_vector(self, line): def get_word_suggestion(self, list_word): """ - Queries the ONNX model to find the nearest neighbor word(s) + Queries the ONNX model to find the nearest neighbor word(s) for the given word or list of words. Args: - list_word (str or list of str): A single word or a list of words + list_word (str or list of str): A single word or a list of words to get suggestions for. Returns: - str or list of str: The nearest neighbor word(s) from the + str or list of str: The nearest neighbor word(s) from the pre-loaded suggestion list. """ if isinstance(list_word, str): @@ -184,23 +204,26 @@ def get_word_suggestion(self, list_word): else: input_words = list_word return_single = False - + # Compute sentence vector for each input word/phrase - # The original code's `get_sentence_vector(' '.join(list(word)))` seems - # intended to treat a list of characters/tokens as a sentence. - # I'll stick to a more standard usage: treat each item in `input_words` + # The original code's `get_sentence_vector(' '.join(list(word)))` seems + # intended to treat a list of characters/tokens as a sentence. + # I'll stick to a more standard usage: treat each item in `input_words` # as a separate phrase/word to encode. - word_input_vecs = [self.get_sentence_vector(' '.join(list(word))) for word in input_words] + word_input_vecs = [ + self.get_sentence_vector(" ".join(list(word))) + for word in input_words + ] # Convert to numpy array for ONNX input (ensure float32) input_data = self.np.array(word_input_vecs, dtype=self.np.float32) # Run ONNX inference indices = self.nn_session.run(None, {"X": input_data})[0] - + # Look up suggestions suggestions = [self.words_for_suggestion[i].tolist() for i in indices] - + return suggestions[0] if return_single else suggestions @@ -209,7 +232,11 @@ def __init__(self): self.model_name = "pythainlp/word-spelling-correction-char2vec" self.model_path = get_hf_hub(self.model_name) self.model_onnx = get_hf_hub(self.model_name, "nearest_neighbors.onnx") - with open(get_hf_hub(self.model_name, "list_word-spelling-correction-char2vec.txt")) as f: + with open( + get_hf_hub( + self.model_name, "list_word-spelling-correction-char2vec.txt" + ) + ) as f: self.list_word = [i.strip() for i in f.readlines()] super().__init__(self.model_path, self.model_onnx, self.list_word) @@ -217,7 +244,9 @@ def __init__(self): _WSC = None -def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[str], List[List[str]]]: +def get_words_spell_suggestion( + list_words: str | list[str], +) -> list[str] | list[list[str]]: """ Get words spell suggestion @@ -243,6 +272,6 @@ def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[ # ['กระเพาะ', 'กระพา', 'กะเพรา', 'กระเพาะปลา', 'พระประธาน']] """ global _WSC - if _WSC==None: + if _WSC is None: _WSC = Words_Spelling_Correction() return _WSC.get_word_suggestion(list_words) diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py index 70c1e7e97..b73c8ffff 100644 --- a/pythainlp/summarize/__init__.py +++ b/pythainlp/summarize/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index a72959a85..49cb824da 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,7 +5,9 @@ Text summarization and keyword extraction """ -from typing import Iterable, List, Optional, Tuple +from __future__ import annotations + +from collections.abc import Iterable from pythainlp.summarize import ( CPE_KMUTT_THAI_SENTENCE_SUM, @@ -22,7 +23,7 @@ def summarize( n: int = 1, engine: str = DEFAULT_SUMMARIZE_ENGINE, tokenizer: str = "newmm", -) -> List[str]: +) -> list[str]: """ This function summarizes text based on frequency of words. @@ -119,13 +120,13 @@ def summarize( def extract_keywords( text: str, - keyphrase_ngram_range: Tuple[int, int] = (1, 2), + keyphrase_ngram_range: tuple[int, int] = (1, 2), max_keywords: int = 5, min_df: int = 1, engine: str = DEFAULT_KEYWORD_EXTRACTION_ENGINE, tokenizer: str = "newmm", - stop_words: Optional[Iterable[str]] = None, -) -> List[str]: + stop_words: Iterable[str] | None = None, +) -> list[str]: """ This function returns most-relevant keywords (and/or keyphrases) from the input document. Each algorithm may produce completely different keywords from each other, @@ -197,7 +198,7 @@ def rank_by_frequency( max_keywords: int = 5, min_df: int = 5, tokenizer: str = "newmm", - stop_words: Optional[Iterable[str]] = None, + stop_words: Iterable[str] | None = None, ): from pythainlp.tokenize import word_tokenize from pythainlp.util.keywords import rank diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index 3adc828f2..93653274d 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Summarization by frequency of words """ + +from __future__ import annotations + from collections import defaultdict from heapq import nlargest from string import punctuation -from typing import List from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import sent_tokenize, word_tokenize @@ -27,7 +28,7 @@ def __rank(ranking, n: int): return nlargest(n, ranking, key=ranking.get) def __compute_frequencies( - self, word_tokenized_sents: List[List[str]] + self, word_tokenized_sents: list[list[str]] ) -> defaultdict: word_freqs = defaultdict(int) for sent in word_tokenized_sents: @@ -48,7 +49,7 @@ def __compute_frequencies( def summarize( self, text: str, n: int, tokenizer: str = "newmm" - ) -> List[str]: + ) -> list[str]: sents = sent_tokenize(text, engine="whitespace+newline") word_tokenized_sents = [ word_tokenize(sent, engine=tokenizer) for sent in sents diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index acfd981c3..f589e1815 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,8 +10,11 @@ https://github.com/MaartenGr/KeyBERT """ + +from __future__ import annotations + from collections import Counter -from typing import Iterable, List, Optional, Tuple, Union +from collections.abc import Iterable import numpy as np from transformers import pipeline @@ -35,13 +37,13 @@ def __init__( def extract_keywords( self, text: str, - keyphrase_ngram_range: Tuple[int, int] = (1, 2), + keyphrase_ngram_range: tuple[int, int] = (1, 2), max_keywords: int = 5, min_df: int = 1, tokenizer: str = "newmm", return_similarity=False, - stop_words: Optional[Iterable[str]] = None, - ) -> Union[List[str], List[Tuple[str, float]]]: + stop_words: Iterable[str] | None = None, + ) -> list[str] | list[tuple[str, float]]: """ Extract Thai keywords and/or keyphrases with KeyBERT algorithm. See https://github.com/MaartenGr/KeyBERT. @@ -87,7 +89,9 @@ def extract_keywords( # 'ควบคุมการเปลี่ยนแปลง', # 'มีพิษ'] - keywords = kb.extract_keyword(text, max_keywords=10, return_similarity=True) + keywords = kb.extract_keyword( + text, max_keywords=10, return_similarity=True + ) # output: [('อวัยวะต่างๆ', 0.3228477063109462), # ('ซ่อมแซมส่วน', 0.31320597838000375), @@ -132,7 +136,7 @@ def extract_keywords( else: return [kw for kw, _ in keywords] - def embed(self, docs: Union[str, List[str]]) -> np.ndarray: + def embed(self, docs: str | list[str]) -> np.ndarray: """ Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. """ @@ -152,11 +156,11 @@ def embed(self, docs: Union[str, List[str]]) -> np.ndarray: def _generate_ngrams( doc: str, - keyphrase_ngram_range: Tuple[int, int], + keyphrase_ngram_range: tuple[int, int], min_df: int, tokenizer_engine: str, stop_words: Iterable[str], -) -> List[str]: +) -> list[str]: assert keyphrase_ngram_range[0] >= 1, ( f"`keyphrase_ngram_range` must start from 1. " f"current value={keyphrase_ngram_range}." @@ -167,7 +171,7 @@ def _generate_ngrams( f"current value={keyphrase_ngram_range}." ) - def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]: + def _join_ngram(ngrams: list[tuple[str, str]]) -> list[str]: ngrams_joined = [] for ng in ngrams: joined = "".join(ng) @@ -201,18 +205,18 @@ def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]: def _rank_keywords( doc_vector: np.ndarray, word_vectors: np.ndarray, - keywords: List[str], + keywords: list[str], max_keywords: int, -) -> List[Tuple[str, float]]: +) -> list[tuple[str, float]]: def l2_norm(v: np.ndarray) -> np.ndarray: vec_size = v.shape[1] result = np.divide( v, np.linalg.norm(v, axis=1).reshape(-1, 1).repeat(vec_size, axis=1), ) - assert np.isclose( - np.linalg.norm(result, axis=1), 1 - ).all(), "Cannot normalize a vector to unit vector." + assert np.isclose(np.linalg.norm(result, axis=1), 1).all(), ( + "Cannot normalize a vector to unit vector." + ) return result def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index bc5e572ac..cb0727804 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Summarization by mT5 model """ -from typing import List + +from __future__ import annotations from transformers import MT5ForConditionalGeneration, T5Tokenizer @@ -45,7 +45,7 @@ def __init__( self.max_length = max_length self.skip_special_tokens = skip_special_tokens - def summarize(self, text: str) -> List[str]: + def summarize(self, text: str) -> list[str]: preprocess_text = text.strip().replace("\n", "") if self.model_name == f"thanathorn/{CPE_KMUTT_THAI_SENTENCE_SUM}": t5_prepared_Text = "simplify: " + preprocess_text diff --git a/pythainlp/tag/__init__.py b/pythainlp/tag/__init__.py index c7ed6412f..891ddd8d2 100644 --- a/pythainlp/tag/__init__.py +++ b/pythainlp/tag/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index 8a6a07701..bf1fdf685 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -17,12 +16,15 @@ This tagger is provided under the terms of the MIT License. """ + +from __future__ import annotations + import json from collections import defaultdict -from typing import Dict, Iterable, List, Tuple, Union +from collections.abc import Iterable -class AveragedPerceptron(): +class AveragedPerceptron: """ An averaged perceptron, as implemented by Matthew Honnibal. @@ -45,7 +47,7 @@ def __init__(self) -> None: # Number of instances seen self.i = 0 - def predict(self, features: Dict): + def predict(self, features: dict): """ Dot-product the features and current weights and return the best label. @@ -60,7 +62,7 @@ def predict(self, features: Dict): # Do a secondary alphabetic sort, for stability return max(self.classes, key=lambda label: (scores[label], label)) - def update(self, truth, guess, features: Dict) -> None: + def update(self, truth, guess, features: dict) -> None: """Update the feature weights.""" def upd_feat(c, f, w, v): @@ -128,7 +130,7 @@ def __init__(self, path: str = "") -> None: self.AP_MODEL_LOC = path self.load(self.AP_MODEL_LOC) - def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]: + def tag(self, tokens: Iterable[str]) -> list[tuple[str, str]]: """Tags a string `tokens`.""" prev, prev2 = self.START output = [] @@ -146,8 +148,8 @@ def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]: def train( self, - sentences: Iterable[Iterable[Tuple[str, str]]], - save_loc: Union[str, None] = None, + sentences: Iterable[Iterable[tuple[str, str]]], + save_loc: str | None = None, nr_iter: int = 5, ) -> None: """ @@ -203,11 +205,11 @@ def load(self, loc: str) -> None: :param str loc: model path """ try: - with open(loc, "r", encoding="utf-8-sig") as f: + with open(loc, encoding="utf-8-sig") as f: w_td_c = json.load(f) - except IOError: + except OSError: msg = "Missing trontagger.json file." - raise IOError(msg) + raise OSError(msg) self.model.weights = w_td_c["weights"] self.tagdict = w_td_c["tagdict"] self.classes = w_td_c["classes"] @@ -233,8 +235,8 @@ def _normalize(self, word: str) -> str: return word.lower() def _get_features( - self, i: int, word: str, context: List[str], prev: str, prev2: str - ) -> Dict: + self, i: int, word: str, context: list[str], prev: str, prev2: str + ) -> dict: """ Map tokens into a feature representation, implemented as a {hashable: float} dict. If the features change, a new model must be @@ -265,7 +267,7 @@ def add(name: str, *args): return features def _make_tagdict( - self, sentences: Iterable[Iterable[Tuple[str, str]]] + self, sentences: Iterable[Iterable[tuple[str, str]]] ) -> None: """Make a tag dictionary for single-tag words.""" counts = defaultdict(lambda: defaultdict(int)) diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index afa771087..a4b4b8ee9 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations # defined strings for special characters CHAR_TO_ESCAPE = {" ": "_"} @@ -32,7 +31,7 @@ } -def pre_process(words: List[str]) -> List[str]: +def pre_process(words: list[str]) -> list[str]: """ Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, @@ -44,8 +43,8 @@ def pre_process(words: List[str]) -> List[str]: def post_process( - word_tags: List[Tuple[str, str]], to_ud: bool = False -) -> List[Tuple[str, str]]: + word_tags: list[tuple[str, str]], to_ud: bool = False +) -> list[tuple[str, str]]: """ Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, diff --git a/pythainlp/tag/chunk.py b/pythainlp/tag/chunk.py index cd5722b7f..c950636b0 100644 --- a/pythainlp/tag/chunk.py +++ b/pythainlp/tag/chunk.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations def chunk_parse( - sent: List[Tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp" -) -> List[str]: + sent: list[tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp" +) -> list[str]: """ This function parses Thai sentence to phrase structure in IOB format. diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index f8dfbc950..c04a8f52a 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from __future__ import annotations from pycrfsuite import Tagger as CRFTagger @@ -13,7 +12,7 @@ def _is_stopword(word: str) -> bool: # check Thai stopword return word in thai_stopwords() -def _doc2features(tokens: List[Tuple[str, str]], index: int) -> Dict: +def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict: """ `tokens` = a POS-tagged sentence [(w1, t1), ...] `index` = the index of the token we want to extract features for @@ -67,6 +66,6 @@ def load_model(self, corpus: str): self.path = path_pythainlp_corpus("crfchunk_orchidpp.model") self.tagger.open(self.path) - def parse(self, token_pos: List[Tuple[str, str]]) -> List[str]: + def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) return self.tagger.tag(self.xseq) diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py index b70072ba5..f4753e54a 100644 --- a/pythainlp/tag/locations.py +++ b/pythainlp/tag/locations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,12 +5,12 @@ Recognizes locations in text """ -from typing import List, Tuple +from __future__ import annotations from pythainlp.corpus import provinces -def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]: +def tag_provinces(tokens: list[str]) -> list[tuple[str, str]]: """ This function recognizes Thailand provinces in text. @@ -26,7 +25,7 @@ def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]: from pythainlp.tag import tag_provinces - text = ['หนองคาย', 'น่าอยู่'] + text = ["หนองคาย", "น่าอยู่"] tag_provinces(text) # output: [('หนองคาย', 'B-LOCATION'), ('น่าอยู่', 'O')] """ diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index e9628a7b4..4639dafbb 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Named-entity recognizer """ -from typing import List, Tuple, Union + +from __future__ import annotations class NER: @@ -26,7 +26,9 @@ class NER: **Note**: The tltk engine supports NER models from tltk only. """ - def __init__(self, engine: str = "thainer-v2", corpus: str = "thainer") -> None: + def __init__( + self, engine: str = "thainer-v2", corpus: str = "thainer" + ) -> None: self.load_engine(engine=engine, corpus=corpus) def load_engine(self, engine: str, corpus: str) -> None: @@ -38,7 +40,10 @@ def load_engine(self, engine: str, corpus: str) -> None: self.engine = ThaiNameTagger() elif engine == "thainer-v2" and corpus == "thainer": from pythainlp.wangchanberta import NamedEntityRecognition - self.engine = NamedEntityRecognition(model="pythainlp/thainer-corpus-v2-base-model") + + self.engine = NamedEntityRecognition( + model="pythainlp/thainer-corpus-v2-base-model" + ) elif engine == "tltk": from pythainlp.tag import tltk @@ -53,16 +58,12 @@ def load_engine(self, engine: str, corpus: str) -> None: self.engine = NamedEntityTagger() else: raise ValueError( - "NER class not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"NER class not support {engine} engine or {corpus} corpus." ) - def tag(self, - text, - pos=False, - tag=False - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: + def tag( + self, text, pos=False, tag=False + ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: """ This function tags named entities in text in IOB format. @@ -116,7 +117,7 @@ def load_engine(self, engine: str = "thai_nner") -> None: self.engine = Thai_NNER() - def tag(self, text) -> Tuple[List[str], List[dict]]: + def tag(self, text) -> tuple[list[str], list[dict]]: """ This function tags nested named entities. diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 678c7a37e..46e0f1005 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Data preprocessing for ORCHID corpus """ -from typing import List, Tuple + +from __future__ import annotations # defined strings for special characters, # from Table 4 in ORCHID paper @@ -126,7 +126,7 @@ def ud_exception(w: str, tag: str) -> str: return tag -def pre_process(words: List[str]) -> List[str]: +def pre_process(words: list[str]) -> list[str]: """ Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, @@ -138,8 +138,8 @@ def pre_process(words: List[str]) -> List[str]: def post_process( - word_tags: List[Tuple[str, str]], to_ud: bool = False -) -> List[Tuple[str, str]]: + word_tags: list[tuple[str, str]], to_ud: bool = False +) -> list[tuple[str, str]]: """ Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index 71e9b720e..2219fdd50 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,8 +5,9 @@ Perceptron part-of-speech tagger """ +from __future__ import annotations + import os -from typing import List, Tuple from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import PerceptronTagger, blackboard, orchid @@ -69,7 +69,7 @@ def _tud_tagger(): return _TUD_TAGGER -def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]: +def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: """ :param list words: a list of tokenized words :param str corpus: corpus name (orchid, pud) diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index a83a3330f..eb1d8e385 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations def pos_tag( - words: List[str], engine: str = "perceptron", corpus: str = "orchid" -) -> List[Tuple[str, str]]: + words: list[str], engine: str = "perceptron", corpus: str = "orchid" +) -> list[tuple[str, str]]: """ Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'. @@ -117,9 +116,7 @@ def pos_tag( from pythainlp.tag.unigram import tag as tag_ else: raise ValueError( - "pos_tag not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"pos_tag not support {engine} engine or {corpus} corpus." ) word_tags = tag_(words, corpus=corpus) @@ -128,10 +125,10 @@ def pos_tag( def pos_tag_sents( - sentences: List[List[str]], + sentences: list[list[str]], engine: str = "perceptron", corpus: str = "orchid", -) -> List[List[Tuple[str, str]]]: +) -> list[list[tuple[str, str]]]: """ Marks sentences with part-of-speech (POS) tags. @@ -180,7 +177,7 @@ def pos_tag_transformers( sentence: str, engine: str = "bert", corpus: str = "blackboard", -) -> List[List[Tuple[str, str]]]: +) -> list[list[tuple[str, str]]]: """ Marks sentences with part-of-speech (POS) tags. @@ -245,15 +242,14 @@ def pos_tag_transformers( tokenizer = AutoTokenizer.from_pretrained(base_model) else: raise ValueError( - "pos_tag_transformers not support {0} engine or {1} corpus.".format( - engine, corpus - ) + f"pos_tag_transformers not support {engine} engine or {corpus} corpus." ) - pipeline = TokenClassificationPipeline(model=model, - tokenizer=tokenizer, - aggregation_strategy="simple", - ) + pipeline = TokenClassificationPipeline( + model=model, + tokenizer=tokenizer, + aggregation_strategy="simple", + ) outputs = pipeline(sentence) word_tags = [[(tag["word"], tag["entity_group"]) for tag in outputs]] diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index ad0a2623a..09432e7f1 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from thai_nner import NNER @@ -13,5 +12,5 @@ class Thai_NNER: def __init__(self, path_model=get_corpus_path("thai_nner", "1.0")) -> None: self.model = NNER(path_model=path_model) - def tag(self, text) -> Tuple[List[str], List[dict]]: + def tag(self, text) -> tuple[list[str], list[dict]]: return self.model.get_tag(text) diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index 8558eef67..c432679d2 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,9 +5,10 @@ Named-entity recognizer """ +from __future__ import annotations + __all__ = ["ThaiNameTagger"] -from typing import Dict, List, Tuple, Union from pythainlp.corpus import get_corpus_path, thai_stopwords from pythainlp.tag import pos_tag @@ -22,7 +22,7 @@ def _is_stopword(word: str) -> bool: # เช็คว่าเป็นคำ return word in thai_stopwords() -def _doc2features(doc, i) -> Dict: +def _doc2features(doc, i) -> dict: word = doc[i][0] postag = doc[i][1] @@ -111,7 +111,7 @@ def __init__(self, version: str = "1.4") -> None: def get_ner( self, text: str, pos: bool = True, tag: bool = False - ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]]]: + ) -> list[tuple[str, str]] | list[tuple[str, str, str]]: """ This function tags named-entities in text in IOB format. diff --git a/pythainlp/tag/tltk.py b/pythainlp/tag/tltk.py index f957a1832..fdb65180d 100644 --- a/pythainlp/tag/tltk.py +++ b/pythainlp/tag/tltk.py @@ -1,22 +1,23 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple, Union +from __future__ import annotations try: from tltk import nlp except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) from pythainlp.tokenize import word_tokenize nlp.pos_load() nlp.ner_load() -def pos_tag(words: List[str], corpus: str = "tnc") -> List[Tuple[str, str]]: +def pos_tag(words: list[str], corpus: str = "tnc") -> list[tuple[str, str]]: if corpus != "tnc": - raise ValueError("tltk not support {0} corpus.".format(0)) + raise ValueError(f"tltk not support {0} corpus.") return nlp.pos_tag_wordlist(words) @@ -26,7 +27,7 @@ def _post_process(text: str) -> str: def get_ner( text: str, pos: bool = True, tag: bool = False -) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]: +) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: """ Named-entity recognizer from **TLTK** diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 34384072b..23537eeec 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Unigram Part-Of-Speech tagger """ + +from __future__ import annotations + import json import os -from typing import List, Tuple from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import blackboard, orchid @@ -75,8 +76,8 @@ def _tud_tagger(): def _find_tag( - words: List[str], dictdata: dict, default_tag: str = "" -) -> List[Tuple[str, str]]: + words: list[str], dictdata: dict, default_tag: str = "" +) -> list[tuple[str, str]]: keys = list(dictdata.keys()) return [ (word, dictdata[word]) if word in keys else (word, default_tag) @@ -84,7 +85,7 @@ def _find_tag( ] -def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]: +def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: """ :param list words: a list of tokenized words :param str corpus: corpus name (orchid or pud) diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py index c3744d1a1..ccc12f15a 100644 --- a/pythainlp/tag/wangchanberta_onnx.py +++ b/pythainlp/tag/wangchanberta_onnx.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import json -from typing import List import numpy as np @@ -16,7 +16,7 @@ def __init__( model_name: str, model_version: str, file_onnx: str, - providers: List[str] = ["CPUExecutionProvider"], + providers: list[str] = ["CPUExecutionProvider"], ) -> None: import sentencepiece as spm from onnxruntime import ( diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 8c64cecc0..865bc0aa9 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -20,6 +19,7 @@ ] from functools import lru_cache + from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie @@ -28,24 +28,27 @@ DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo" + @lru_cache def word_dict_trie(): """Lazy load default word dict trie with cache""" return Trie(thai_words()) + @lru_cache def syllable_dict_trie(): """Lazy load default syllable dict trie with cache""" return Trie(thai_syllables()) + from pythainlp.tokenize.core import ( Tokenizer, + display_cell_tokenize, paragraph_tokenize, sent_tokenize, subword_tokenize, syllable_tokenize, word_detokenize, word_tokenize, - display_cell_tokenize, ) from pythainlp.tokenize.thai2fit import thai2fit_tokenizer diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index 463b45fb0..8df8f53c0 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,15 +5,17 @@ Utility functions for tokenize module. """ +from __future__ import annotations + import re -from typing import Callable, List +from collections.abc import Callable _DIGITS_WITH_SEPARATOR = re.compile(r"(\d+[\.\,:])+\d+") def apply_postprocessors( - segments: List[str], postprocessors: Callable[[List[str]], List[str]] -) -> List[str]: + segments: list[str], postprocessors: Callable[[list[str]], list[str]] +) -> list[str]: """ A list of callables to apply to a raw segmentation result. """ @@ -24,7 +25,7 @@ def apply_postprocessors( return segments -def rejoin_formatted_num(segments: List[str]) -> List[str]: +def rejoin_formatted_num(segments: list[str]) -> list[str]: """ Rejoin well-known formatted numeric that are over-tokenized. The formatted numeric are numbers separated by ":", ",", or ".", @@ -73,7 +74,7 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]: return tokens_joined -def strip_whitespace(segments: List[str]) -> List[str]: +def strip_whitespace(segments: list[str]) -> list[str]: """ Strip whitespace(s) off each token and remove whitespace tokens. :param List[str] segments: result from word tokenizer diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py index e5c59c44f..169c750cd 100644 --- a/pythainlp/tokenize/attacut.py +++ b/pythainlp/tokenize/attacut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -* # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -8,7 +7,8 @@ :See Also: * `GitHub repository `_ """ -from typing import Dict, List + +from __future__ import annotations from attacut import Tokenizer @@ -22,14 +22,14 @@ def __init__(self, model="attacut-sc"): self._tokenizer = Tokenizer(model=self._MODEL_NAME) - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: return self._tokenizer.tokenize(text) -_tokenizers: Dict[str, AttacutTokenizer] = {} +_tokenizers: dict[str, AttacutTokenizer] = {} -def segment(text: str, model: str = "attacut-sc") -> List[str]: +def segment(text: str, model: str = "attacut-sc") -> list[str]: """ Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai :param str text: text to be tokenized to words diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py index 5237da2a9..74e46b9c0 100644 --- a/pythainlp/tokenize/budoux.py +++ b/pythainlp/tokenize/budoux.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,7 +10,8 @@ used and `budoux` is missing, a clear ImportError is raised with an installation hint. """ -from typing import List + +from __future__ import annotations _parser = None @@ -32,7 +32,7 @@ def _init_parser(): return budoux.load_default_thai_parser() -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """Segment `text` into tokens using budoux. The function returns a list of strings. If `budoux` is not available diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index a6cdaf737..234be457e 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,17 +5,19 @@ Generic functions of tokenizers """ +from __future__ import annotations + import copy import re -from typing import Iterable, List, Optional, Union +from collections.abc import Iterable from pythainlp.tokenize import ( DEFAULT_SENT_TOKENIZE_ENGINE, DEFAULT_SUBWORD_TOKENIZE_ENGINE, - syllable_dict_trie, DEFAULT_SYLLABLE_TOKENIZE_ENGINE, - word_dict_trie, DEFAULT_WORD_TOKENIZE_ENGINE, + syllable_dict_trie, + word_dict_trie, ) from pythainlp.tokenize._utils import ( apply_postprocessors, @@ -27,8 +28,8 @@ def word_detokenize( - segments: Union[List[List[str]], List[str]], output: str = "str" -) -> Union[List[str], str]: + segments: list[list[str]] | list[str], output: str = "str" +) -> list[str] | str: """ Word detokenizer. @@ -97,11 +98,11 @@ def word_detokenize( def word_tokenize( text: str, - custom_dict: Optional[Trie] = None, + custom_dict: Trie | None = None, engine: str = DEFAULT_WORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, join_broken_num: bool = True, -) -> List[str]: +) -> list[str]: """ Word tokenizer. @@ -361,10 +362,10 @@ def map_indices_to_words(index_list, sentences): def sent_tokenize( - text: Union[str, List[str]], + text: str | list[str], engine: str = DEFAULT_SENT_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: +) -> list[str]: """ Sentence tokenizer. @@ -532,7 +533,7 @@ def paragraph_tokenize( engine: str = "wtp-mini", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[List[str]]: +) -> list[list[str]]: """ Paragraph tokenizer. @@ -600,7 +601,7 @@ def subword_tokenize( text: str, engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: +) -> list[str]: """ Subword tokenizer for tokenizing text into units smaller than syllables. @@ -692,9 +693,7 @@ def subword_tokenize( words = word_tokenize(text) for word in words: segments.extend( - word_tokenize( - text=word, custom_dict=syllable_dict_trie() - ) + word_tokenize(text=word, custom_dict=syllable_dict_trie()) ) elif engine == "ssg": from pythainlp.tokenize.ssg import segment @@ -723,7 +722,7 @@ def syllable_tokenize( text: str, engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE, keep_whitespace: bool = True, -) -> List[str]: +) -> list[str]: """ Syllable tokenizer @@ -755,7 +754,7 @@ def syllable_tokenize( ) -def display_cell_tokenize(text: str) -> List[str]: +def display_cell_tokenize(text: str) -> list[str]: """ Display cell tokenizer. @@ -864,7 +863,7 @@ class Tokenizer: def __init__( self, - custom_dict: Union[Trie, Iterable[str], str] = [], + custom_dict: Trie | Iterable[str] | str = [], engine: str = "newmm", keep_whitespace: bool = True, join_broken_num: bool = True, @@ -896,7 +895,7 @@ def __init__( self.__keep_whitespace = keep_whitespace self.__join_broken_num = join_broken_num - def word_tokenize(self, text: str) -> List[str]: + def word_tokenize(self, text: str) -> list[str]: """ Main tokenization function. diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 9746d0113..19bf255e6 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -17,8 +16,9 @@ POS features are not used due to unreliable POS tagging available """ +from __future__ import annotations + import os -from typing import List import pycrfsuite @@ -128,8 +128,8 @@ def extract_features( - doc: List[str], window: int = 2, max_n_gram: int = 3 -) -> List[List[str]]: + doc: list[str], window: int = 2, max_n_gram: int = 3 +) -> list[list[str]]: """ Extract features for CRF by sliding `max_n_gram` of tokens for +/- `window` from the current token @@ -168,12 +168,12 @@ def extract_features( # ngram features for n_gram in range(1, min(max_n_gram + 1, 2 + window * 2)): for j in range(i - window, i + window + 2 - n_gram): - feature_position = f"{n_gram}_{j-i}_{j-i+n_gram}" - word_ = f'{"|".join(doc[j:(j+n_gram)])}' + feature_position = f"{n_gram}_{j - i}_{j - i + n_gram}" + word_ = f"{'|'.join(doc[j : (j + n_gram)])}" word_features += [f"word_{feature_position}={word_}"] - ender_ = f'{"|".join(doc_ender[j:(j+n_gram)])}' + ender_ = f"{'|'.join(doc_ender[j : (j + n_gram)])}" word_features += [f"ender_{feature_position}={ender_}"] - starter_ = f'{"|".join(doc_starter[j:(j+n_gram)])}' + starter_ = f"{'|'.join(doc_starter[j : (j + n_gram)])}" word_features += [f"starter_{feature_position}={starter_}"] # append to feature per word doc_features.append(word_features) @@ -186,7 +186,7 @@ def extract_features( _tagger.open(os.path.join(corpus_path(), _CRFCUT_DATA_FILENAME)) -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ CRF-based sentence segmentation. @@ -206,7 +206,7 @@ def segment(text: str) -> List[str]: if toks[idx].strip().endswith(("!", ".", "?")): labs[idx] = "E" # Spaces or empty strings would no longer be treated as end of sentence. - elif (idx == 0 or labs[idx-1] == "E") and toks[idx].strip() == "": + elif (idx == 0 or labs[idx - 1] == "E") and toks[idx].strip() == "": labs[idx] = "I" sentences = [] diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py index 4b008de40..0087fa10f 100644 --- a/pythainlp/tokenize/deepcut.py +++ b/pythainlp/tokenize/deepcut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -12,7 +11,7 @@ * `GitHub repository `_ """ -from typing import List, Union +from __future__ import annotations try: from deepcut import tokenize @@ -21,9 +20,7 @@ from pythainlp.util import Trie -def segment( - text: str, custom_dict: Union[Trie, List[str], str] = [] -) -> List[str]: +def segment(text: str, custom_dict: Trie | list[str] | str = []) -> list[str]: if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 557be3cef..cb0563ab1 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -19,24 +18,28 @@ and backward longest matching techniques." In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001. """ -from functools import lru_cache + +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp import thai_follow_vowels from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer + @lru_cache def _cut_etcc(): """Lazy load ETCC tokenizer with cache""" return Tokenizer(get_corpus("etcc.txt"), engine="longest") + _PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]" _RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR) -def _cut_subword(tokens: List[str]) -> List[str]: +def _cut_subword(tokens: list[str]) -> list[str]: len_tokens = len(tokens) i = 0 while True: @@ -50,7 +53,7 @@ def _cut_subword(tokens: List[str]) -> List[str]: return tokens -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ Segmenting text into ETCCs. diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 30d8180bc..eacfb1494 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2019 Ponrawee Prasertsom # SPDX-License-Identifier: Apache-2.0 @@ -7,7 +6,8 @@ GitHub: https://github.com/PyThaiNLP/Han-solo """ -from typing import List + +from __future__ import annotations from pythainlp.corpus import path_pythainlp_corpus @@ -119,7 +119,7 @@ def featurize( _to_feature = Featurizer() -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: x = _to_feature.featurize(text)["X"] y_pred = tagger.tag(x) list_cut = [] diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index 969f47271..b777ca126 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,8 +10,10 @@ `_ """ + +from __future__ import annotations + import re -from typing import Dict, List, Optional, Union from pythainlp import thai_tonemarks from pythainlp.tokenize import word_dict_trie @@ -48,7 +49,7 @@ def __init__(self, trie: Trie): self.__trie = trie @staticmethod - def __search_nonthai(text: str) -> Union[None, str]: + def __search_nonthai(text: str) -> None | str: match = _RE_NONTHAI.search(text) if match.group(0): return match.group(0).lower() @@ -137,22 +138,26 @@ def __segment(self, text: str): # Group consecutive spaces into one token grouped_tokens = [] for token in tokens: - if token.isspace() and grouped_tokens and grouped_tokens[-1].isspace(): + if ( + token.isspace() + and grouped_tokens + and grouped_tokens[-1].isspace() + ): grouped_tokens[-1] += token else: grouped_tokens.append(token) return grouped_tokens - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: tokens = self.__segment(text) return tokens -_tokenizers: Dict[int, LongestMatchTokenizer] = {} +_tokenizers: dict[int, LongestMatchTokenizer] = {} -def segment(text: str, custom_dict: Optional[Trie] = None) -> List[str]: +def segment(text: str, custom_dict: Trie | None = None) -> list[str]: """ Dictionary-based longest matching word segmentation. diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index 2ac0b0b49..d6f9c7840 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,9 +12,11 @@ `_ """ +from __future__ import annotations + import re from collections import defaultdict -from typing import Iterator, List, Optional +from collections.abc import Iterator from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie @@ -48,7 +49,7 @@ def __init__(self, value, multi=None, in_dict=True): def _multicut( - text: str, custom_dict: Optional[Trie] = None + text: str, custom_dict: Trie | None = None ) -> Iterator[LatticeString]: """Return LatticeString""" if not custom_dict: @@ -100,7 +101,7 @@ def serialize(p, p2): # helper function q.add(i) -def mmcut(text: str) -> List[str]: +def mmcut(text: str) -> list[str]: res = [] for w in _multicut(text): mm = min(w.multi, key=lambda x: x.count("/")) @@ -108,7 +109,7 @@ def mmcut(text: str) -> List[str]: return res -def _combine(ww: List[LatticeString]) -> Iterator[str]: +def _combine(ww: list[LatticeString]) -> Iterator[str]: if ww == []: yield "" else: @@ -121,9 +122,7 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]: yield m.replace("/", "|") + "|" + tail -def segment( - text: str, custom_dict: Optional[Trie] = None -) -> List[str]: +def segment(text: str, custom_dict: Trie | None = None) -> list[str]: """Dictionary-based maximum matching word segmentation. :param text: text to be tokenized @@ -143,9 +142,7 @@ def segment( return list(_multicut(text, custom_dict=custom_dict)) -def find_all_segment( - text: str, custom_dict: Optional[Trie] = None -) -> List[str]: +def find_all_segment(text: str, custom_dict: Trie | None = None) -> list[str]: """Get all possible segment variations. :param text: input string to be tokenized diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py index 1ac0fa8ab..75e42c17f 100644 --- a/pythainlp/tokenize/nercut.py +++ b/pythainlp/tokenize/nercut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,7 +10,10 @@ Code by Wannaphong Phatthiyaphaibun """ -from typing import Iterable, List + +from __future__ import annotations + +from collections.abc import Iterable from pythainlp.tag.named_entity import NER @@ -29,7 +31,7 @@ def segment( "TIME", ], tagger=_thainer, -) -> List[str]: +) -> list[str]: """ Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index ec30356e5..58dd70080 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -15,10 +14,13 @@ * \ https://colab.research.google.com/drive/14Ibg-ngZXj15RKwjNwoZlOT32fQBOrBx#scrollTo=MYZ7NzAR7Dmw """ + +from __future__ import annotations + import re from collections import defaultdict +from collections.abc import Generator from heapq import heappop, heappush -from typing import Generator, List, Optional from pythainlp.tokenize import word_dict_trie from pythainlp.tokenize.tcc_p import tcc_pos @@ -57,7 +59,7 @@ def _bfs_paths_graph( graph: defaultdict, start: int, goal: int -) -> Generator[List[int], None, None]: +) -> Generator[list[int], None, None]: queue = [(start, [start])] while queue: (vertex, path) = queue.pop(0) @@ -140,9 +142,9 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: def segment( text: str, - custom_dict: Optional[Trie] = None, + custom_dict: Trie | None = None, safe_mode: bool = False, -) -> List[str]: +) -> list[str]: """Maximal-matching word segmentation constrained by Thai Character Cluster. A dictionary-based word segmentation using maximal matching algorithm, diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index c29dc0874..310928ee1 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from sys import stderr -from typing import List from nlpo3 import load_dict as nlpo3_load_dict from nlpo3 import segment as nlpo3_segment @@ -45,7 +45,7 @@ def segment( custom_dict: str = _NLPO3_DEFAULT_DICT_NAME, safe_mode: bool = False, parallel_mode: bool = False, -) -> List[str]: +) -> list[str]: """Break text into tokens. Python binding for nlpO3. It is newmm engine in Rust. diff --git a/pythainlp/tokenize/oskut.py b/pythainlp/tokenize/oskut.py index 3c6037a15..58cb96344 100644 --- a/pythainlp/tokenize/oskut.py +++ b/pythainlp/tokenize/oskut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,7 +9,8 @@ :See Also: * `GitHub repository `_ """ -from typing import List + +from __future__ import annotations import oskut @@ -18,7 +18,7 @@ oskut.load_model(engine=DEFAULT_ENGINE) -def segment(text: str, engine: str = "ws") -> List[str]: +def segment(text: str, engine: str = "ws") -> list[str]: global DEFAULT_ENGINE if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py index 5e27116f6..217577a64 100644 --- a/pythainlp/tokenize/pyicu.py +++ b/pythainlp/tokenize/pyicu.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,13 +9,16 @@ :See Also: * `GitHub repository `_ """ + +from __future__ import annotations + import re -from typing import List from icu import BreakIterator, Locale bd = BreakIterator.createWordInstance(Locale("th")) + def _gen_words(text: str) -> str: global bd bd.setText(text) @@ -26,7 +28,7 @@ def _gen_words(text: str) -> str: p = q -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ :param str text: text to be tokenized into words :return: list of words, tokenized from the text @@ -34,6 +36,6 @@ def segment(text: str) -> List[str]: if not text or not isinstance(text, str): return [] - text = re.sub("([^\u0E00-\u0E7F\n ]+)", " \\1 ", text) + text = re.sub("([^\u0e00-\u0e7f\n ]+)", " \\1 ", text) return list(_gen_words(text)) diff --git a/pythainlp/tokenize/sefr_cut.py b/pythainlp/tokenize/sefr_cut.py index e8434ba49..7a123e541 100644 --- a/pythainlp/tokenize/sefr_cut.py +++ b/pythainlp/tokenize/sefr_cut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,7 +8,8 @@ :See Also: * `GitHub repository `_ """ -from typing import List + +from __future__ import annotations import sefr_cut @@ -17,7 +17,7 @@ sefr_cut.load_model(engine=DEFAULT_ENGINE) -def segment(text: str, engine: str = "ws1000") -> List[str]: +def segment(text: str, engine: str = "ws1000") -> list[str]: global DEFAULT_ENGINE if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py index 6ea6daade..e7abeabc3 100644 --- a/pythainlp/tokenize/ssg.py +++ b/pythainlp/tokenize/ssg.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations from ssg import syllable_tokenize -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ Syllable tokenizer using ssg """ diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py index 81a92b30c..786050397 100644 --- a/pythainlp/tokenize/tcc.py +++ b/pythainlp/tokenize/tcc.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,8 +12,10 @@ `_) * Python code: Korakot Chaovavanich """ + +from __future__ import annotations + import re -from typing import List, Set _RE_TCC = ( """\ @@ -48,9 +49,7 @@ ก็ อึ หึ -""".replace( - "k", "(cc?[d|ิ]?[์])?" - ) +""".replace("k", "(cc?[d|ิ]?[์])?") .replace("c", "[ก-ฮ]") .replace("t", "[่-๋]?") .replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel @@ -83,7 +82,7 @@ def tcc(text: str) -> str: p += n -def tcc_pos(text: str) -> Set[int]: +def tcc_pos(text: str) -> set[int]: """ TCC positions @@ -103,7 +102,7 @@ def tcc_pos(text: str) -> Set[int]: return p_set -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ Subword segmentation diff --git a/pythainlp/tokenize/tcc_p.py b/pythainlp/tokenize/tcc_p.py index fe4376bbb..1d7fd7325 100644 --- a/pythainlp/tokenize/tcc_p.py +++ b/pythainlp/tokenize/tcc_p.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -14,8 +13,10 @@ `_) * Python code: Korakot Chaovavanich """ + +from __future__ import annotations + import re -from typing import List, Set _RE_TCC = ( """\ @@ -48,9 +49,7 @@ ก็ อึ หึ -""".replace( - "k", "(cc?[dิ]?[์])?" - ) +""".replace("k", "(cc?[dิ]?[์])?") .replace("c", "[ก-ฮ]") .replace("t", "[่-๋]?") .replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel @@ -83,7 +82,7 @@ def tcc(text: str) -> str: p += n -def tcc_pos(text: str) -> Set[int]: +def tcc_pos(text: str) -> set[int]: """ TCC positions @@ -103,7 +102,7 @@ def tcc_pos(text: str) -> Set[int]: return p_set -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ Subword segmentation diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py index 31556ae2e..e6f1059f3 100644 --- a/pythainlp/tokenize/thai2fit.py +++ b/pythainlp/tokenize/thai2fit.py @@ -1,12 +1,13 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 from functools import lru_cache + from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer + @lru_cache def thai2fit_tokenizer(): """Lazy load Thai2Fit tokenizer with cache""" diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 2c74f8875..15bc321e7 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2020 Nakhun Chumpolsathien # SPDX-License-Identifier: Apache-2.0 @@ -15,21 +14,22 @@ school={Beijing Institute of Technology} """ +from __future__ import annotations + import math import operator import re -from typing import List from pythainlp.tokenize import word_tokenize -def list_to_string(list: List[str]) -> str: +def list_to_string(list: list[str]) -> str: string = "".join(list) string = " ".join(string.split()) return string -def middle_cut(sentences: List[str]) -> List[str]: +def middle_cut(sentences: list[str]) -> list[str]: new_text = "" for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) @@ -86,7 +86,7 @@ def middle_cut(sentences: List[str]) -> List[str]: class ThaiSentenceSegmentor: def split_into_sentences( self, text: str, isMiddleCut: bool = False - ) -> List[str]: + ) -> list[str]: # Declare Variables th_alphabets = "([ก-๙])" th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)" diff --git a/pythainlp/tokenize/tltk.py b/pythainlp/tokenize/tltk.py index fb044caf0..90b1ab4b2 100644 --- a/pythainlp/tokenize/tltk.py +++ b/pythainlp/tokenize/tltk.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations try: from tltk.nlp import syl_segment from tltk.nlp import word_segment as tltk_segment except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: if not text or not isinstance(text, str): return [] text = text.replace(" ", "") @@ -22,7 +23,7 @@ def segment(text: str) -> List[str]: return _temp -def syllable_tokenize(text: str) -> List[str]: +def syllable_tokenize(text: str) -> list[str]: if not text or not isinstance(text, str): return [] _temp = syl_segment(text) @@ -32,7 +33,7 @@ def syllable_tokenize(text: str) -> List[str]: return _temp -def sent_tokenize(text: str) -> List[str]: +def sent_tokenize(text: str) -> list[str]: text = text.replace(" ", "") _temp = tltk_segment(text).replace("", " ").replace("|", "") _temp = _temp.split("") diff --git a/pythainlp/tokenize/wtsplit.py b/pythainlp/tokenize/wtsplit.py index 952f605a8..80ff22800 100644 --- a/pythainlp/tokenize/wtsplit.py +++ b/pythainlp/tokenize/wtsplit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,7 +6,8 @@ GitHub: https://github.com/bminixhofer/wtpsplit """ -from typing import List + +from __future__ import annotations from wtpsplit import WtP @@ -22,7 +22,7 @@ def _tokenize( tokenize: str = "sentence", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[str]: +) -> list[str]: global _MODEL_NAME, _MODEL if _MODEL_NAME != model: @@ -60,7 +60,7 @@ def tokenize( tokenize: str = "sentence", paragraph_threshold: float = 0.5, style: str = "newline", -) -> List[str]: +) -> list[str]: _model_load = "" if size == "tiny": _model_load = "wtp-bert-tiny" diff --git a/pythainlp/tools/__init__.py b/pythainlp/tools/__init__.py index 1d36048d1..3d6743c66 100644 --- a/pythainlp/tools/__init__.py +++ b/pythainlp/tools/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/tools/core.py b/pythainlp/tools/core.py index f7c190745..62997e743 100644 --- a/pythainlp/tools/core.py +++ b/pythainlp/tools/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Generic support functions for PyThaiNLP. """ +from __future__ import annotations + import sys import warnings @@ -33,6 +34,7 @@ def warn_deprecation( message += f" Please use '{replacing_func}' instead." warnings.warn(message, DeprecationWarning, stacklevel=2) + def safe_print(text: str): """Print text to console, handling UnicodeEncodeError. diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index a5fa84208..92aebb7a1 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations import math import random @@ -52,7 +51,7 @@ def search_location_of_character(char: str): def find_neighbour_locations( loc: tuple, char: str, - kernel: List = [(-1, -1), (-1, 0), (1, 1), (0, 1), (0, -1), (1, 0)], + kernel: list = [(-1, -1), (-1, 0), (1, 1), (0, 1), (0, -1), (1, 0)], ): language_ix, is_shift, row, pos = loc diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index ddf81a6e7..49deb97af 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,6 +6,9 @@ For text processing and text conversion, see pythainlp.util """ + +from __future__ import annotations + import os from pythainlp import __file__ as pythainlp_file @@ -27,7 +29,7 @@ def get_full_data_path(path: str) -> str: from pythainlp.tools import get_full_data_path - get_full_data_path('ttc_freq.txt') + get_full_data_path("ttc_freq.txt") # output: '/root/pythainlp-data/ttc_freq.txt' """ return os.path.join(get_pythainlp_data_path(), path) diff --git a/pythainlp/translate/__init__.py b/pythainlp/translate/__init__.py index ce83658cc..410d0d40b 100644 --- a/pythainlp/translate/__init__.py +++ b/pythainlp/translate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index f7387271f..493fffdbd 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations class Translate: @@ -99,11 +98,8 @@ def translate(self, text: str) -> str: def word_translate( - word: str, - src: str, - target: str, - engine: str="word2word" - ) -> Union[List[str], None]: + word: str, src: str, target: str, engine: str = "word2word" +) -> list[str] | None: """ Translate word from source language to target language. @@ -119,18 +115,21 @@ def word_translate( Translate word from Thai to English:: from pythainlp.translate import word_translate - print(word_translate("แมว","th","en")) + + print(word_translate("แมว", "th", "en")) # output: ['cat', 'cats', 'kitty', 'kitten', 'Cat'] Translate word from English to Thai:: from pythainlp.translate import word_translate - print(word_translate("cat","en","th")) + + print(word_translate("cat", "en", "th")) # output: ['แมว', 'แมวป่า', 'ข่วน', 'เลี้ยง', 'อาหาร'] """ - if engine=="word2word": + if engine == "word2word": from .word2word_translate import translate + return translate(word=word, src=src, target=target) else: raise NotImplementedError( diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index f23065daf..e13e9264a 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,17 +8,24 @@ Website: https://airesearch.in.th/releases/machine-translation-models/ """ + +from __future__ import annotations + import os try: from fairseq.models.transformer import TransformerModel except ImportError: - raise ImportError("Not found fairseq! Please install fairseq by pip install fairseq") + raise ImportError( + "Not found fairseq! Please install fairseq by pip install fairseq" + ) try: from sacremoses import MosesTokenizer except ImportError: - raise ImportError("Not found sacremoses! Please install sacremoses by pip install sacremoses") + raise ImportError( + "Not found sacremoses! Please install sacremoses by pip install sacremoses" + ) from pythainlp.corpus import download, get_corpus_path diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index 2699a8dd4..da00ef6ba 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from transformers import M2M100ForConditionalGeneration from .tokenization_small100 import SMALL100Tokenizer @@ -22,12 +23,14 @@ def __init__( pretrained: str = "alirezamsh/small100", ) -> None: self.pretrained = pretrained - self.model = M2M100ForConditionalGeneration.from_pretrained(self.pretrained) + self.model = M2M100ForConditionalGeneration.from_pretrained( + self.pretrained + ) self.tgt_lang = None if use_gpu: self.model = self.model.cuda() - def translate(self, text: str, tgt_lang: str="en") -> str: + def translate(self, text: str, tgt_lang: str = "en") -> str: """ Translate text from X to X @@ -57,10 +60,14 @@ def translate(self, text: str, tgt_lang: str="en") -> str: # output: 'Test du système' """ - if tgt_lang!=self.tgt_lang: - self.tokenizer = SMALL100Tokenizer.from_pretrained(self.pretrained, tgt_lang=tgt_lang) + if tgt_lang != self.tgt_lang: + self.tokenizer = SMALL100Tokenizer.from_pretrained( + self.pretrained, tgt_lang=tgt_lang + ) self.tgt_lang = tgt_lang self.translated = self.model.generate( **self.tokenizer(text, return_tensors="pt") ) - return self.tokenizer.batch_decode(self.translated, skip_special_tokens=True)[0] + return self.tokenizer.batch_decode( + self.translated, skip_special_tokens=True + )[0] diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 4857143f7..44db1faa0 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -14,6 +13,8 @@ - Huggingface https://huggingface.co/Helsinki-NLP/opus-mt-th-fr """ +from __future__ import annotations + class ThFrTranslator: """ @@ -36,6 +37,7 @@ def __init__( pretrained: str = "Helsinki-NLP/opus-mt-th-fr", ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + self.tokenizer_thzh = AutoTokenizer.from_pretrained(pretrained) self.model_thzh = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 9a11dc2da..e7d32e6c0 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -21,11 +21,13 @@ # limitations under the License. """Tokenization classes for SMALL100.""" +from __future__ import annotations + import json import os from pathlib import Path from shutil import copyfile -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any import sentencepiece from transformers.tokenization_utils import BatchEncoding, PreTrainedTokenizer @@ -114,8 +116,8 @@ class SMALL100Tokenizer(PreTrainedTokenizer): pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP model_input_names = ["input_ids", "attention_mask"] - prefix_tokens: List[int] = [] - suffix_tokens: List[int] = [] + prefix_tokens: list[int] = [] + suffix_tokens: list[int] = [] def __init__( self, @@ -128,21 +130,29 @@ def __init__( pad_token="", unk_token="", language_codes="m2m100", - sp_model_kwargs: Optional[Dict[str, Any]] = None, + sp_model_kwargs: dict[str, Any] | None = None, num_madeup_words=8, **kwargs, ) -> None: - self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.sp_model_kwargs = ( + {} if sp_model_kwargs is None else sp_model_kwargs + ) self.language_codes = language_codes fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes] - self.lang_code_to_token = {lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code} + self.lang_code_to_token = { + lang_code: f"__{lang_code}__" + for lang_code in fairseq_language_code + } - kwargs["additional_special_tokens"] = kwargs.get("additional_special_tokens", []) + kwargs["additional_special_tokens"] = kwargs.get( + "additional_special_tokens", [] + ) kwargs["additional_special_tokens"] += [ self.get_lang_token(lang_code) for lang_code in fairseq_language_code - if self.get_lang_token(lang_code) not in kwargs["additional_special_tokens"] + if self.get_lang_token(lang_code) + not in kwargs["additional_special_tokens"] ] super().__init__( @@ -167,10 +177,16 @@ def __init__( self.encoder_size = len(self.encoder) self.lang_token_to_id = { - self.get_lang_token(lang_code): self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code) + self.get_lang_token(lang_code): self.encoder_size + i + for i, lang_code in enumerate(fairseq_language_code) + } + self.lang_code_to_id = { + lang_code: self.encoder_size + i + for i, lang_code in enumerate(fairseq_language_code) + } + self.id_to_lang_token = { + v: k for k, v in self.lang_token_to_id.items() } - self.lang_code_to_id = {lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)} - self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()} self._tgt_lang = tgt_lang if tgt_lang is not None else "en" self.cur_lang_id = self.get_lang_id(self._tgt_lang) @@ -180,7 +196,11 @@ def __init__( @property def vocab_size(self) -> int: - return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words + return ( + len(self.encoder) + + len(self.lang_token_to_id) + + self.num_madeup_words + ) @property def tgt_lang(self) -> str: @@ -191,7 +211,7 @@ def tgt_lang(self, new_tgt_lang: str) -> None: self._tgt_lang = new_tgt_lang self.set_lang_special_tokens(self._tgt_lang) - def _tokenize(self, text: str) -> List[str]: + def _tokenize(self, text: str) -> list[str]: return self.sp_model.encode(text, out_type=str) def _convert_token_to_id(self, token): @@ -205,13 +225,16 @@ def _convert_id_to_token(self, index: int) -> str: return self.id_to_lang_token[index] return self.decoder.get(index, self.unk_token) - def convert_tokens_to_string(self, tokens: List[str]) -> str: + def convert_tokens_to_string(self, tokens: list[str]) -> str: """Converts a sequence of tokens (strings for sub-words) in a single string.""" return self.sp_model.decode(tokens) def get_special_tokens_mask( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False - ) -> List[int]: + self, + token_ids_0: list[int], + token_ids_1: list[int] | None = None, + already_has_special_tokens: bool = False, + ) -> list[int]: """ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. @@ -228,18 +251,25 @@ def get_special_tokens_mask( if already_has_special_tokens: return super().get_special_tokens_mask( - token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, ) prefix_ones = [1] * len(self.prefix_tokens) suffix_ones = [1] * len(self.suffix_tokens) if token_ids_1 is None: return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones - return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones + return ( + prefix_ones + + ([0] * len(token_ids_0)) + + ([0] * len(token_ids_1)) + + suffix_ones + ) def build_inputs_with_special_tokens( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: + self, token_ids_0: list[int], token_ids_1: list[int] | None = None + ) -> list[int]: """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. An MBART sequence has the following format, where `X` represents the sequence: @@ -264,19 +294,26 @@ def build_inputs_with_special_tokens( if self.prefix_tokens is None: return token_ids_0 + token_ids_1 + self.suffix_tokens else: - return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens + return ( + self.prefix_tokens + + token_ids_0 + + token_ids_1 + + self.suffix_tokens + ) - def get_vocab(self) -> Dict: - vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + def get_vocab(self) -> dict: + vocab = { + self.convert_ids_to_tokens(i): i for i in range(self.vocab_size) + } vocab.update(self.added_tokens_encoder) return vocab - def __getstate__(self) -> Dict: + def __getstate__(self) -> dict: state = self.__dict__.copy() state["sp_model"] = None return state - def __setstate__(self, d: Dict) -> None: + def __setstate__(self, d: dict) -> None: self.__dict__ = d # for backward compatibility @@ -285,20 +322,26 @@ def __setstate__(self, d: Dict) -> None: self.sp_model = load_spm(self.spm_file, self.sp_model_kwargs) - def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + def save_vocabulary( + self, save_directory: str, filename_prefix: str | None = None + ) -> tuple[str]: save_dir = Path(save_directory) if not save_dir.is_dir(): raise OSError(f"{save_directory} should be a directory") vocab_save_path = save_dir / ( - (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"] + (filename_prefix + "-" if filename_prefix else "") + + self.vocab_files_names["vocab_file"] ) spm_save_path = save_dir / ( - (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"] + (filename_prefix + "-" if filename_prefix else "") + + self.vocab_files_names["spm_file"] ) save_json(self.encoder, vocab_save_path) - if os.path.abspath(self.spm_file) != os.path.abspath(spm_save_path) and os.path.isfile(self.spm_file): + if os.path.abspath(self.spm_file) != os.path.abspath( + spm_save_path + ) and os.path.isfile(self.spm_file): copyfile(self.spm_file, spm_save_path) elif not os.path.isfile(self.spm_file): with open(spm_save_path, "wb") as fi: @@ -309,8 +352,8 @@ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = def prepare_seq2seq_batch( self, - src_texts: List[str], - tgt_texts: Optional[List[str]] = None, + src_texts: list[str], + tgt_texts: list[str] | None = None, tgt_lang: str = "ro", **kwargs, ) -> BatchEncoding: @@ -318,10 +361,14 @@ def prepare_seq2seq_batch( self.set_lang_special_tokens(self.tgt_lang) return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs) - def _build_translation_inputs(self, raw_inputs, tgt_lang: Optional[str], **extra_kwargs): + def _build_translation_inputs( + self, raw_inputs, tgt_lang: str | None, **extra_kwargs + ): """Used by translation pipeline, to prepare inputs for the generate function""" if tgt_lang is None: - raise ValueError("Translation requires a `tgt_lang` for this model") + raise ValueError( + "Translation requires a `tgt_lang` for this model" + ) self.tgt_lang = tgt_lang inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs) return inputs @@ -348,14 +395,16 @@ def get_lang_id(self, lang: str) -> int: return self.lang_token_to_id[lang_token] -def load_spm(path: str, sp_model_kwargs: Dict[str, Any]) -> sentencepiece.SentencePieceProcessor: +def load_spm( + path: str, sp_model_kwargs: dict[str, Any] +) -> sentencepiece.SentencePieceProcessor: spm = sentencepiece.SentencePieceProcessor(**sp_model_kwargs) spm.Load(str(path)) return spm -def load_json(path: str) -> Union[Dict, List]: - with open(path, "r") as f: +def load_json(path: str) -> dict | list: + with open(path) as f: return json.load(f) diff --git a/pythainlp/translate/word2word_translate.py b/pythainlp/translate/word2word_translate.py index fe027ff33..1c603dc71 100644 --- a/pythainlp/translate/word2word_translate.py +++ b/pythainlp/translate/word2word_translate.py @@ -1,73 +1,77 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Union +from __future__ import annotations + from word2word import Word2word -support_list = set(['zh_tw', - 'el', - 'te', - 'hu', - 'eu', - 'ko', - 'ru', - 'lv', - 'bg', - 'sk', - 'vi', - 'gl', - 'et', - 'ta', - 'fa', - 'it', - 'ms', - 'id', - 'pt', - 'fr', - 'sr', - 'mk', - 'sv', - 'si', - 'en', - 'ka', - 'uk', - 'sl', - 'hi', - 'ca', - 'lt', - 'es', - 'no', - 'de', - 'he', - 'cs', - 'ze_zh', - 'fi', - 'pl', - 'tl', - 'is', - 'ze_en', - 'kk', - 'bn', - 'tr', - 'ur', - 'pt_br', - 'ar', - 'ro', - 'bs', - 'ml', - 'zh_cn', - 'da', - 'hr', - 'sq', - 'af', - 'eo', - 'nl', - 'ja', - 'th']) +support_list = set( + [ + "zh_tw", + "el", + "te", + "hu", + "eu", + "ko", + "ru", + "lv", + "bg", + "sk", + "vi", + "gl", + "et", + "ta", + "fa", + "it", + "ms", + "id", + "pt", + "fr", + "sr", + "mk", + "sv", + "si", + "en", + "ka", + "uk", + "sl", + "hi", + "ca", + "lt", + "es", + "no", + "de", + "he", + "cs", + "ze_zh", + "fi", + "pl", + "tl", + "is", + "ze_en", + "kk", + "bn", + "tr", + "ur", + "pt_br", + "ar", + "ro", + "bs", + "ml", + "zh_cn", + "da", + "hr", + "sq", + "af", + "eo", + "nl", + "ja", + "th", + ] +) -def translate(word: str, src: str, target: str) -> Union[List[str], None]: +def translate(word: str, src: str, target: str) -> list[str] | None: """ Word translate @@ -78,10 +82,8 @@ def translate(word: str, src: str, target: str) -> Union[List[str], None]: :rtype: Union[List[str], None] """ if src not in support_list or target not in support_list: - raise NotImplementedError( - f"word2word doesn't support {src}-{target}." - ) - elif src==target: + raise NotImplementedError(f"word2word doesn't support {src}-{target}.") + elif src == target: return [word] _engine = Word2word(src, target) - return _engine(word) \ No newline at end of file + return _engine(word) diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 1ac82eeb5..2cf5c056b 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,6 +10,8 @@ - Facebook post https://web.facebook.com/aibuildersx/posts/166736255494822 """ +from __future__ import annotations + class ThZhTranslator: """ @@ -30,6 +31,7 @@ def __init__( pretrained: str = "Lalita/marianmt-th-zh_cn", ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + self.tokenizer_thzh = AutoTokenizer.from_pretrained(pretrained) self.model_thzh = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 4dd1ab9b9..466dd4d3d 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/transliterate/core.py b/pythainlp/transliterate/core.py index 6881b071d..df96a5253 100644 --- a/pythainlp/transliterate/core.py +++ b/pythainlp/transliterate/core.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations DEFAULT_ROMANIZE_ENGINE = "royin" DEFAULT_TRANSLITERATE_ENGINE = "thaig2p" @@ -92,9 +92,9 @@ def select_romanize_engine(engine: str): else: rom_engine = select_romanize_engine(engine) trans_word = [] - for subword in text.split(' '): + for subword in text.split(" "): trans_word.append(rom_engine(subword)) - new_word = ' '.join(trans_word) + new_word = " ".join(trans_word) return new_word diff --git a/pythainlp/transliterate/ipa.py b/pythainlp/transliterate/ipa.py index 0193be693..b1c6b6fa7 100644 --- a/pythainlp/transliterate/ipa.py +++ b/pythainlp/transliterate/ipa.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,7 +9,8 @@ * `GitHub \ `_ """ -from typing import List + +from __future__ import annotations import epitran @@ -21,9 +21,9 @@ def transliterate(text: str) -> str: return _EPI_THA.transliterate(text) -def trans_list(text: str) -> List[str]: +def trans_list(text: str) -> list[str]: return _EPI_THA.trans_list(text) -def xsampa_list(text: str) -> List[str]: +def xsampa_list(text: str) -> list[str]: return _EPI_THA.xsampa_list(text) diff --git a/pythainlp/transliterate/iso_11940.py b/pythainlp/transliterate/iso_11940.py index c4b784ca1..326827f32 100644 --- a/pythainlp/transliterate/iso_11940.py +++ b/pythainlp/transliterate/iso_11940.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,6 +8,9 @@ * `Wikipedia \ `_ """ + +from __future__ import annotations + _consonants = { "ก": "k", "ข": "k̄h", diff --git a/pythainlp/transliterate/lookup.py b/pythainlp/transliterate/lookup.py index 02b7dd49c..5b89228bb 100644 --- a/pythainlp/transliterate/lookup.py +++ b/pythainlp/transliterate/lookup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,7 +9,9 @@ Zenodo. https://doi.org/10.5281/zenodo.6716672 """ -from typing import Callable, Optional +from __future__ import annotations + +from collections.abc import Callable from pythainlp.corpus.th_en_translit import ( TRANSLITERATE_DICT, @@ -21,7 +22,7 @@ _TRANSLITERATE_IDX = 0 -def follow_rtgs(text: str) -> Optional[bool]: +def follow_rtgs(text: str) -> bool | None: """ Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). :param str text: Text to look up. Must be a self-contained word. diff --git a/pythainlp/transliterate/pyicu.py b/pythainlp/transliterate/pyicu.py index 9a99066eb..c0f330116 100644 --- a/pythainlp/transliterate/pyicu.py +++ b/pythainlp/transliterate/pyicu.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,6 +9,9 @@ * `GitHub \ `_ """ + +from __future__ import annotations + from icu import Transliterator _ICU_THAI_TO_LATIN = Transliterator.createInstance("Thai-Latin") diff --git a/pythainlp/transliterate/royin.py b/pythainlp/transliterate/royin.py index 5a3ffc3e7..74e592ca0 100644 --- a/pythainlp/transliterate/royin.py +++ b/pythainlp/transliterate/royin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,6 +9,9 @@ :See Also: * `Wikipedia `_ """ + +from __future__ import annotations + import re from pythainlp import thai_consonants, word_tokenize @@ -157,7 +159,7 @@ def _replace_consonants(word: str, consonants: str) -> str: _LO_LING = "\u0e25" # ล _WO_WAEN = "\u0e27" # ว _DOUBLE_RO_RUA = _RO_RUA + _RO_RUA - + # Consonants that can be second in a cluster _CLUSTER_SECOND = {_RO_RUA, _LO_LING, _WO_WAEN} @@ -168,7 +170,7 @@ def _replace_consonants(word: str, consonants: str) -> str: mod_chars = [] j = 0 # j is the index of consonants vowel_seen = False # Track if we've seen a vowel (non-consonant character) - + for i in range(len(word)): if skip: skip = False @@ -194,24 +196,32 @@ def _replace_consonants(word: str, consonants: str) -> str: elif not vowel_seen: # Building initial consonant cluster # Check if we've added any actual initial consonants (non-empty romanized characters) # We check for non-vowel characters since mod_chars contains romanized output - has_initial = any(c and c not in _ROMANIZED_VOWELS for c in mod_chars) - + has_initial = any( + c and c not in _ROMANIZED_VOWELS for c in mod_chars + ) + if not has_initial: # First consonant in the cluster initial = _CONSONANTS[consonants[j]][0] - if initial: # Only append if not empty (e.g., อ has empty initial) + if ( + initial + ): # Only append if not empty (e.g., อ has empty initial) mod_chars.append(initial) j += 1 else: # Check if this consonant can be part of a cluster is_cluster_consonant = word[i] in _CLUSTER_SECOND - is_last_char = (i + 1 >= len(word)) - has_vowel_next = not is_last_char and word[i+1] not in _CONSONANTS - + is_last_char = i + 1 >= len(word) + has_vowel_next = ( + not is_last_char and word[i + 1] not in _CONSONANTS + ) + # Cluster consonants (ร/r, ล/l, ว/w) are part of initial cluster if: # - followed by a vowel, OR # - not the last character (e.g., กรม/krom: ก/k+ร/r are cluster, ม/m is final) - if is_cluster_consonant and (has_vowel_next or not is_last_char): + if is_cluster_consonant and ( + has_vowel_next or not is_last_char + ): # This is part of initial cluster (ร/r, ล/l, or ว/w after first consonant) mod_chars.append(_CONSONANTS[consonants[j]][0]) j += 1 @@ -244,7 +254,9 @@ def _replace_consonants(word: str, consonants: str) -> str: vowel_seen = True j += 1 else: # After vowel - could be final consonant or start of new syllable - has_vowel_next = (i + 1 < len(word) and word[i+1] not in _CONSONANTS) + has_vowel_next = ( + i + 1 < len(word) and word[i + 1] not in _CONSONANTS + ) if has_vowel_next: # Consonant followed by vowel - start of new syllable mod_chars.append(_CONSONANTS[consonants[j]][0]) @@ -260,9 +272,9 @@ def _replace_consonants(word: str, consonants: str) -> str: # support function for romanize() def _romanize(word: str) -> str: # Special case: single ห character should be empty (silent) - if word == 'ห': - return '' - + if word == "ห": + return "" + word = _replace_vowels(_normalize(word)) consonants = _RE_CONSONANT.findall(word) @@ -276,14 +288,16 @@ def _romanize(word: str) -> str: return word -def _should_add_syllable_separator(prev_word: str, curr_word: str, prev_romanized: str) -> bool: +def _should_add_syllable_separator( + prev_word: str, curr_word: str, prev_romanized: str +) -> bool: """ Determine if 'a' should be added between two romanized syllables. - + This applies when: - Previous word has explicit vowel and ends with consonant - Current word is a 2-consonant cluster with no vowels (e.g., 'กร') - + :param prev_word: The previous Thai word/token :param curr_word: The current Thai word/token :param prev_romanized: The romanized form of the previous word @@ -291,22 +305,24 @@ def _should_add_syllable_separator(prev_word: str, curr_word: str, prev_romanize """ if not prev_romanized or len(curr_word) < 2: return False - + # Check if previous word has explicit vowel prev_normalized = _normalize(prev_word) prev_after_vowels = _replace_vowels(prev_normalized) prev_consonants = _RE_CONSONANT.findall(prev_word) has_explicit_vowel_prev = len(prev_after_vowels) > len(prev_consonants) - + # Check if current word is 2 Thai consonants with no vowel consonants_in_word = _RE_CONSONANT.findall(curr_word) vowels_in_word = len(curr_word) - len(consonants_in_word) - + # Add 'a' if conditions are met - return (has_explicit_vowel_prev and - len(consonants_in_word) == 2 and - vowels_in_word == 0 and - prev_romanized[-1] not in _ROMANIZED_VOWELS) + return ( + has_explicit_vowel_prev + and len(consonants_in_word) == 2 + and vowels_in_word == 0 + and prev_romanized[-1] not in _ROMANIZED_VOWELS + ) def romanize(text: str) -> str: @@ -322,17 +338,17 @@ def romanize(text: str) -> str: """ words = word_tokenize(text) romanized_words = [] - + for i, word in enumerate(words): romanized = _romanize(word) - + # Check if we need to add syllable separator 'a' if i > 0 and romanized: - prev_word = words[i-1] - prev_romanized = romanized_words[-1] if romanized_words else '' + prev_word = words[i - 1] + prev_romanized = romanized_words[-1] if romanized_words else "" if _should_add_syllable_separator(prev_word, word, prev_romanized): - romanized = 'a' + romanized - + romanized = "a" + romanized + romanized_words.append(romanized) - + return "".join(romanized_words) diff --git a/pythainlp/transliterate/spoonerism.py b/pythainlp/transliterate/spoonerism.py index 1c91b3f23..cc9a55d30 100644 --- a/pythainlp/transliterate/spoonerism.py +++ b/pythainlp/transliterate/spoonerism.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp import thai_consonants from pythainlp.transliterate import pronunciate diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 334668b8b..f4d55f48a 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Romanization of Thai words based on machine-learnt engine ("thai2rom") """ +from __future__ import annotations + import random import torch @@ -139,7 +140,9 @@ def forward(self, sequences, sequences_lengths): sequences = self.dropout(sequences) sequences_packed = nn.utils.rnn.pack_padded_sequence( - sequences, sequences_lengths.clone().to("cpu", torch.int64), batch_first=True + sequences, + sequences_lengths.clone().to("cpu", torch.int64), + batch_first=True, ) sequences_output, hidden = self.rnn(sequences_packed, hidden) diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index efdd6a4ae..822129c6f 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -1,10 +1,12 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") """ + +from __future__ import annotations + import json import numpy as np diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index 3448b78cb..215b33328 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,6 +6,8 @@ GitHub : https://github.com/wannaphong/thai-g2p """ +from __future__ import annotations + import random import numpy as np @@ -124,7 +125,6 @@ def __init__( self.dropout = nn.Dropout(dropout) def forward(self, sequences, sequences_lengths): - # sequences: (batch_size, sequence_length=MAX_LENGTH) # sequences_lengths: (batch_size) @@ -198,9 +198,7 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze( - 2 - ) # (batch_size, sequence_len) + ).squeeze(2) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -297,7 +295,6 @@ def create_mask(self, source_seq): def forward( self, source_seq, source_seq_len, target_seq, teacher_forcing_ratio=0.5 ): - # source_seq: (batch_size, MAX_LENGTH) # source_seq_len: (batch_size, 1) # target_seq: (batch_size, MAX_LENGTH) diff --git a/pythainlp/transliterate/thaig2p_v2.py b/pythainlp/transliterate/thaig2p_v2.py index 4b0643cea..b9d4ba617 100644 --- a/pythainlp/transliterate/thaig2p_v2.py +++ b/pythainlp/transliterate/thaig2p_v2.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -9,6 +8,8 @@ """ # Use a pipeline as a high-level helper +from __future__ import annotations + from transformers import pipeline @@ -18,7 +19,11 @@ class ThaiG2P: """ def __init__(self, device: str = "cpu"): - self.pipe = pipeline("text2text-generation", model="pythainlp/thaig2p-v2.0", device=device) + self.pipe = pipeline( + "text2text-generation", + model="pythainlp/thaig2p-v2.0", + device=device, + ) def g2p(self, text: str) -> str: return self.pipe(text)[0]["generated_text"] diff --git a/pythainlp/transliterate/tltk.py b/pythainlp/transliterate/tltk.py index 12da6c25c..bbeb7ba9c 100644 --- a/pythainlp/transliterate/tltk.py +++ b/pythainlp/transliterate/tltk.py @@ -1,11 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + try: from tltk.nlp import g2p, th2ipa, th2roman except ImportError: - raise ImportError("Not found tltk! Please install tltk by pip install tltk") + raise ImportError( + "Not found tltk! Please install tltk by pip install tltk" + ) def romanize(text: str) -> str: diff --git a/pythainlp/transliterate/umt5_thaig2p.py b/pythainlp/transliterate/umt5_thaig2p.py index 2b30d3b39..868d4a8f4 100644 --- a/pythainlp/transliterate/umt5_thaig2p.py +++ b/pythainlp/transliterate/umt5_thaig2p.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ -umt5-thai-g2p-v2-0.5k +umt5-thai-g2p-v2-0.5k huggingface: https://huggingface.co/B-K/umt5-thai-g2p-v2-0.5k """ # Use a pipeline as a high-level helper +from __future__ import annotations + from transformers import pipeline @@ -18,7 +19,11 @@ class Umt5ThaiG2P: """ def __init__(self, device: str = "cpu"): - self.pipe = pipeline("text2text-generation", model="B-K/umt5-thai-g2p-v2-0.5k", device=device) + self.pipe = pipeline( + "text2text-generation", + model="B-K/umt5-thai-g2p-v2-0.5k", + device=device, + ) def g2p(self, text: str) -> str: return self.pipe(text)[0]["generated_text"] diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 2c206bbe7..3880e76e8 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,19 +6,17 @@ GitHub : https://github.com/wannaphong/Thai_W2P """ -from typing import Union +from __future__ import annotations import numpy as np from pythainlp.corpus import download, get_corpus_path _GRAPHEMES = list( - "พจใงต้ืฮแาฐฒฤๅูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" - + " ณิฑชฉซทรฏฬํัฃวก่ป์ผฆบี๊ธญฌษะไ๋นโภ?" + "พจใงต้ืฮแาฐฒฤๅูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" + " ณิฑชฉซทรฏฬํัฃวก่ป์ผฆบี๊ธญฌษะไ๋นโภ?" ) _PHONEMES = list( - "-พจใงต้ืฮแาฐฒฤูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" - + " ณิฑชฉซทรํฬฏ–ัฃวก่ปผ์ฆบี๊ธฌญะไษ๋นโภ?" + "-พจใงต้ืฮแาฐฒฤูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" + " ณิฑชฉซทรํฬฏ–ัฃวก่ปผ์ฆบี๊ธฌญะไษ๋นโภ?" ) _MODEL_NAME = "thai_w2p" @@ -50,7 +47,7 @@ def _load_vocab(): return g2idx, idx2g, p2idx, idx2p -class Thai_W2P(): +class Thai_W2P: def __init__(self): super().__init__() self.graphemes = hp.graphemes @@ -133,7 +130,7 @@ def _encode(self, word: str) -> np.ndarray: return x - def _short_word(self, word: str) -> Union[str, None]: + def _short_word(self, word: str) -> str | None: self.word = word if self.word.endswith("."): self.word = self.word.replace(".", "") diff --git a/pythainlp/transliterate/wunsen.py b/pythainlp/transliterate/wunsen.py index 31d8b9eb8..92ed17faf 100644 --- a/pythainlp/transliterate/wunsen.py +++ b/pythainlp/transliterate/wunsen.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -11,6 +10,9 @@ * `GitHub \ `_ """ + +from __future__ import annotations + from wunsen import ThapSap @@ -86,9 +88,7 @@ def transliterate( # output: 'โอฮาโย' wt.transliterate( - "ohayou", - lang="jp", - jp_input="Hepburn-no diacritic" + "ohayou", lang="jp", jp_input="Hepburn-no diacritic" ) # output: 'โอฮาโย' diff --git a/pythainlp/ulmfit/__init__.py b/pythainlp/ulmfit/__init__.py index a7128781b..307843ce5 100644 --- a/pythainlp/ulmfit/__init__.py +++ b/pythainlp/ulmfit/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 760523802..d30f34cd9 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -1,12 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Universal Language Model Fine-tuning for Text Classification (ULMFiT). """ + +from __future__ import annotations + import collections -from typing import Callable, Collection, Optional +from collections.abc import Callable, Collection import numpy as np import torch @@ -67,7 +69,7 @@ def process_thai( text: str, pre_rules: Collection = pre_rules_th_sparse, - tok_func: Optional[Callable] = None, + tok_func: Callable | None = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: """ @@ -222,7 +224,7 @@ def merge_wgts(em_sz, wgts, itos_pre, itos_new): from pythainlp.ulmfit import merge_wgts import torch - wgts = {'0.encoder.weight': torch.randn(5,3)} + wgts = {"0.encoder.weight": torch.randn(5, 3)} itos_pre = ["แมว", "คน", "หนู"] itos_new = ["ปลา", "เต่า", "นก"] em_sz = 3 diff --git a/pythainlp/ulmfit/preprocess.py b/pythainlp/ulmfit/preprocess.py index f9b89ef38..82b94a6c1 100644 --- a/pythainlp/ulmfit/preprocess.py +++ b/pythainlp/ulmfit/preprocess.py @@ -1,13 +1,15 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Preprocessing for ULMFiT """ + +from __future__ import annotations + import html import re -from typing import Collection, List +from collections.abc import Collection import emoji @@ -105,14 +107,14 @@ def replace_rep_after(text: str) -> str: def _replace_rep(m): c, cc = m.groups() - return f"{c}{_TK_REP}{len(cc)+1} " + return f"{c}{_TK_REP}{len(cc) + 1} " re_rep = re.compile(r"(\S)(\1{3,})") return re_rep.sub(_replace_rep, text) -def replace_wrep_post(toks: Collection[str]) -> List[str]: +def replace_wrep_post(toks: Collection[str]) -> list[str]: """ Replace repetitive words after tokenization; fastai `replace_wrep` does not work well with Thai. @@ -186,7 +188,7 @@ def rm_brackets(text: str) -> str: return new_line -def ungroup_emoji(toks: Collection[str]) -> List[str]: +def ungroup_emoji(toks: Collection[str]) -> list[str]: """ Ungroup Zero Width Joiner (ZVJ) Emojis @@ -201,7 +203,7 @@ def ungroup_emoji(toks: Collection[str]) -> List[str]: return res -def lowercase_all(toks: Collection[str]) -> List[str]: +def lowercase_all(toks: Collection[str]) -> list[str]: """ Lowercase all English words; English words in Thai texts don't usually have nuances of capitalization. @@ -239,7 +241,7 @@ def _replace_rep(m): return re_rep.sub(_replace_rep, text) -def replace_wrep_post_nonum(toks: Collection[str]) -> List[str]: +def replace_wrep_post_nonum(toks: Collection[str]) -> list[str]: """ Replace reptitive words post tokenization; fastai `replace_wrep` does not work well with Thai. @@ -274,7 +276,7 @@ def replace_wrep_post_nonum(toks: Collection[str]) -> List[str]: return res[1:] -def remove_space(toks: Collection[str]) -> List[str]: +def remove_space(toks: Collection[str]) -> list[str]: """ Do not include space for bag-of-word models. diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index 0156a4ff8..2b73dbf60 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,7 +5,9 @@ Tokenzier classes for ULMFiT """ -from typing import Collection, List +from __future__ import annotations + +from collections.abc import Collection from pythainlp.tokenize import thai2fit_tokenizer @@ -17,7 +18,7 @@ class BaseTokenizer: def __init__(self, lang: str): self.lang = lang - def tokenizer(self, t: str) -> List[str]: + def tokenizer(self, t: str) -> list[str]: return t.split(" ") def add_special_cases(self, toks: Collection[str]): @@ -35,7 +36,7 @@ def __init__(self, lang: str = "th"): self.lang = lang @staticmethod - def tokenizer(text: str) -> List[str]: + def tokenizer(text: str) -> list[str]: """ This function tokenizes text using *newmm* engine and the dictionary specifically for `ulmfit` related functions diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index a36aa5cdd..a1c28bbe7 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -99,6 +98,7 @@ from pythainlp.util.keywords import find_keyword, rank from pythainlp.util.lcs import longest_common_subsequence from pythainlp.util.normalize import ( + expand_maiyamok, maiyamok, normalize, remove_dangling, @@ -107,41 +107,40 @@ remove_tonemark, remove_zw, reorder_vowels, - expand_maiyamok, ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa +from pythainlp.util.pronounce import ( + rhyme, + spelling, + thai_consonant_to_spelling, + tone_to_spelling, +) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) from pythainlp.util.strftime import thai_strftime + +# sound_syllable and pronounce have to be imported last, +# to prevent circular import issues. +# Other imports should be above this line, sorted. +from pythainlp.util.syllable import ( + sound_syllable, + syllable_length, + syllable_open_close_detector, + tone_detector, +) from pythainlp.util.thai import ( + analyze_thai_text, count_thai_chars, countthai, display_thai_char, isthai, isthaichar, thai_word_tone_detector, - analyze_thai_text, ) from pythainlp.util.thai_lunar_date import th_zodiac, to_lunar_date from pythainlp.util.thaiwordcheck import is_native_thai from pythainlp.util.time import thaiword_to_time, time_to_thaiword from pythainlp.util.trie import Trie, dict_trie from pythainlp.util.wordtonum import text_to_num, thaiword_to_num, words_to_num - -# sound_syllable and pronounce have to be imported last, -# to prevent circular import issues. -# Other imports should be above this line, sorted. -from pythainlp.util.syllable import ( - sound_syllable, - syllable_length, - syllable_open_close_detector, - tone_detector, -) -from pythainlp.util.pronounce import ( - rhyme, - spelling, - tone_to_spelling, - thai_consonant_to_spelling, -) diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index a46b876c7..fbb844314 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -1,14 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Thai abbreviation tools """ -from typing import List, Tuple, Union +from __future__ import annotations -def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[float, None]]]: + +def abbreviation_to_full_text( + text: str, top_k: int = 2 +) -> list[tuple[str, float | None]]: """ This function converts Thai text (with abbreviation) to full text. @@ -29,7 +31,7 @@ def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[ abbreviation_to_full_text(text) # output: [ - # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), + # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), # ('โรงแรมของเราน่าอยู่', tensor(0.2438)) # ] """ diff --git a/pythainlp/util/collate.py b/pythainlp/util/collate.py index 0c28426cc..10bd9a511 100644 --- a/pythainlp/util/collate.py +++ b/pythainlp/util/collate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,8 +5,11 @@ Thai collation (sorted according to Thai dictionary order) Simple implementation using regular expressions """ + +from __future__ import annotations + import re -from typing import Iterable, List +from collections.abc import Iterable _RE_TONE = re.compile(r"[็-์]") _RE_LV_C = re.compile(r"([เ-ไ])([ก-ฮ])") @@ -22,7 +24,7 @@ def _thkey(word: str) -> str: return cv + tone -def collate(data: Iterable, reverse: bool = False) -> List[str]: +def collate(data: Iterable, reverse: bool = False) -> list[str]: """ This function sorts strings (almost) according to Thai dictionary. diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index 6e101b607..d8c3bfc14 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -12,6 +11,7 @@ # AD คือ ค.ศ. # AH ปีฮิจเราะห์ศักราชเป็นปีพุทธศักราช จะต้องบวกด้วย 1122 # ไม่ได้รองรับปี พ.ศ. ก่อนการเปลี่ยนวันขึ้นปีใหม่ของประเทศไทย +from __future__ import annotations __all__ = [ "convert_years", @@ -25,7 +25,6 @@ import re from datetime import datetime, timedelta -from typing import Union try: from zoneinfo import ZoneInfo @@ -84,17 +83,20 @@ ["กันยายน", "กันยา", "ก.ย.", "09", "9"], ["ตุลาคม", "ตุลา", "ต.ค.", "10"], ["พฤศจิกายน", "พฤศจิกา", "พ.ย.", "11"], - ["ธันวาคม", "ธันวา", "ธ.ค.", "12"] + ["ธันวาคม", "ธันวา", "ธ.ค.", "12"], ] -thai_full_month_lists_regex = "(" + '|'.join( - ['|'.join(i) for i in thai_full_month_lists] -) + ")" +thai_full_month_lists_regex = ( + "(" + "|".join(["|".join(i) for i in thai_full_month_lists]) + ")" +) year_all_regex = r"(\d\d\d\d|\d\d)" -dates_list = "(" + '|'.join( - [str(i) for i in range(32, 0, -1)] + [ - "0" + str(i) for i in range(1, 10) - ] -) + ")" +dates_list = ( + "(" + + "|".join( + [str(i) for i in range(32, 0, -1)] + + ["0" + str(i) for i in range(1, 10)] + ) + + ")" +) _DAY = { "วันนี้": 0, @@ -145,7 +147,7 @@ def convert_years(year: str, src="be", target="ad") -> str: # พ.ศ. - 543  = ค.ศ. if target == "ad": output_year = str(int(year) - 543) - # พ.ศ. - 2324 = ร.ศ.  + # พ.ศ. - 2324 = ร.ศ. elif target == "re": output_year = str(int(year) - 2324) # พ.ศ. - 1122 = ฮ.ศ. @@ -200,7 +202,7 @@ def thai_strptime( fmt: str, year: str = "be", add_year: int = None, - tzinfo=ZoneInfo("Asia/Bangkok") + tzinfo=ZoneInfo("Asia/Bangkok"), ): """ Thai strptime @@ -264,27 +266,28 @@ def thai_strptime( if "%f" in fmt: fmt = fmt.replace("%f", r"(\d+)") keys = [ - i.strip().strip('-').strip(':').strip('.') - for i in _old.split("%") if i != '' + i.strip().strip("-").strip(":").strip(".") + for i in _old.split("%") + if i != "" ] y = re.findall(fmt, text) - data = {i: ''.join(list(j)) for i, j in zip(keys, y[0])} + data = {i: "".join(list(j)) for i, j in zip(keys, y[0])} H = 0 M = 0 S = 0 f = 0 - d = data['d'] - m = _find_month(data['B']) - y = data['Y'] + d = data["d"] + m = _find_month(data["B"]) + y = data["Y"] if "H" in keys: - H = data['H'] + H = data["H"] if "M" in keys: - M = data['M'] + M = data["M"] if "S" in keys: - S = data['S'] + S = data["S"] if "f" in keys: - f = data['f'] + f = data["f"] if int(y) < 100 and year == "be": if add_year is None: y = str(2500 + int(y)) @@ -305,7 +308,7 @@ def thai_strptime( minute=int(M), second=int(S), microsecond=int(f), - tzinfo=tzinfo + tzinfo=tzinfo, ) @@ -369,9 +372,7 @@ def reign_year_to_ad(reign_year: int, reign: int) -> int: return ad -def thaiword_to_date( - text: str, date: datetime = None -) -> Union[datetime, None]: +def thaiword_to_date(text: str, date: datetime = None) -> datetime | None: """ Convert Thai relative date to :class:`datetime.datetime`. diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index 9fa2294c4..ebd622069 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Convert digits """ +from __future__ import annotations + _arabic_thai = { "0": "๐", "1": "๑", @@ -78,7 +79,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: from pythainlp.util import thai_digit_to_arabic_digit - text = 'เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท' + text = "เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท" thai_digit_to_arabic_digit(text) # output: เป็นจำนวน 123,400.25 บาท @@ -104,7 +105,7 @@ def arabic_digit_to_thai_digit(text: str) -> str: from pythainlp.util import arabic_digit_to_thai_digit - text = 'เป็นจำนวน 123,400.25 บาท' + text = "เป็นจำนวน 123,400.25 บาท" arabic_digit_to_thai_digit(text) # output: เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index 7be117158..0f324c5ea 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -6,6 +6,8 @@ Convert emojis """ +from __future__ import annotations + import re _emoji_th = { diff --git a/pythainlp/util/encoding.py b/pythainlp/util/encoding.py index a741fc99e..3f0c164d0 100644 --- a/pythainlp/util/encoding.py +++ b/pythainlp/util/encoding.py @@ -2,7 +2,10 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -def tis620_to_utf8(text: str)->str: +from __future__ import annotations + + +def tis620_to_utf8(text: str) -> str: """ Convert TIS-620 to UTF-8 diff --git a/pythainlp/util/keyboard.py b/pythainlp/util/keyboard.py index f4e824b98..a86c2ce65 100644 --- a/pythainlp/util/keyboard.py +++ b/pythainlp/util/keyboard.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Functions related to keyboard layout. """ +from __future__ import annotations + EN_TH_KEYB_PAIRS = { "Z": "(", "z": "ผ", diff --git a/pythainlp/util/keywords.py b/pythainlp/util/keywords.py index 13da2db1b..a555c41f3 100644 --- a/pythainlp/util/keywords.py +++ b/pythainlp/util/keywords.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from collections import Counter -from typing import Dict, List from pythainlp.corpus import thai_stopwords _STOPWORDS = thai_stopwords() -def rank(words: List[str], exclude_stopwords: bool = False) -> Counter: +def rank(words: list[str], exclude_stopwords: bool = False) -> Counter: """ Count word frequencies given a list of Thai words with an option to exclude stopwords. @@ -72,7 +72,7 @@ def rank(words: List[str], exclude_stopwords: bool = False) -> Counter: return Counter(words) -def find_keyword(word_list: List[str], min_len: int = 3) -> Dict[str, int]: +def find_keyword(word_list: list[str], min_len: int = 3) -> dict[str, int]: """ This function counts the frequencies of words in the list where stopword is excluded and returns a frequency dictionary. diff --git a/pythainlp/util/lcs.py b/pythainlp/util/lcs.py index 6dac741f1..a6f29f8d4 100644 --- a/pythainlp/util/lcs.py +++ b/pythainlp/util/lcs.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + def longest_common_subsequence(str1: str, str2: str) -> str: """ @@ -48,7 +49,6 @@ def longest_common_subsequence(str1: str, str2: str) -> str: i = m j = n while i > 0 and j > 0: - # If current character in str1 and str2 are same, then # current character is part of LCS if str1[i - 1] == str2[j - 1]: diff --git a/pythainlp/util/morse.py b/pythainlp/util/morse.py index 6717e22d8..6b5544d1c 100644 --- a/pythainlp/util/morse.py +++ b/pythainlp/util/morse.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations THAI_MORSE_CODE = { "ก": "--.", @@ -146,6 +146,7 @@ def morse_encode(text: str, lang: str = "th") -> str: :: from pythainlp.util.morse import morse_encode + print(morse_encode("แมว", lang="th")) # output: .-.- -- .-- diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index 975d8188b..7f9b29421 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,8 +5,9 @@ Text normalization """ +from __future__ import annotations + import re -from typing import List, Union from pythainlp import thai_above_vowels as above_v from pythainlp import thai_below_vowels as below_v @@ -251,7 +251,7 @@ def normalize(text: str) -> str: return text -def expand_maiyamok(sent: Union[str, List[str]]) -> List[str]: +def expand_maiyamok(sent: str | list[str]) -> list[str]: """ Expand Maiyamok. @@ -311,7 +311,7 @@ def expand_maiyamok(sent: Union[str, List[str]]) -> List[str]: return output_toks[::-1] -def maiyamok(sent: Union[str, List[str]]) -> List[str]: +def maiyamok(sent: str | list[str]) -> list[str]: """ Expand Maiyamok. diff --git a/pythainlp/util/numtoword.py b/pythainlp/util/numtoword.py index 7d1f43ad5..8f19367b2 100644 --- a/pythainlp/util/numtoword.py +++ b/pythainlp/util/numtoword.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -10,6 +9,8 @@ https://suksit.com/post/writing-bahttext-in-php/ """ +from __future__ import annotations + __all__ = ["bahttext", "num_to_thaiword"] _VALUES = [ @@ -61,7 +62,7 @@ def bahttext(number: float) -> str: elif number == 0: ret = "ศูนย์บาทถ้วน" else: - num_int, num_dec = "{:.2f}".format(number).split(".") + num_int, num_dec = f"{number:.2f}".split(".") num_int = int(num_int) num_dec = int(num_dec) diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index ecdb2d84f..c8c75d5f6 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -1,12 +1,14 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Phonemes util """ -from functools import lru_cache + +from __future__ import annotations + import unicodedata +from functools import lru_cache from pythainlp.tokenize import Tokenizer from pythainlp.util.trie import Trie @@ -194,6 +196,7 @@ def nectec_to_ipa(pronunciation: str) -> str: dict_ipa_rtgs_final = {"w": "o"} + @lru_cache def _ipa_cut(): """Lazy load IPA tokenizer with cache""" diff --git a/pythainlp/util/pronounce.py b/pythainlp/util/pronounce.py index 4c69d67f4..e3dd01002 100644 --- a/pythainlp/util/pronounce.py +++ b/pythainlp/util/pronounce.py @@ -1,22 +1,21 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List +from __future__ import annotations + import re +from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_words from pythainlp.khavee import KhaveeVerifier -from pythainlp.tokenize import syllable_tokenize -from pythainlp.tokenize import Tokenizer -from pythainlp import thai_consonants, thai_tonemarks +from pythainlp.tokenize import Tokenizer, syllable_tokenize from pythainlp.util import remove_tonemark kv = KhaveeVerifier() all_thai_words_dict = None -def rhyme(word: str) -> List[str]: +def rhyme(word: str) -> list[str]: """ Find Thai rhyme @@ -44,10 +43,12 @@ def rhyme(word: str) -> List[str]: return sorted(list_sumpus) -thai_vowel = ''.join(( - "อะ,อา,อิ,อี,อึ,อื,อุ,อู,เอะ,เอ,แอะ,แอ,เอียะ,เอีย,เอือะ,เอือ,อัวะ,อัว,โอะ,", - "โอ,เอาะ,ออ,เออะ,เออ,อำ,ใอ,ไอ,เอา,ฤ,ฤๅ,ฦ,ฦๅ" -)).split(",") +thai_vowel = "".join( + ( + "อะ,อา,อิ,อี,อึ,อื,อุ,อู,เอะ,เอ,แอะ,แอ,เอียะ,เอีย,เอือะ,เอือ,อัวะ,อัว,โอะ,", + "โอ,เอาะ,ออ,เออะ,เออ,อำ,ใอ,ไอ,เอา,ฤ,ฤๅ,ฦ,ฦๅ", + ) +).split(",") thai_vowel_all = [ ("([ก-ฮ])ะ", "\\1อะ"), ("([ก-ฮ])า", "\\1อา"), @@ -116,7 +117,7 @@ def tone_to_spelling(t: str) -> str: from pythainlp.util import tone_to_spelling - print(tone_to_spelling("่")) # ไม้เอก + print(tone_to_spelling("่")) # ไม้เอก # output: ไม้เอก """ if t == "่": @@ -130,7 +131,7 @@ def tone_to_spelling(t: str) -> str: return t -def spelling(word: str) -> List[str]: +def spelling(word: str) -> list[str]: """ Thai word to spelling @@ -154,8 +155,7 @@ def spelling(word: str) -> List[str]: if not word or not isinstance(word, str): return [] thai_vowel_tokenizer = Tokenizer( - custom_dict=thai_vowel + list(thai_consonants), - engine="longest" + custom_dict=thai_vowel + list(thai_consonants), engine="longest" ) word_pre = remove_tonemark(word).replace("็", "") tone = [tone_to_spelling(i) for i in word if i in thai_tonemarks] @@ -169,8 +169,9 @@ def spelling(word: str) -> List[str]: break list_word_output = thai_vowel_tokenizer.word_tokenize(word_output) output = [ - i for i in [thai_consonant_to_spelling(i) for i in list_word_output] - if '์' not in i + i + for i in [thai_consonant_to_spelling(i) for i in list_word_output] + if "์" not in i ] if word_pre == word: return output + [word] diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index 62e55aa2c..dfff7e87c 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -1,15 +1,16 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Removement of repeated consonants at the end of words """ -from typing import Iterable, List, Tuple + +from __future__ import annotations + +from collections.abc import Iterable from pythainlp import thai_consonants as consonants from pythainlp.corpus import thai_words -from pythainlp.util.trie import Trie # used by remove_trailing_repeat_consonants() # contains all words that has repeating consonants at the end @@ -212,8 +213,8 @@ def _is_last_consonant_repeater(word: str) -> bool: def _find_longest_consonant_repeaters_match( - segment_head: str, repeaters: List[str] -) -> Tuple[str, int]: + segment_head: str, repeaters: list[str] +) -> tuple[str, int]: """ Find the longest word that matches the segment. diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index 8ab43f0aa..a3228ed79 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from functools import lru_cache +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp import ( thai_above_vowels, @@ -49,10 +49,13 @@ for i in thai_below_vowels: dict_vowel[i] = "อ" + i + @lru_cache def _cut(): """Lazy load vowel tokenizer with cache""" - return Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") + return Tokenizer( + list(dict_vowel.keys()) + list(thai_consonants), engine="mm" + ) def _clean(w): @@ -81,7 +84,7 @@ def _clean(w): return w -def spell_syllable(text: str) -> List[str]: +def spell_syllable(text: str) -> list[str]: """ Spell out syllables in Thai word distribution form. @@ -106,7 +109,7 @@ def spell_syllable(text: str) -> List[str]: return c_only + v_only + t_only + [text] -def spell_word(text: str) -> List[str]: +def spell_word(text: str) -> list[str]: """ Spell out words in Thai word distribution form. diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index e96299b44..962d519da 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Thai date/time formatting. """ +from __future__ import annotations + import warnings from datetime import datetime from string import digits @@ -36,7 +37,7 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: str_ = "" try: str_ = dt_obj.strftime(f"%{fmt_char}") - if not str_ or str_ == "%{}".format(fmt_char): + if not str_ or str_ == f"%{fmt_char}": # Normalize outputs for unsupported directives # in different platforms: # "%Q" may result "", "%Q", or "Q", make it all "Q" @@ -114,21 +115,13 @@ def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: ).zfill(2) elif fmt_char == "v": # BSD extension, ' 6-Oct-1976' - str_ = "{:>2}-{}-{}".format( - dt_obj.day, - thai_abbr_months[dt_obj.month - 1], - str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4), - ) + str_ = f"{dt_obj.day:>2}-{thai_abbr_months[dt_obj.month - 1]}-{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" elif fmt_char == "X": # Locale’s appropriate time representation. str_ = dt_obj.strftime("%H:%M:%S") elif fmt_char == "x": # Locale’s appropriate date representation. - str_ = "{}/{}/{}".format( - str(dt_obj.day).zfill(2), - str(dt_obj.month).zfill(2), - str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4), - ) + str_ = f"{str(dt_obj.day).zfill(2)}/{str(dt_obj.month).zfill(2)}/{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" elif fmt_char == "Y": # Year with century str_ = (str(dt_obj.year + _BE_AD_DIFFERENCE)).zfill(4) diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index acb78e769..0fd51aef9 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,6 +5,8 @@ Syllable tools """ +from __future__ import annotations + import re from pythainlp import thai_consonants, thai_tonemarks diff --git a/pythainlp/util/thai.py b/pythainlp/util/thai.py index d673d1beb..4fb053828 100644 --- a/pythainlp/util/thai.py +++ b/pythainlp/util/thai.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -6,8 +5,9 @@ Check if it is Thai text """ +from __future__ import annotations + import string -from typing import Tuple from collections import defaultdict from pythainlp import ( @@ -217,7 +217,7 @@ def display_thai_char(ch: str) -> str: return ch -def thai_word_tone_detector(word: str) -> Tuple[str, str]: +def thai_word_tone_detector(word: str) -> tuple[str, str]: """ Thai tone detector for word. @@ -345,9 +345,9 @@ def analyze_thai_text(text: str) -> dict: # Check if the character is in our mapping if char in THAI_CHAR_NAMES: name = THAI_CHAR_NAMES[char] - results[name]+=1 + results[name] += 1 else: # If the character is not a known Thai character, classify it as character - results[char]+=1 + results[char] += 1 return dict(results) diff --git a/pythainlp/util/thai_lunar_date.py b/pythainlp/util/thai_lunar_date.py index b416d9443..92dd75890 100644 --- a/pythainlp/util/thai_lunar_date.py +++ b/pythainlp/util/thai_lunar_date.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,10 +6,11 @@ > https://gist.github.com/touchiep/99f4f5bb349d6b983ef78697630ab78e """ +from __future__ import annotations + from datetime import date, timedelta -from typing import Dict, List, Tuple, Union -_YEAR_DEV: Dict[int, float] = { +_YEAR_DEV: dict[int, float] = { 0: 0, 1901: 0.122733000004352, 1906: 1.91890000045229e-02, @@ -190,7 +190,7 @@ _DAYS_384 = [29, 30, 29, 30, 29, 30, 29, 30, 30, 29, 30, 29, 30, 29, 30] # Zodiac names in Thai, English, and Numeric representations -_ZODIAC: Dict[int, List[Union[str, int]]] = { +_ZODIAC: dict[int, list[str | int]] = { 1: [ "ชวด", "ฉลู", @@ -223,7 +223,7 @@ } -def _calculate_f_year_f_dev(year: int) -> Tuple[int, float]: +def _calculate_f_year_f_dev(year: int) -> tuple[int, float]: if year in _YEAR_DEV: return year, _YEAR_DEV[year] @@ -308,7 +308,7 @@ def number_day_in_year(year: int) -> int: return 365 -def th_zodiac(year: int, output_type: int = 1) -> Union[str, int]: +def th_zodiac(year: int, output_type: int = 1) -> str | int: """ Thai Zodiac Year Name Converts a Gregorian year to its corresponding Zodiac name. diff --git a/pythainlp/util/thaiwordcheck.py b/pythainlp/util/thaiwordcheck.py index cc4bed0ab..106dfbdf8 100644 --- a/pythainlp/util/thaiwordcheck.py +++ b/pythainlp/util/thaiwordcheck.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from pythainlp.tools import warn_deprecation diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index 85738b94a..ae5fd9b90 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,9 +6,11 @@ Convert time string or time object to Thai words. """ + +from __future__ import annotations + from datetime import datetime, time from functools import lru_cache -from typing import Union from pythainlp.tokenize import Tokenizer from pythainlp.util.numtoword import num_to_thaiword @@ -45,12 +46,13 @@ "ครึ่ง": 30, } + @lru_cache def _thai_time_cut(): """Lazy load Thai time tokenizer with cache""" - return Tokenizer( - custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" - ) + return Tokenizer(custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm") + + _THAI_TIME_AFFIX = [ "โมงเช้า", "บ่ายโมง", @@ -121,7 +123,7 @@ def _format( m: int, s: int, fmt: str = "24h", - precision: Union[str, None] = None, + precision: str | None = None, ) -> str: text = "" if fmt == "6h": @@ -153,9 +155,9 @@ def _format( def time_to_thaiword( - time_data: Union[time, datetime, str], + time_data: time | datetime | str, fmt: str = "24h", - precision: Union[str, None] = None, + precision: str | None = None, ) -> str: """ Spell out time as Thai words. diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index d08aa07c1..869cad34d 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,7 +6,10 @@ Designed to be used for tokenizer's dictionary, but can be for other purposes. """ -from typing import Iterable, Iterator, List, Union + +from __future__ import annotations + +from collections.abc import Iterable, Iterator class Trie(Iterable[str]): @@ -69,7 +71,7 @@ def remove(self, word: str) -> None: break del parent.children[ch] # remove from parent dict - def prefixes(self, text: str) -> List[str]: + def prefixes(self, text: str) -> list[str]: """ List all possible words from first sequence of characters in a word. @@ -98,7 +100,7 @@ def __len__(self) -> int: return len(self.words) -def dict_trie(dict_source: Union[str, Iterable[str], Trie]) -> Trie: +def dict_trie(dict_source: str | Iterable[str] | Trie) -> Trie: """ Create a dictionary trie from a file or an iterable. @@ -111,7 +113,7 @@ def dict_trie(dict_source: Union[str, Iterable[str], Trie]) -> Trie: if isinstance(dict_source, str) and len(dict_source) > 0: # dict_source is a path to dictionary text file - with open(dict_source, "r", encoding="utf8") as f: + with open(dict_source, encoding="utf8") as f: _vocabs = f.read().splitlines() trie = Trie(_vocabs) elif isinstance(dict_source, Iterable) and not isinstance( diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 6b320b41e..3087cd7d9 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -8,9 +7,11 @@ First version of the code adapted from Korakot Chaovavanich's notebook https://colab.research.google.com/drive/148WNIeclf0kOU6QxKd6pcfwpSs8l-VKD#scrollTo=EuVDd0nNuI8Q """ -from functools import lru_cache + +from __future__ import annotations + import re -from typing import List +from functools import lru_cache from pythainlp.corpus import thai_words from pythainlp.tokenize import Tokenizer @@ -45,9 +46,9 @@ "แสน": 100000, # "ล้าน" was excluded as a special case } -_valid_tokens = ( - set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} -) +_valid_tokens = set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} + + @lru_cache def _tokenizer(): """Lazy load Thai numeral tokenizer with cache""" @@ -67,7 +68,9 @@ def _check_is_thainum(word: str): @lru_cache def _tokenizer_thaiwords(): """Lazy load Thai words tokenizer with cache""" - _dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] + _dict_words = [ + i for i in list(thai_words()) if not _check_is_thainum(i)[0] + ] _dict_words += list(_digits.keys()) _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] return Tokenizer(_dict_words) @@ -171,7 +174,7 @@ def words_to_num(words: list) -> float: return num -def text_to_num(text: str) -> List[str]: +def text_to_num(text: str) -> list[str]: """ Thai text to list of Thai words with floating point numbers diff --git a/pythainlp/wangchanberta/__init__.py b/pythainlp/wangchanberta/__init__.py index d3d565fe5..88d8e1957 100644 --- a/pythainlp/wangchanberta/__init__.py +++ b/pythainlp/wangchanberta/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index c6a071ea4..f14322fc6 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import re import warnings -from typing import List, Tuple, Union from transformers import ( CamembertTokenizer, @@ -56,7 +56,7 @@ def _clear_tag(self, tag): def get_ner( self, text: str, pos: bool = False, tag: bool = False - ) -> Union[List[Tuple[str, str]], str]: + ) -> list[tuple[str, str]] | str: """ This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -165,7 +165,7 @@ def _fix_span_error(self, words, ner): def get_ner( self, text: str, pos: bool = False, tag: bool = False - ) -> Union[List[Tuple[str, str]], str]: + ) -> list[tuple[str, str]] | str: """ This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -225,7 +225,7 @@ def get_ner( return ner_tag -def segment(text: str) -> List[str]: +def segment(text: str) -> list[str]: """ Subword tokenize. SentencePiece from wangchanberta model. diff --git a/pythainlp/word_vector/__init__.py b/pythainlp/word_vector/__init__.py index c8b0f9a6d..7aa5165f5 100644 --- a/pythainlp/word_vector/__init__.py +++ b/pythainlp/word_vector/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -7,6 +6,7 @@ Initial code from https://github.com/cstorm125/thai2fit """ + __all__ = [ "WordVector", ] diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 886c596f2..c032a0dc5 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple +from __future__ import annotations from gensim.models import KeyedVectors from gensim.models.keyedvectors import Word2VecKeyedVectors @@ -74,7 +73,7 @@ def get_model(self) -> Word2VecKeyedVectors: """ return self.model - def doesnt_match(self, words: List[str]) -> str: + def doesnt_match(self, words: list[str]) -> str: """ This function returns one word that is mostly unrelated to other words in the list. We use the function :func:`doesnt_match` @@ -96,7 +95,7 @@ def doesnt_match(self, words: List[str]) -> str: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> words = ['อาหารเช้า', 'อาหารเที่ยง', 'อาหารเย็น', 'พริกไทย'] + >>> words = ["อาหารเช้า", "อาหารเที่ยง", "อาหารเย็น", "พริกไทย"] >>> wv.doesnt_match(words) พริกไทย @@ -106,15 +105,15 @@ def doesnt_match(self, words: List[str]) -> str: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> words = ['ดีไซน์เนอร์', 'พนักงานเงินเดือน', 'หมอ', 'เรือ'] + >>> words = ["ดีไซน์เนอร์", "พนักงานเงินเดือน", "หมอ", "เรือ"] >>> wv.doesnt_match(words) เรือ """ return self.model.doesnt_match(words) def most_similar_cosmul( - self, positive: List[str], negative: List[str] - ) -> List[Tuple[str, float]]: + self, positive: list[str], negative: list[str] + ) -> list[tuple[str, float]]: """ This function finds the top-10 words that are most similar with respect to two lists of words labeled as positive and negative. @@ -147,7 +146,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['แม่น้ำ'] + >>> list_positive = ["แม่น้ำ"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ลำน้ำ', 0.8206598162651062), ('ทะเลสาบ', 0.775945782661438), @@ -162,7 +161,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['นายก', 'รัฐมนตรี', 'ประเทศ'] + >>> list_positive = ["นายก", "รัฐมนตรี", "ประเทศ"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('รองนายกรัฐมนตรี', 0.2730445861816406), @@ -180,7 +179,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['ประเทศ', 'ไทย', 'จีน', 'ญี่ปุ่น'] + >>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ประเทศจีน', 0.22022421658039093), ('เกาหลี', 0.2196873426437378), @@ -191,8 +190,8 @@ def most_similar_cosmul( ('อังกฤษ', 0.19610872864723206), ('ฮ่องกง', 0.1928885132074356), ('ฝรั่งเศส', 0.18383873999118805), ('พม่า', 0.18369348347187042)] >>> - >>> list_positive = ['ประเทศ', 'ไทย', 'จีน', 'ญี่ปุ่น'] - >>> list_negative = ['อเมริกา'] + >>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"] + >>> list_negative = ["อเมริกา"] >>> wv.most_similar_cosmul(list_positive, list_negative) [('ประเทศไทย', 0.3278159201145172), ('เกาหลี', 0.3201899230480194), ('ประเทศจีน', 0.31755179166793823), ('พม่า', 0.30845439434051514), @@ -207,7 +206,7 @@ def most_similar_cosmul( >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> list_positive = ['เมนูอาหารไทย'] + >>> list_positive = ["เมนูอาหารไทย"] >>> list_negative = [] >>> wv.most_similar_cosmul(list_positive, list_negative) KeyError: "word 'เมนูอาหารไทย' not in vocabulary" @@ -239,7 +238,7 @@ def similarity(self, word1: str, word2: str) -> float: >>> from pythainlp.word_vector import WordVector >>> wv = WordVector() - >>> wv.similarity('รถไฟ', 'รถไฟฟ้า') + >>> wv.similarity("รถไฟ", "รถไฟฟ้า") 0.43387136 @@ -249,7 +248,7 @@ def similarity(self, word1: str, word2: str) -> float: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> wv.similarity('เสือดาว', 'รถไฟฟ้า') + >>> wv.similarity("เสือดาว", "รถไฟฟ้า") 0.04300258 """ @@ -282,7 +281,7 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: >>> from pythainlp.word_vector import WordVector >>> >>> wv = WordVector() - >>> sentence = 'อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735' + >>> sentence = "อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735" >>> wv.sentence_vectorizer(sentence, use_mean=True) array([[-0.00421414, -0.08881307, 0.05081136, -0.05632929, -0.06607185, 0.03059357, -0.113882 , -0.00074836, 0.05035743, diff --git a/pythainlp/wsd/__init__.py b/pythainlp/wsd/__init__.py index f2933bb1a..aeb7cd113 100644 --- a/pythainlp/wsd/__init__.py +++ b/pythainlp/wsd/__init__.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ Thai Word Sense Disambiguation (WSD) """ + __all__ = ["get_sense"] from pythainlp.wsd.core import get_sense diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index add2b3ad7..8b947bf02 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -from typing import List, Tuple, Union +from __future__ import annotations from pythainlp.corpus import thai_wsd_dict from pythainlp.tokenize import Tokenizer @@ -53,7 +52,7 @@ def get_sense( device: str = "cpu", custom_dict: dict = dict(), custom_tokenizer: Tokenizer = _word_cut, -) -> List[Tuple[str, float]]: +) -> list[tuple[str, float]]: """ Get word sense from the sentence. This function will get definition and distance from context in sentence. diff --git a/setup.py b/setup.py index 0b3951cd1..0b0a3c2d0 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 From 01a0a1fc7966c133867a6d8fdd1fb9382275238d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 22:34:41 +0000 Subject: [PATCH 3/3] Fix circular import issues caused by import sorting Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/__init__.py | 28 ++++++++++++++-------------- pythainlp/util/__init__.py | 32 ++++++++++++++++---------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index ad212dfa4..94b158c0e 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -92,6 +92,20 @@ def corpus_db_path() -> str: return _CORPUS_DB_PATH +from pythainlp.corpus.core import ( + download, + get_corpus, + get_corpus_as_is, + get_corpus_db, + get_corpus_db_detail, + get_corpus_default_db, + get_corpus_path, + get_hf_hub, + get_path_folder_corpus, + make_safe_directory_name, + path_pythainlp_corpus, + remove, +) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.common import ( countries, find_synonyms, @@ -109,20 +123,6 @@ def corpus_db_path() -> str: thai_words, thai_wsd_dict, ) -from pythainlp.corpus.core import ( - download, - get_corpus, - get_corpus_as_is, - get_corpus_db, - get_corpus_db_detail, - get_corpus_default_db, - get_corpus_path, - get_hf_hub, - get_path_folder_corpus, - make_safe_directory_name, - path_pythainlp_corpus, - remove, -) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.icu import thai_icu_words from pythainlp.corpus.volubilis import thai_volubilis_words from pythainlp.corpus.wikipedia import thai_wikipedia_titles diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index a1c28bbe7..ffee432f3 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -110,26 +110,10 @@ ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa -from pythainlp.util.pronounce import ( - rhyme, - spelling, - thai_consonant_to_spelling, - tone_to_spelling, -) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) from pythainlp.util.strftime import thai_strftime - -# sound_syllable and pronounce have to be imported last, -# to prevent circular import issues. -# Other imports should be above this line, sorted. -from pythainlp.util.syllable import ( - sound_syllable, - syllable_length, - syllable_open_close_detector, - tone_detector, -) from pythainlp.util.thai import ( analyze_thai_text, count_thai_chars, @@ -144,3 +128,19 @@ from pythainlp.util.time import thaiword_to_time, time_to_thaiword from pythainlp.util.trie import Trie, dict_trie from pythainlp.util.wordtonum import text_to_num, thaiword_to_num, words_to_num + +# sound_syllable and pronounce have to be imported last, +# to prevent circular import issues. +# Other imports should be above this line, sorted. +from pythainlp.util.syllable import ( + sound_syllable, + syllable_length, + syllable_open_close_detector, + tone_detector, +) +from pythainlp.util.pronounce import ( + rhyme, + spelling, + thai_consonant_to_spelling, + tone_to_spelling, +)