diff --git a/pyproject.toml b/pyproject.toml
index d84632ba3..b01c8fd28 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,7 @@
[tool.ruff]
line-length = 79
indent-width = 4
-target-version = "py310"
+target-version = "py39"
[tool.ruff.format]
quote-style = "double"
diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py
index 7e0a61544..bd43ddd25 100644
--- a/pythainlp/__init__.py
+++ b/pythainlp/__init__.py
@@ -1,64 +1,63 @@
-# -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
-# SPDX-FileType: SOURCE
-# SPDX-License-Identifier: Apache-2.0
-__version__ = "5.2.0"
-
-thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars
-
-thai_vowels = (
- "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37"
- + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47"
-) # 20
-thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5
-thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4
-thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7
-thai_below_vowels = "\u0e38\u0e39" # 2
-
-thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4
-
-# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan:
-# These signs can be part of a word
-thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars
-
-# Any Thai character that can be part of a word
-thai_letters = "".join(
- [thai_consonants, thai_vowels, thai_tonemarks, thai_signs]
-) # 74
-
-# Fongman, Angkhankhu, Khomut:
-# These characters are section markers
-thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars
-
-thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10
-thai_symbols = "\u0e3f" # Thai Bath ฿
-
-# All Thai characters that are presented in Unicode
-thai_characters = "".join(
- [thai_letters, thai_punctuations, thai_digits, thai_symbols]
-)
-# Thai pangram by Sungsit Sawaiwan
-# CC BY-SA License
-# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html
-thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐
-๏ จับฅอคนบั่นต้อง อาญา
-ขุดฆ่าโคตรฃัตติยา ซ่านม้วย
-ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ
-สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ
-๏ กตัญญูไป่พร้อม ปฐมฌาน
-เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ
-ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ
-โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛
-๑๒ กรกฎาคม ๒๕๕๘"""
-
-from pythainlp.soundex import soundex
-from pythainlp.spell import correct, spell
-from pythainlp.tag import pos_tag
-from pythainlp.tokenize import (
- Tokenizer,
- sent_tokenize,
- subword_tokenize,
- word_tokenize,
-)
-from pythainlp.transliterate import romanize, transliterate
-from pythainlp.util import collate, thai_strftime
+# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
+# SPDX-FileType: SOURCE
+# SPDX-License-Identifier: Apache-2.0
+__version__ = "5.2.0"
+
+thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars
+
+thai_vowels = (
+ "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37"
+ + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47"
+) # 20
+thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5
+thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4
+thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7
+thai_below_vowels = "\u0e38\u0e39" # 2
+
+thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4
+
+# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan:
+# These signs can be part of a word
+thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars
+
+# Any Thai character that can be part of a word
+thai_letters = "".join(
+ [thai_consonants, thai_vowels, thai_tonemarks, thai_signs]
+) # 74
+
+# Fongman, Angkhankhu, Khomut:
+# These characters are section markers
+thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars
+
+thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10
+thai_symbols = "\u0e3f" # Thai Bath ฿
+
+# All Thai characters that are presented in Unicode
+thai_characters = "".join(
+ [thai_letters, thai_punctuations, thai_digits, thai_symbols]
+)
+# Thai pangram by Sungsit Sawaiwan
+# CC BY-SA License
+# Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html
+thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐
+๏ จับฅอคนบั่นต้อง อาญา
+ขุดฆ่าโคตรฃัตติยา ซ่านม้วย
+ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ
+สืบอยู่เต็มศึกด้วย ฝุ่นฟ้ากีฬา กามฦๅ ฯ
+๏ กตัญญูไป่พร้อม ปฐมฌาน
+เกมส๎วัฒน์ปฏิภาณ ห่อนล้ำ
+ทฤษฎีถ่อยๆ สังหาร เกณฑ์โทษ
+โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛
+๑๒ กรกฎาคม ๒๕๕๘"""
+
+from pythainlp.soundex import soundex
+from pythainlp.spell import correct, spell
+from pythainlp.tag import pos_tag
+from pythainlp.tokenize import (
+ Tokenizer,
+ sent_tokenize,
+ subword_tokenize,
+ word_tokenize,
+)
+from pythainlp.transliterate import romanize, transliterate
+from pythainlp.util import collate, thai_strftime
diff --git a/pythainlp/__main__.py b/pythainlp/__main__.py
index cb9d53a45..91a280366 100644
--- a/pythainlp/__main__.py
+++ b/pythainlp/__main__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py
index 4a30d29c6..f8d7b41ae 100644
--- a/pythainlp/ancient/__init__.py
+++ b/pythainlp/ancient/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py
index 8769993fe..15e41e5f3 100644
--- a/pythainlp/ancient/aksonhan.py
+++ b/pythainlp/ancient/aksonhan.py
@@ -1,7 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
from pythainlp import thai_consonants, thai_tonemarks
from pythainlp.corpus import thai_orst_words
diff --git a/pythainlp/ancient/currency.py b/pythainlp/ancient/currency.py
index bdf685e96..0725d780d 100644
--- a/pythainlp/ancient/currency.py
+++ b/pythainlp/ancient/currency.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
def convert_currency(value: float, from_unit: str) -> dict:
"""
@@ -44,14 +45,14 @@ def convert_currency(value: float, from_unit: str) -> dict:
# }
"""
conversion_factors_to_att = {
- 'เบี้ย': 1,
- 'อัฐ': 100, # 1 อัฐ = 100 เบี้ย
- 'ไพ': 2 * 100, # 1 ไพ = 2 อัฐ
- 'เฟื้อง': 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ
- 'สลึง': 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง
- 'บาท': 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง
- 'ตำลึง': 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท
- 'ชั่ง': 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง
+ "เบี้ย": 1,
+ "อัฐ": 100, # 1 อัฐ = 100 เบี้ย
+ "ไพ": 2 * 100, # 1 ไพ = 2 อัฐ
+ "เฟื้อง": 4 * 2 * 100, # 1 เฟื้อง = 4 ไพ
+ "สลึง": 2 * 4 * 2 * 100, # 1 สลึง = 2 เฟื้อง
+ "บาท": 4 * 2 * 4 * 2 * 100, # 1 บาท = 4 สลึง
+ "ตำลึง": 4 * 4 * 2 * 4 * 2 * 100, # 1 ตำลึง = 4 บาท
+ "ชั่ง": 20 * 4 * 4 * 2 * 4 * 2 * 100, # 1 ชั่ง = 20 ตำลึง
}
if from_unit not in conversion_factors_to_att:
diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py
index c82c17995..8935333b6 100644
--- a/pythainlp/augment/__init__.py
+++ b/pythainlp/augment/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py
index 1707936a4..f4a69dee5 100644
--- a/pythainlp/augment/lm/__init__.py
+++ b/pythainlp/augment/lm/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py
index a91af4b52..6bf31b237 100644
--- a/pythainlp/augment/lm/fasttext.py
+++ b/pythainlp/augment/lm/fasttext.py
@@ -1,9 +1,9 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import itertools
-from typing import List, Tuple
from gensim.models.fasttext import FastText as FastText_gensim
from gensim.models.keyedvectors import KeyedVectors
@@ -30,7 +30,7 @@ def __init__(self, model_path: str):
self.model = FastText_gensim.load(model_path)
self.dict_wv = list(self.model.key_to_index.keys())
- def tokenize(self, text: str) -> List[str]:
+ def tokenize(self, text: str) -> list[str]:
"""
Thai text tokenization for fastText
@@ -41,7 +41,7 @@ def tokenize(self, text: str) -> List[str]:
"""
return word_tokenize(text, engine="icu")
- def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]:
+ def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]:
"""
:param str sent: text of sentence
:param float p: probability
@@ -61,7 +61,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]:
def augment(
self, sentence: str, n_sent: int = 1, p: float = 0.7
- ) -> List[Tuple[str]]:
+ ) -> list[tuple[str]]:
"""
Text Augment from fastText
diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py
index fa7e9a8d3..73bdddf5a 100644
--- a/pythainlp/augment/lm/phayathaibert.py
+++ b/pythainlp/augment/lm/phayathaibert.py
@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
import random
import re
-from typing import List
from pythainlp.phayathaibert.core import ThaiTextProcessor
@@ -57,7 +56,7 @@ def generate(
def augment(
self, text: str, num_augs: int = 3, sample: bool = False
- ) -> List[str]:
+ ) -> list[str]:
"""
Text augmentation from PhayaThaiBERT
diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py
index 962e5f7cf..64702d4fe 100644
--- a/pythainlp/augment/lm/wangchanberta.py
+++ b/pythainlp/augment/lm/wangchanberta.py
@@ -1,9 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-
-from typing import List
+from __future__ import annotations
from transformers import (
CamembertTokenizer,
@@ -52,7 +50,7 @@ def generate(self, sentence: str, num_replace_tokens: int = 3):
masked_text = self.input_text
return self.sent2
- def augment(self, sentence: str, num_replace_tokens: int = 3) -> List[str]:
+ def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]:
"""
Text augmentation from WangchanBERTa
diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py
index 08786229f..8a51b2593 100644
--- a/pythainlp/augment/word2vec/__init__.py
+++ b/pythainlp/augment/word2vec/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py
index 635553f40..8d0a1a54c 100644
--- a/pythainlp/augment/word2vec/bpemb_wv.py
+++ b/pythainlp/augment/word2vec/bpemb_wv.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
from pythainlp.augment.word2vec.core import Word2VecAug
@@ -22,7 +21,7 @@ def __init__(self, lang: str = "th", vs: int = 100000, dim: int = 300):
self.model = self.bpemb_temp.emb
self.load_w2v()
- def tokenizer(self, text: str) -> List[str]:
+ def tokenizer(self, text: str) -> list[str]:
"""
:param str text: Thai text
:rtype: List[str]
@@ -39,7 +38,7 @@ def load_w2v(self):
def augment(
self, sentence: str, n_sent: int = 1, p: float = 0.7
- ) -> List[Tuple[str]]:
+ ) -> list[tuple[str]]:
"""
Text Augment using word2vec from BPEmb
diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py
index 345bf9097..72559ae43 100644
--- a/pythainlp/augment/word2vec/core.py
+++ b/pythainlp/augment/word2vec/core.py
@@ -1,9 +1,9 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import itertools
-from typing import List, Tuple
class Word2VecAug:
@@ -28,7 +28,7 @@ def __init__(
self.model = model
self.dict_wv = list(self.model.key_to_index.keys())
- def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]:
+ def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]:
"""
:param str sent: text of sentence
:param float p: probability
@@ -48,7 +48,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> List[List[str]]:
def augment(
self, sentence: str, n_sent: int = 1, p: float = 0.7
- ) -> List[Tuple[str]]:
+ ) -> list[tuple[str]]:
"""
:param str sentence: text of sentence
:param int n_sent: maximum number of synonymous sentences
diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py
index 3ae90e24b..fb73b76e6 100644
--- a/pythainlp/augment/word2vec/ltw2v.py
+++ b/pythainlp/augment/word2vec/ltw2v.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
from pythainlp.augment.word2vec.core import Word2VecAug
from pythainlp.corpus import get_corpus_path
@@ -21,7 +20,7 @@ def __init__(self):
self.ltw2v_wv = get_corpus_path("ltw2v")
self.load_w2v()
- def tokenizer(self, text: str) -> List[str]:
+ def tokenizer(self, text: str) -> list[str]:
"""
:param str text: Thai text
:rtype: List[str]
@@ -36,7 +35,7 @@ def load_w2v(self): # insert substitute
def augment(
self, sentence: str, n_sent: int = 1, p: float = 0.7
- ) -> List[Tuple[str]]:
+ ) -> list[tuple[str]]:
"""
Text Augment using word2vec from Thai2Fit
diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py
index 16aae20c2..5d0326852 100644
--- a/pythainlp/augment/word2vec/thai2fit.py
+++ b/pythainlp/augment/word2vec/thai2fit.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
from pythainlp.augment.word2vec.core import Word2VecAug
from pythainlp.corpus import get_corpus_path
@@ -21,7 +20,7 @@ def __init__(self):
self.thai2fit_wv = get_corpus_path("thai2fit_wv")
self.load_w2v()
- def tokenizer(self, text: str) -> List[str]:
+ def tokenizer(self, text: str) -> list[str]:
"""
:param str text: Thai text
:rtype: List[str]
@@ -37,7 +36,7 @@ def load_w2v(self):
def augment(
self, sentence: str, n_sent: int = 1, p: float = 0.7
- ) -> List[Tuple[str]]:
+ ) -> list[tuple[str]]:
"""
Text Augment using word2vec from Thai2Fit
diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py
index ea2ab3d70..074e4a60c 100644
--- a/pythainlp/augment/wordnet.py
+++ b/pythainlp/augment/wordnet.py
@@ -1,10 +1,12 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l
"""
+
+from __future__ import annotations
+
__all__ = [
"WordNetAug",
"postype2wordnet",
@@ -12,7 +14,6 @@
import itertools
from collections import OrderedDict
-from typing import List
from nltk.corpus import wordnet as wn
@@ -126,7 +127,7 @@ def __init__(self):
def find_synonyms(
self, word: str, pos: str = None, postag_corpus: str = "orchid"
- ) -> List[str]:
+ ) -> list[str]:
"""
Find synonyms using wordnet
@@ -162,7 +163,7 @@ def augment(
max_syn_sent: int = 6,
postag: bool = True,
postag_corpus: str = "orchid",
- ) -> List[List[str]]:
+ ) -> list[list[str]]:
"""
Text Augment using wordnet
diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py
index 66a012ffe..cc25d2705 100644
--- a/pythainlp/benchmarks/__init__.py
+++ b/pythainlp/benchmarks/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py
index 35bd0ae6c..ce9991cfc 100644
--- a/pythainlp/benchmarks/word_tokenization.py
+++ b/pythainlp/benchmarks/word_tokenization.py
@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
import re
import sys
-from typing import List, Tuple
import numpy as np
import pandas as pd
@@ -18,13 +17,13 @@
)
# regex for removing repeated separators, i.e. ||||
-MULTIPLE_SEPS_RX = re.compile("{sep}+".format(sep=re.escape(SEPARATOR)))
+MULTIPLE_SEPS_RX = re.compile(f"{re.escape(SEPARATOR)}+")
# regex for removing tags, i.e. ,
TAG_RX = re.compile(r"<\/?[A-Z]+>")
# regex for removing trailing separators, i.e. a|dog| -> a|dog
-TAILING_SEP_RX = re.compile("{sep}$".format(sep=re.escape(SEPARATOR)))
+TAILING_SEP_RX = re.compile(f"{re.escape(SEPARATOR)}$")
def _f1(precision: float, recall: float) -> float:
@@ -68,7 +67,7 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict:
return dict(items)
-def benchmark(ref_samples: List[str], samples: List[str]) -> pd.DataFrame:
+def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame:
"""
Performance benchmarking for samples.
@@ -184,9 +183,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict:
correctly_tokenised_words = np.sum(tokenization_indicators)
- tokenization_indicators = list(
- map(str, tokenization_indicators)
- )
+ tokenization_indicators = list(map(str, tokenization_indicators))
return {
"char_level": {
@@ -256,9 +253,9 @@ def _find_word_boundaries(bin_reps) -> list:
def _find_words_correctly_tokenised(
- ref_boundaries: List[Tuple[int, int]],
- predicted_boundaries: List[Tuple[int, int]],
-) -> Tuple[int]:
+ ref_boundaries: list[tuple[int, int]],
+ predicted_boundaries: list[tuple[int, int]],
+) -> tuple[int]:
"""
Find whether each word is correctly tokenized.
diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py
index 28698b252..2459ff1cd 100644
--- a/pythainlp/chat/__init__.py
+++ b/pythainlp/chat/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py
index 5f198ec84..3659b9b53 100644
--- a/pythainlp/chat/core.py
+++ b/pythainlp/chat/core.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import torch
@@ -69,7 +70,7 @@ def chat(self, text: str) -> str:
import torch
chatbot = ChatBotModel()
- chatbot.load_model(device="cpu",torch_dtype=torch.bfloat16)
+ chatbot.load_model(device="cpu", torch_dtype=torch.bfloat16)
print(chatbot.chat("สวัสดี"))
# output: ยินดีที่ได้รู้จัก
diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py
index af282e47f..ae305872b 100644
--- a/pythainlp/classify/__init__.py
+++ b/pythainlp/classify/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py
index e2a7fbe57..b3ccac213 100644
--- a/pythainlp/classify/param_free.py
+++ b/pythainlp/classify/param_free.py
@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
import gzip
import json
-from typing import List, Tuple
import numpy as np
@@ -20,7 +19,11 @@ class GzipModel:
:param str model_path: Path for loading model (if you saved the model)
"""
- def __init__(self, training_data: List[Tuple[str, str]] = None, model_path: str = None):
+ def __init__(
+ self,
+ training_data: list[tuple[str, str]] = None,
+ model_path: str = None,
+ ):
if model_path is not None:
self.load(model_path)
else:
@@ -47,7 +50,7 @@ def predict(self, x1: str, k: int = 1) -> str:
from pythainlp.classify import GzipModel
- training_data = [
+ training_data = [
("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"),
("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"),
("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"),
@@ -56,7 +59,7 @@ def predict(self, x1: str, k: int = 1) -> str:
("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"),
("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"),
("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"),
- ("นี่เป็นบทความหนึ่ง", "Neutral")
+ ("นี่เป็นบทความหนึ่ง", "Neutral"),
]
model = GzipModel(training_data)
print(model.predict("ฉันดีใจ", k=1))
@@ -85,13 +88,17 @@ def save(self, path: str):
:param str path: path for save model
"""
with open(path, "w") as f:
- json.dump({
- "training_data": self.training_data.tolist(),
- "Cx2_list": self.Cx2_list
- }, f, ensure_ascii=False)
+ json.dump(
+ {
+ "training_data": self.training_data.tolist(),
+ "Cx2_list": self.Cx2_list,
+ },
+ f,
+ ensure_ascii=False,
+ )
def load(self, path: str):
- with open(path, "r") as f:
+ with open(path) as f:
data = json.load(f)
self.Cx2_list = data["Cx2_list"]
self.training_data = np.array(data["training_data"])
diff --git a/pythainlp/cli/__init__.py b/pythainlp/cli/__init__.py
index 8d6649e6a..2bf309c52 100644
--- a/pythainlp/cli/__init__.py
+++ b/pythainlp/cli/__init__.py
@@ -1,19 +1,23 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""Command line helpers."""
+from __future__ import annotations
+
import io
import sys
from argparse import ArgumentError, ArgumentParser
-from pythainlp.cli import data, tokenize, soundex, tag, benchmark, misspell
+
+from pythainlp.cli import benchmark, data, misspell, soundex, tag, tokenize
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8")
# a command should start with a verb when possible
-COMMANDS = sorted(["data", "soundex", "tag", "tokenize", "benchmark", "misspell"])
+COMMANDS = sorted(
+ ["data", "soundex", "tag", "tokenize", "benchmark", "misspell"]
+)
CLI_NAME = "thainlp"
@@ -37,6 +41,7 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None:
parser.print_help()
raise ArgumentError(None, "No command provided.")
+
if __name__ == "__main__":
# Create a simple mapping from command name to the imported module
COMMAND_MAP = {
@@ -54,6 +59,9 @@ def exit_if_empty(command: str, parser: ArgumentParser) -> None:
COMMAND_MAP[command].run()
else:
if len(sys.argv) < 2:
- print(f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}", file=sys.stderr)
+ print(
+ f"Error: No command provided. Choose one of: {list(COMMAND_MAP.keys())}",
+ file=sys.stderr,
+ )
else:
- print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr)
\ No newline at end of file
+ print(f"Error: Unknown command '{sys.argv[1]}'", file=sys.stderr)
diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py
index 68d59c10d..e342ff9fd 100644
--- a/pythainlp/cli/benchmark.py
+++ b/pythainlp/cli/benchmark.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -8,13 +7,12 @@
import json
import os
-
from pythainlp import cli
from pythainlp.tools import safe_print
def _read_file(path):
- with open(path, "r", encoding="utf-8") as f:
+ with open(path, encoding="utf-8") as f:
lines = map(lambda r: r.strip(), f.readlines())
return list(lines)
@@ -77,9 +75,9 @@ def __init__(self, name, argv):
actual = _read_file(args.input_file)
expected = _read_file(args.test_file)
- assert len(actual) == len(
- expected
- ), "Input and test files do not have the same number of samples"
+ assert len(actual) == len(expected), (
+ "Input and test files do not have the same number of samples"
+ )
safe_print(
"Benchmarking %s against %s with %d samples in total"
@@ -88,6 +86,7 @@ def __init__(self, name, argv):
try:
import yaml
+
from pythainlp.benchmarks import word_tokenization
except ImportError:
raise ImportError(
diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py
index 0c5c704ed..37cba0b93 100644
--- a/pythainlp/cli/data.py
+++ b/pythainlp/cli/data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/cli/misspell.py b/pythainlp/cli/misspell.py
index 11077282a..66b4601e4 100644
--- a/pythainlp/cli/misspell.py
+++ b/pythainlp/cli/misspell.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -54,7 +53,7 @@ def __init__(self, argv):
if args.seed is not None:
random.seed(args.seed)
- with open(args.file, "r", encoding="utf-8") as f:
+ with open(args.file, encoding="utf-8") as f:
lines = f.readlines()
misspelled_lines = [
diff --git a/pythainlp/cli/soundex.py b/pythainlp/cli/soundex.py
index 60b0a1808..d6f97f0db 100644
--- a/pythainlp/cli/soundex.py
+++ b/pythainlp/cli/soundex.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py
index 6d4632cd0..bd8cabd47 100644
--- a/pythainlp/cli/tag.py
+++ b/pythainlp/cli/tag.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py
index 7a12914d2..156879b10 100644
--- a/pythainlp/cli/tokenize.py
+++ b/pythainlp/cli/tokenize.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py
index 24c11cdcd..366fff8e9 100644
--- a/pythainlp/coref/__init__.py
+++ b/pythainlp/coref/__init__.py
@@ -1,10 +1,10 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
PyThaiNLP Coreference Resolution
"""
+
__all__ = ["coreference_resolution"]
from pythainlp.coref.core import coreference_resolution
diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py
index 2b996d42a..32c537b3b 100644
--- a/pythainlp/coref/_fastcoref.py
+++ b/pythainlp/coref/_fastcoref.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List
+from __future__ import annotations
import spacy
@@ -30,7 +29,7 @@ def _to_json(self, _predict):
"clusters": _predict.get_clusters(as_strings=False),
}
- def predict(self, texts: List[str]) -> List[dict]:
+ def predict(self, texts: list[str]) -> list[dict]:
return [
self._to_json(pred) for pred in self.model.predict(texts=texts)
]
diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py
index 6dcb3ec75..4ffe50f35 100644
--- a/pythainlp/coref/core.py
+++ b/pythainlp/coref/core.py
@@ -1,14 +1,13 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List
+from __future__ import annotations
_MODEL = None
def coreference_resolution(
- texts: List[str], model_name: str = "han-coref-v1.0", device: str = "cpu"
+ texts: list[str], model_name: str = "han-coref-v1.0", device: str = "cpu"
):
"""
Coreference Resolution
diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py
index 45f0e3c62..4fd6d2f6e 100644
--- a/pythainlp/coref/han_coref.py
+++ b/pythainlp/coref/han_coref.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import spacy
from pythainlp.coref._fastcoref import FastCoref
diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py
index 424a25976..94b158c0e 100644
--- a/pythainlp/corpus/__init__.py
+++ b/pythainlp/corpus/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,6 +8,8 @@
Including download manager.
"""
+from __future__ import annotations
+
__all__ = [
"corpus_db_path",
"corpus_db_url",
@@ -99,9 +100,9 @@ def corpus_db_path() -> str:
get_corpus_db_detail,
get_corpus_default_db,
get_corpus_path,
+ get_hf_hub,
get_path_folder_corpus,
make_safe_directory_name,
- get_hf_hub,
path_pythainlp_corpus,
remove,
) # these imports must come before other pythainlp.corpus.* imports
diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py
index 6e23a7da4..285d02838 100644
--- a/pythainlp/corpus/common.py
+++ b/pythainlp/corpus/common.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -7,6 +6,8 @@
Common lists of words.
"""
+from __future__ import annotations
+
import ast
__all__ = [
@@ -26,45 +27,44 @@
"thai_wsd_dict",
]
-from typing import FrozenSet, List, Union
from pythainlp.corpus import get_corpus, get_corpus_as_is, get_corpus_path
from pythainlp.tools import warn_deprecation
-_THAI_COUNTRIES: FrozenSet[str] = frozenset()
+_THAI_COUNTRIES: frozenset[str] = frozenset()
_THAI_COUNTRIES_FILENAME = "countries_th.txt"
-_THAI_THAILAND_PROVINCES: FrozenSet[str] = frozenset()
-_THAI_THAILAND_PROVINCES_DETAILS: List[dict] = []
+_THAI_THAILAND_PROVINCES: frozenset[str] = frozenset()
+_THAI_THAILAND_PROVINCES_DETAILS: list[dict] = []
_THAI_THAILAND_PROVINCES_FILENAME = "thailand_provinces_th.csv"
-_THAI_SYLLABLES: FrozenSet[str] = frozenset()
+_THAI_SYLLABLES: frozenset[str] = frozenset()
_THAI_SYLLABLES_FILENAME = "syllables_th.txt"
-_THAI_WORDS: FrozenSet[str] = frozenset()
+_THAI_WORDS: frozenset[str] = frozenset()
_THAI_WORDS_FILENAME = "words_th.txt"
-_THAI_STOPWORDS: FrozenSet[str] = frozenset()
+_THAI_STOPWORDS: frozenset[str] = frozenset()
_THAI_STOPWORDS_FILENAME = "stopwords_th.txt"
-_THAI_NEGATIONS: FrozenSet[str] = frozenset()
+_THAI_NEGATIONS: frozenset[str] = frozenset()
_THAI_NEGATIONS_FILENAME = "negations_th.txt"
-_THAI_FAMLIY_NAMES: FrozenSet[str] = frozenset()
+_THAI_FAMLIY_NAMES: frozenset[str] = frozenset()
_THAI_FAMLIY_NAMES_FILENAME = "family_names_th.txt"
-_THAI_FEMALE_NAMES: FrozenSet[str] = frozenset()
+_THAI_FEMALE_NAMES: frozenset[str] = frozenset()
_THAI_FEMALE_NAMES_FILENAME = "person_names_female_th.txt"
-_THAI_MALE_NAMES: FrozenSet[str] = frozenset()
+_THAI_MALE_NAMES: frozenset[str] = frozenset()
_THAI_MALE_NAMES_FILENAME = "person_names_male_th.txt"
-_THAI_ORST_WORDS: FrozenSet[str] = frozenset()
+_THAI_ORST_WORDS: frozenset[str] = frozenset()
_THAI_DICT: dict[str, list] = {}
_THAI_WSD_DICT: dict[str, list] = {}
_THAI_SYNONYMS: dict[str, list] = {}
-def countries() -> FrozenSet[str]:
+def countries() -> frozenset[str]:
"""
Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย",
"แอลจีเรีย", and "ลาว".
@@ -81,7 +81,7 @@ def countries() -> FrozenSet[str]:
return _THAI_COUNTRIES
-def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]:
+def provinces(details: bool = False) -> frozenset[str] | list[dict]:
"""
Return a frozenset of Thailand province names in Thai such as "กระบี่",
"กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี".
@@ -124,7 +124,7 @@ def provinces(details: bool = False) -> Union[FrozenSet[str], List[dict]]:
return _THAI_THAILAND_PROVINCES
-def thai_syllables() -> FrozenSet[str]:
+def thai_syllables() -> frozenset[str]:
"""
Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ",
"โมน", "โม่ง", "กา", "ก่า", and, "ก้า".
@@ -142,7 +142,7 @@ def thai_syllables() -> FrozenSet[str]:
return _THAI_SYLLABLES
-def thai_words() -> FrozenSet[str]:
+def thai_words() -> frozenset[str]:
"""
Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ",
and "พิษภัย". \n(See: `dev/pythainlp/corpus/words_th.txt\
@@ -158,7 +158,7 @@ def thai_words() -> FrozenSet[str]:
return _THAI_WORDS
-def thai_orst_words() -> FrozenSet[str]:
+def thai_orst_words() -> frozenset[str]:
"""
Return a frozenset of Thai words from Royal Society of Thailand
\n(See: `dev/pythainlp/corpus/thai_orst_words.txt\
@@ -174,7 +174,7 @@ def thai_orst_words() -> FrozenSet[str]:
return _THAI_ORST_WORDS
-def thai_stopwords() -> FrozenSet[str]:
+def thai_stopwords() -> frozenset[str]:
"""
Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ",
"การ", and "ประการหนึ่ง". \n(See: `dev/pythainlp/corpus/stopwords_th.txt\
@@ -197,7 +197,7 @@ def thai_stopwords() -> FrozenSet[str]:
return _THAI_STOPWORDS
-def thai_negations() -> FrozenSet[str]:
+def thai_negations() -> frozenset[str]:
"""
Return a frozenset of Thai negation words including "ไม่" and "แต่".
\n(See: `dev/pythainlp/corpus/negations_th.txt\
@@ -213,7 +213,7 @@ def thai_negations() -> FrozenSet[str]:
return _THAI_NEGATIONS
-def thai_family_names() -> FrozenSet[str]:
+def thai_family_names() -> frozenset[str]:
"""
Return a frozenset of Thai family names
\n(See: `dev/pythainlp/corpus/family_names_th.txt\
@@ -229,7 +229,7 @@ def thai_family_names() -> FrozenSet[str]:
return _THAI_FAMLIY_NAMES
-def thai_female_names() -> FrozenSet[str]:
+def thai_female_names() -> frozenset[str]:
"""
Return a frozenset of Thai female names
\n(See: `dev/pythainlp/corpus/person_names_female_th.txt\
@@ -245,7 +245,7 @@ def thai_female_names() -> FrozenSet[str]:
return _THAI_FEMALE_NAMES
-def thai_male_names() -> FrozenSet[str]:
+def thai_male_names() -> frozenset[str]:
"""
Return a frozenset of Thai male names
\n(See: `dev/pythainlp/corpus/person_names_male_th.txt\
@@ -360,7 +360,7 @@ def thai_synonym() -> dict:
return thai_synonyms()
-def find_synonyms(word: str) -> List[str]:
+def find_synonyms(word: str) -> list[str]:
"""
Find synonyms
diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py
index d3372bfe2..ecaa33197 100644
--- a/pythainlp/corpus/core.py
+++ b/pythainlp/corpus/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,10 +5,11 @@
Corpus related functions.
"""
+from __future__ import annotations
+
import json
import os
import re
-from typing import Union
from pythainlp import __version__
from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path
@@ -45,7 +45,7 @@ def get_corpus_db_detail(name: str, version: str = "") -> dict:
:return: details about corpus
:rtype: dict
"""
- with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
+ with open(corpus_db_path(), encoding="utf-8-sig") as f:
local_db = json.load(f)
if not version:
@@ -133,7 +133,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset:
"""
path = path_pythainlp_corpus(filename)
lines = []
- with open(path, "r", encoding="utf-8-sig") as fh:
+ with open(path, encoding="utf-8-sig") as fh:
lines = fh.read().splitlines()
if not comments:
@@ -173,13 +173,13 @@ def get_corpus_as_is(filename: str) -> list:
"""
path = path_pythainlp_corpus(filename)
lines = []
- with open(path, "r", encoding="utf-8-sig") as fh:
+ with open(path, encoding="utf-8-sig") as fh:
lines = fh.read().splitlines()
return lines
-def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]:
+def get_corpus_default_db(name: str, version: str = "") -> str | None:
"""
Get model path from default_db.json
@@ -211,7 +211,7 @@ def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]:
def get_corpus_path(
name: str, version: str = "", force: bool = False
-) -> Union[str, None]:
+) -> str | None:
"""
Get corpus path.
@@ -252,9 +252,8 @@ def get_corpus_path(
print(get_corpus_path('wiki_lm_lstm'))
# output: /root/pythainlp-data/thwiki_model_lstm.pth
"""
- from typing import Dict
- CUSTOMIZE: Dict[str, str] = {
+ CUSTOMIZE: dict[str, str] = {
# "the corpus name":"path"
}
if name in list(CUSTOMIZE):
@@ -450,7 +449,7 @@ def download(
# check if corpus is available
if name in corpus_db:
- with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
+ with open(corpus_db_path(), encoding="utf-8-sig") as f:
local_db = json.load(f)
corpus = corpus_db[name]
@@ -525,9 +524,7 @@ def download(
# This awkward behavior is for backward-compatibility with
# database files generated previously using TinyDB
if local_db["_default"]:
- corpus_no = (
- max((int(no) for no in local_db["_default"])) + 1
- )
+ corpus_no = max(int(no) for no in local_db["_default"]) + 1
else:
corpus_no = 1
local_db["_default"][str(corpus_no)] = {
@@ -588,7 +585,7 @@ def remove(name: str) -> bool:
if _CHECK_MODE == "1":
print("PyThaiNLP is read-only mode. It can't download.")
return False
- with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
+ with open(corpus_db_path(), encoding="utf-8-sig") as f:
db = json.load(f)
data = [
corpus for corpus in db["_default"].values() if corpus["name"] == name
@@ -617,7 +614,7 @@ def get_path_folder_corpus(name, version, *path):
return os.path.join(get_corpus_path(name, version), *path)
-def make_safe_directory_name(name:str) -> str:
+def make_safe_directory_name(name: str) -> str:
"""
Make safe directory name
@@ -626,17 +623,40 @@ def make_safe_directory_name(name:str) -> str:
:rtype: str
"""
# Replace invalid characters with an underscore
- safe_name = re.sub(r'[<>:"/\\|?*]', '_', name)
+ safe_name = re.sub(r'[<>:"/\\|?*]', "_", name)
# Remove leading/trailing spaces or periods (especially important for Windows)
- safe_name = safe_name.strip(' .')
+ safe_name = safe_name.strip(" .")
# Prevent names that are reserved on Windows
- reserved_names = ['CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9']
+ reserved_names = [
+ "CON",
+ "PRN",
+ "AUX",
+ "NUL",
+ "COM1",
+ "COM2",
+ "COM3",
+ "COM4",
+ "COM5",
+ "COM6",
+ "COM7",
+ "COM8",
+ "COM9",
+ "LPT1",
+ "LPT2",
+ "LPT3",
+ "LPT4",
+ "LPT5",
+ "LPT6",
+ "LPT7",
+ "LPT8",
+ "LPT9",
+ ]
if safe_name.upper() in reserved_names:
- safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict
+ safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict
return safe_name
-def get_hf_hub(repo_id:str, filename: str=None) -> str:
+def get_hf_hub(repo_id: str, filename: str = None) -> str:
"""
HuggingFace Hub in :mod:`pythainlp` data directory.
@@ -653,19 +673,16 @@ def get_hf_hub(repo_id:str, filename: str=None) -> str:
Please installing the package via 'pip install huggingface-hub'.
""")
except Exception as e:
- raise Exception(f"An unexpected error occurred: {e}")
+ raise RuntimeError(f"An unexpected error occurred: {e}") from e
hf_root = get_full_data_path("hf_models")
name_dir = make_safe_directory_name(repo_id)
root_project = os.path.join(hf_root, name_dir)
- if filename!=None:
+ if filename is not None:
output_path = hf_hub_download(
- repo_id=repo_id,
- filename=filename,
- local_dir=root_project
+ repo_id=repo_id, filename=filename, local_dir=root_project
)
else:
output_path = snapshot_download(
- repo_id=repo_id,
- local_dir=root_project
+ repo_id=repo_id, local_dir=root_project
)
return output_path
diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py
index 3854c598f..ada98f81a 100644
--- a/pythainlp/corpus/icu.py
+++ b/pythainlp/corpus/icu.py
@@ -1,18 +1,18 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Provides an optional word list from International Components for Unicode (ICU) dictionary.
"""
-from typing import FrozenSet
+
+from __future__ import annotations
from pythainlp.corpus.common import get_corpus
_THAI_ICU_FILENAME = "icubrk_th.txt"
-def thai_icu_words() -> FrozenSet[str]:
+def thai_icu_words() -> frozenset[str]:
"""
Return a frozenset of words from the Thai dictionary for BreakIterator of the
International Components for Unicode (ICU).
diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py
index 6e5b5ba1d..0a0b540fc 100644
--- a/pythainlp/corpus/oscar.py
+++ b/pythainlp/corpus/oscar.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,17 +8,18 @@
https://web.facebook.com/groups/colab.thailand/permalink/1524070061101680/
"""
+from __future__ import annotations
+
__all__ = ["word_freqs", "unigram_word_freqs"]
from collections import defaultdict
-from typing import List, Tuple
from pythainlp.corpus import get_corpus_path
_OSCAR_FILENAME = "oscar_icu"
-def word_freqs() -> List[Tuple[str, int]]:
+def word_freqs() -> list[tuple[str, int]]:
"""
Get word frequency from OSCAR Corpus (words tokenized using ICU)
"""
@@ -29,7 +29,7 @@ def word_freqs() -> List[Tuple[str, int]]:
return freqs
path = str(path)
- with open(path, "r", encoding="utf-8-sig") as f:
+ with open(path, encoding="utf-8-sig") as f:
lines = list(f.readlines())
del lines[0]
for line in lines:
@@ -53,7 +53,7 @@ def unigram_word_freqs() -> dict[str, int]:
return freqs
path = str(path)
- with open(path, "r", encoding="utf-8-sig") as fh:
+ with open(path, encoding="utf-8-sig") as fh:
lines = list(fh.readlines())
del lines[0]
for i in lines:
diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py
index 8f8a61948..4a163a1f2 100644
--- a/pythainlp/corpus/th_en_translit.py
+++ b/pythainlp/corpus/th_en_translit.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -10,6 +9,8 @@
Zenodo. https://doi.org/10.5281/zenodo.6716672
"""
+from __future__ import annotations
+
__all__ = [
"get_transliteration_dict",
"TRANSLITERATE_EN",
@@ -43,7 +44,7 @@ def get_transliteration_dict() -> defaultdict:
lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []}
)
try:
- with open(path, "r", encoding="utf-8") as f:
+ with open(path, encoding="utf-8") as f:
# assume that the first row contains column names, so skip it.
for line in f.readlines()[1:]:
stripped = line.strip()
diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py
index 36a3e0311..7aef7b5c2 100644
--- a/pythainlp/corpus/tnc.py
+++ b/pythainlp/corpus/tnc.py
@@ -1,10 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project.
# SPDX-License-Identifier: Apache-2.0
"""
Thai National Corpus word frequency
"""
+from __future__ import annotations
+
__all__ = [
"bigram_word_freqs",
"trigram_word_freqs",
@@ -13,7 +14,6 @@
]
from collections import defaultdict
-from typing import List, Tuple
from pythainlp.corpus import get_corpus, get_corpus_path
@@ -22,7 +22,7 @@
_TRIGRAM_CORPUS_NAME = "tnc_trigram_word_freqs"
-def word_freqs() -> List[Tuple[str, int]]:
+def word_freqs() -> list[tuple[str, int]]:
"""
Get word frequency from Thai National Corpus (TNC)
\n(See: `dev/pythainlp/corpus/tnc_freq.txt\
@@ -54,7 +54,7 @@ def unigram_word_freqs() -> dict[str, int]:
return freqs
-def bigram_word_freqs() -> dict[Tuple[str, str], int]:
+def bigram_word_freqs() -> dict[tuple[str, str], int]:
"""
Get bigram word frequency from Thai National Corpus (TNC)
"""
@@ -64,7 +64,7 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]:
return freqs
path = str(path)
- with open(path, "r", encoding="utf-8-sig") as fh:
+ with open(path, encoding="utf-8-sig") as fh:
for i in fh.readlines():
temp = i.strip().split(" ")
freqs[(temp[0], temp[1])] = int(temp[-1])
@@ -72,7 +72,7 @@ def bigram_word_freqs() -> dict[Tuple[str, str], int]:
return freqs
-def trigram_word_freqs() -> dict[Tuple[str, str, str], int]:
+def trigram_word_freqs() -> dict[tuple[str, str, str], int]:
"""
Get trigram word frequency from Thai National Corpus (TNC)
"""
@@ -82,7 +82,7 @@ def trigram_word_freqs() -> dict[Tuple[str, str, str], int]:
return freqs
path = str(path)
- with open(path, "r", encoding="utf-8-sig") as fh:
+ with open(path, encoding="utf-8-sig") as fh:
for i in fh.readlines():
temp = i.strip().split(" ")
freqs[(temp[0], temp[1], temp[2])] = int(temp[-1])
diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py
index c6e470f74..a32051259 100644
--- a/pythainlp/corpus/ttc.py
+++ b/pythainlp/corpus/ttc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,17 +8,18 @@
https://www.facebook.com/photo.php?fbid=363640477387469&set=gm.434330506948445&type=3&permPage=1
"""
+from __future__ import annotations
+
__all__ = ["word_freqs", "unigram_word_freqs"]
from collections import defaultdict
-from typing import List, Tuple
from pythainlp.corpus import get_corpus
_UNIGRAM_FILENAME = "ttc_freq.txt"
-def word_freqs() -> List[Tuple[str, int]]:
+def word_freqs() -> list[tuple[str, int]]:
"""
Get word frequency from Thai Textbook Corpus (TTC)
\n(See: `dev/pythainlp/corpus/ttc_freq.txt\
diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py
index b33b3ef11..d4e86c156 100644
--- a/pythainlp/corpus/util.py
+++ b/pythainlp/corpus/util.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -13,15 +12,17 @@
`_
"""
+from __future__ import annotations
+
from collections import Counter
-from typing import Callable, Iterable, Iterator, List, Set, Tuple
+from collections.abc import Callable, Iterable, Iterator
from pythainlp.corpus import thai_words
from pythainlp.tokenize import newmm
from pythainlp.util import Trie
-def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]:
+def index_pairs(words: list[str]) -> Iterator[tuple[int, int]]:
"""
Return beginning and ending indexes of word pairs
"""
@@ -32,9 +33,9 @@ def index_pairs(words: List[str]) -> Iterator[Tuple[int, int]]:
def find_badwords(
- tokenize: Callable[[str], List[str]],
+ tokenize: Callable[[str], list[str]],
training_data: Iterable[Iterable[str]],
-) -> Set[str]:
+) -> set[str]:
"""
Find words that do not work well with the `tokenize` function
for the provided `training_data`.
@@ -68,10 +69,10 @@ def find_badwords(
def revise_wordset(
- tokenize: Callable[[str], List[str]],
+ tokenize: Callable[[str], list[str]],
orig_words: Iterable[str],
training_data: Iterable[Iterable[str]],
-) -> Set[str]:
+) -> set[str]:
"""
Revise a set of words that could improve tokenization performance of
a dictionary-based `tokenize` function.
@@ -119,7 +120,7 @@ def revise_wordset(
def revise_newmm_default_wordset(
training_data: Iterable[Iterable[str]],
-) -> Set[str]:
+) -> set[str]:
"""
Revise a set of word that could improve tokenization performance of
`pythainlp.tokenize.newmm`, a dictionary-based tokenizer and a default
diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py
index de65e1e1f..8f21a5dba 100644
--- a/pythainlp/corpus/volubilis.py
+++ b/pythainlp/corpus/volubilis.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Provides an optional word list from the Volubilis dictionary.
"""
-from typing import FrozenSet
+
+from __future__ import annotations
from pythainlp.corpus.common import get_corpus
@@ -13,7 +13,7 @@
_VOLUBILIS_FILENAME = "volubilis_words_th.txt"
-def thai_volubilis_words() -> FrozenSet[str]:
+def thai_volubilis_words() -> frozenset[str]:
"""
Return a frozenset of Thai words from the Volubilis dictionary
diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py
index 2c0ce7c2e..a036f4568 100644
--- a/pythainlp/corpus/wikipedia.py
+++ b/pythainlp/corpus/wikipedia.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Provides an optional word list from Thai Wikipedia titles.
"""
-from typing import FrozenSet
+
+from __future__ import annotations
from pythainlp.corpus.common import get_corpus
@@ -13,7 +13,7 @@
_WIKIPEDIA_TITLES_FILENAME = "wikipedia_titles_th.txt"
-def thai_wikipedia_titles() -> FrozenSet[str]:
+def thai_wikipedia_titles() -> frozenset[str]:
"""
Return a frozenset of words from Thai Wikipedia titles corpus.
They are mostly nouns and noun phrases,
@@ -31,6 +31,8 @@ def thai_wikipedia_titles() -> FrozenSet[str]:
"""
global _WIKIPEDIA_TITLES
if not _WIKIPEDIA_TITLES:
- _WIKIPEDIA_TITLES = get_corpus(_WIKIPEDIA_TITLES_FILENAME, comments=False)
+ _WIKIPEDIA_TITLES = get_corpus(
+ _WIKIPEDIA_TITLES_FILENAME, comments=False
+ )
return _WIKIPEDIA_TITLES
diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py
index 658314b4a..6ea2a0097 100644
--- a/pythainlp/corpus/wordnet.py
+++ b/pythainlp/corpus/wordnet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,6 +10,9 @@
For more on usage, see NLTK Howto:
https://www.nltk.org/howto/wordnet.html
"""
+
+from __future__ import annotations
+
import nltk
try:
@@ -88,7 +90,7 @@ def synset(name_synsets):
>>> from pythainlp.corpus.wordnet import synset
>>>
- >>> difficult = synset('difficult.a.01')
+ >>> difficult = synset("difficult.a.01")
>>> difficult
Synset('difficult.a.01')
>>>
@@ -252,13 +254,13 @@ def lemma(name_synsets):
>>> from pythainlp.corpus.wordnet import lemma
>>>
- >>> lemma('practice.v.01.exercise')
+ >>> lemma("practice.v.01.exercise")
Lemma('practice.v.01.exercise')
>>>
- >>> lemma('drill.v.03.exercise')
+ >>> lemma("drill.v.03.exercise")
Lemma('drill.v.03.exercise')
>>>
- >>> lemma('exercise.n.01.exercise')
+ >>> lemma("exercise.n.01.exercise")
Lemma('exercise.n.01.exercise')
"""
return wordnet.lemma(name_synsets)
@@ -282,7 +284,7 @@ def lemma_from_key(key):
>>> from pythainlp.corpus.wordnet import lemma, lemma_from_key
>>>
- >>> practice = lemma('practice.v.01.exercise')
+ >>> practice = lemma("practice.v.01.exercise")
>>> practice.key()
exercise%2:41:00::
>>> lemma_from_key(practice.key())
@@ -317,9 +319,9 @@ def path_similarity(synsets1, synsets2):
>>> from pythainlp.corpus.wordnet import path_similarity, synset
>>>
- >>> entity = synset('entity.n.01')
- >>> obj = synset('object.n.01')
- >>> cat = synset('cat.n.01')
+ >>> entity = synset("entity.n.01")
+ >>> obj = synset("object.n.01")
+ >>> cat = synset("cat.n.01")
>>>
>>> path_similarity(entity, obj)
0.3333333333333333
@@ -355,9 +357,9 @@ def lch_similarity(synsets1, synsets2):
>>> from pythainlp.corpus.wordnet import lch_similarity, synset
>>>
- >>> entity = synset('entity.n.01')
- >>> obj = synset('object.n.01')
- >>> cat = synset('cat.n.01')
+ >>> entity = synset("entity.n.01")
+ >>> obj = synset("object.n.01")
+ >>> cat = synset("cat.n.01")
>>>
>>> lch_similarity(entity, obj)
2.538973871058276
@@ -387,9 +389,9 @@ def wup_similarity(synsets1, synsets2):
>>> from pythainlp.corpus.wordnet import wup_similarity, synset
>>>
- >>> entity = synset('entity.n.01')
- >>> obj = synset('object.n.01')
- >>> cat = synset('cat.n.01')
+ >>> entity = synset("entity.n.01")
+ >>> obj = synset("object.n.01")
+ >>> cat = synset("cat.n.01")
>>>
>>> wup_similarity(entity, obj)
0.5
diff --git a/pythainlp/el/__init__.py b/pythainlp/el/__init__.py
index d076062b1..f6e8a6a28 100644
--- a/pythainlp/el/__init__.py
+++ b/pythainlp/el/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/el/_multiel.py b/pythainlp/el/_multiel.py
index babe2b292..2c73b64b0 100644
--- a/pythainlp/el/_multiel.py
+++ b/pythainlp/el/_multiel.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py
index dbb30d97e..b991d78e2 100644
--- a/pythainlp/el/core.py
+++ b/pythainlp/el/core.py
@@ -1,12 +1,16 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Union
+from __future__ import annotations
class EntityLinker:
- def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata"):
+ def __init__(
+ self,
+ model_name: str = "bela",
+ device: str = "cuda",
+ tag: str = "wikidata",
+ ):
"""
EntityLinker
@@ -21,24 +25,30 @@ def __init__(self, model_name:str="bela", device:str="cuda", tag:str="wikidata")
self.device = device
self.tag = tag
if self.model_name not in ["bela"]:
- raise NotImplementedError(f"EntityLinker doesn't support {model_name} model.")
+ raise NotImplementedError(
+ f"EntityLinker doesn't support {model_name} model."
+ )
if self.tag not in ["wikidata"]:
- raise NotImplementedError(f"EntityLinker doesn't support {tag} tag.")
+ raise NotImplementedError(
+ f"EntityLinker doesn't support {tag} tag."
+ )
from pythainlp.el._multiel import MultiEL
+
self.model = MultiEL(model_name=self.model_name, device=self.device)
- def get_el(self, list_text:Union[List[str], str])->Union[List[dict], str]:
+
+ def get_el(self, list_text: list[str] | str) -> list[dict] | str:
"""
Get Entity Linking from Thai Text
-
+
:param str Union[List[str], str]: list of Thai text or text
:return: list of entity linking
:rtype: Union[List[dict], str]
-
+
:Example:
::
from pythainlp.el import EntityLinker
-
+
el = EntityLinker(device="cuda")
print(el.get_el("จ๊อบเคยเป็นซีอีโอบริษัทแอปเปิล"))
# output: [{'offsets': [11, 23],
diff --git a/pythainlp/generate/__init__.py b/pythainlp/generate/__init__.py
index ab68623f9..b3cb92c4a 100644
--- a/pythainlp/generate/__init__.py
+++ b/pythainlp/generate/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py
index 14102f14c..5f3bb60fc 100644
--- a/pythainlp/generate/core.py
+++ b/pythainlp/generate/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,8 +8,9 @@
https://towardsdatascience.com/understanding-word-n-grams-and-n-gram-probability-in-natural-language-processing-9d9eef0fa058
"""
+from __future__ import annotations
+
import random
-from typing import List, Union
from pythainlp.corpus.oscar import (
unigram_word_freqs as oscar_word_freqs_unigram,
@@ -52,7 +52,7 @@ def gen_sentence(
prob: float = 0.001,
output_str: bool = True,
duplicate: bool = False,
- ) -> Union[List[str], str]:
+ ) -> list[str] | str:
"""
:param str start_seq: word to begin sentence with
:param int N: number of words
@@ -148,7 +148,7 @@ def gen_sentence(
prob: float = 0.001,
output_str: bool = True,
duplicate: bool = False,
- ) -> Union[List[str], str]:
+ ) -> list[str] | str:
"""
:param str start_seq: word to begin sentence with
:param int N: number of words
@@ -240,7 +240,7 @@ def gen_sentence(
prob: float = 0.001,
output_str: bool = True,
duplicate: bool = False,
- ) -> Union[List[str], str]:
+ ) -> list[str] | str:
"""
:param str start_seq: word to begin sentence with
:param int N: number of words
diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py
index 445a063af..2498937f2 100644
--- a/pythainlp/generate/thai2fit.py
+++ b/pythainlp/generate/thai2fit.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,11 +8,12 @@
https://github.com/PyThaiNLP/tutorials/blob/master/source/notebooks/text_generation.ipynb
"""
+from __future__ import annotations
+
__all__ = ["gen_sentence"]
import pickle
import random
-from typing import List, Union
# fastai
import fastai
@@ -88,7 +88,7 @@ def gen_sentence(
N: int = 4,
prob: float = 0.001,
output_str: bool = True,
-) -> Union[List[str], str]:
+) -> list[str] | str:
"""
Text generator using Thai2fit
diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py
index cdb63efad..1e93b6413 100644
--- a/pythainlp/generate/wangchanglm.py
+++ b/pythainlp/generate/wangchanglm.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import re
import torch
@@ -9,34 +10,32 @@
class WangChanGLM:
def __init__(self):
- self.exclude_pattern = re.compile(r'[^ก-๙]+')
+ self.exclude_pattern = re.compile(r"[^ก-๙]+")
self.stop_token = "\n"
self.PROMPT_DICT = {
"prompt_input": (
": {input}\n: {instruction}\n: "
),
- "prompt_no_input": (
- ": {instruction}\n: "
- ),
- "prompt_chatbot": (
- ": {human}\n: {bot}"
- ),
+ "prompt_no_input": (": {instruction}\n: "),
+ "prompt_chatbot": (": {human}\n: {bot}"),
}
- def is_exclude(self, text:str)->bool:
+
+ def is_exclude(self, text: str) -> bool:
return bool(self.exclude_pattern.search(text))
+
def load_model(
self,
- model_path:str="pythainlp/wangchanglm-7.5B-sft-en-sharded",
- return_dict:bool=True,
- load_in_8bit:bool=False,
- device:str="cuda",
+ model_path: str = "pythainlp/wangchanglm-7.5B-sft-en-sharded",
+ return_dict: bool = True,
+ load_in_8bit: bool = False,
+ device: str = "cuda",
torch_dtype=torch.float16,
- offload_folder:str="./",
- low_cpu_mem_usage:bool=True
+ offload_folder: str = "./",
+ low_cpu_mem_usage: bool = True,
):
"""
Load model
-
+
:param str model_path: model path
:param bool return_dict: return dict
:param bool load_in_8bit: load model in 8bit
@@ -47,6 +46,7 @@ def load_model(
"""
import pandas as pd
from transformers import AutoModelForCausalLM, AutoTokenizer
+
self.device = device
self.torch_dtype = torch_dtype
self.model_path = model_path
@@ -57,27 +57,30 @@ def load_model(
device_map=device,
torch_dtype=torch_dtype,
offload_folder=offload_folder,
- low_cpu_mem_usage=low_cpu_mem_usage
+ low_cpu_mem_usage=low_cpu_mem_usage,
)
self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
- self.df = pd.DataFrame(self.tokenizer.vocab.items(), columns=['text', 'idx'])
- self.df['is_exclude'] = self.df.text.map(self.is_exclude)
+ self.df = pd.DataFrame(
+ self.tokenizer.vocab.items(), columns=["text", "idx"]
+ )
+ self.df["is_exclude"] = self.df.text.map(self.is_exclude)
self.exclude_ids = self.df[self.df.is_exclude is True].idx.tolist()
+
def gen_instruct(
self,
- text:str,
- max_new_tokens:int=512,
- top_p:float=0.95,
- temperature:float=0.9,
- top_k:int=50,
- no_repeat_ngram_size:int=2,
- typical_p:float=1.,
- thai_only:bool=True,
- skip_special_tokens:bool=True
+ text: str,
+ max_new_tokens: int = 512,
+ top_p: float = 0.95,
+ temperature: float = 0.9,
+ top_k: int = 50,
+ no_repeat_ngram_size: int = 2,
+ typical_p: float = 1.0,
+ thai_only: bool = True,
+ skip_special_tokens: bool = True,
):
"""
Generate Instruct
-
+
:param str text: text
:param int max_new_tokens: maximum number of new tokens
:param float top_p: top p
@@ -95,43 +98,47 @@ def gen_instruct(
if thai_only:
output_tokens = self.model.generate(
input_ids=batch["input_ids"],
- max_new_tokens=max_new_tokens, # 512
- begin_suppress_tokens = self.exclude_ids,
+ max_new_tokens=max_new_tokens, # 512
+ begin_suppress_tokens=self.exclude_ids,
no_repeat_ngram_size=no_repeat_ngram_size,
- #oasst k50
+ # oasst k50
top_k=top_k,
- top_p=top_p, # 0.95
+ top_p=top_p, # 0.95
typical_p=typical_p,
- temperature=temperature, # 0.9
+ temperature=temperature, # 0.9
)
else:
output_tokens = self.model.generate(
input_ids=batch["input_ids"],
- max_new_tokens=max_new_tokens, # 512
+ max_new_tokens=max_new_tokens, # 512
no_repeat_ngram_size=no_repeat_ngram_size,
- #oasst k50
+ # oasst k50
top_k=top_k,
- top_p=top_p, # 0.95
+ top_p=top_p, # 0.95
typical_p=typical_p,
- temperature=temperature, # 0.9
+ temperature=temperature, # 0.9
)
- return self.tokenizer.decode(output_tokens[0][len(batch["input_ids"][0]):], skip_special_tokens=skip_special_tokens)
+ return self.tokenizer.decode(
+ output_tokens[0][len(batch["input_ids"][0]) :],
+ skip_special_tokens=skip_special_tokens,
+ )
+
def instruct_generate(
self,
instruct: str,
context: str = None,
max_new_tokens=512,
- temperature: float =0.9,
+ temperature: float = 0.9,
top_p: float = 0.95,
- top_k:int=50,
- no_repeat_ngram_size:int=2,
- typical_p:float=1,
- thai_only:bool=True,
- skip_special_tokens:bool=True
+ top_k: int = 50,
+ no_repeat_ngram_size: int = 2,
+ typical_p: float = 1,
+ thai_only: bool = True,
+ skip_special_tokens: bool = True,
):
"""
Generate Instruct
-
+
:param str instruct: Instruct
:param str context: context
:param int max_new_tokens: maximum number of new tokens
@@ -153,7 +160,7 @@ def instruct_generate(
model = WangChanGLM()
- model.load_model(device="cpu",torch_dtype=torch.bfloat16)
+ model.load_model(device="cpu", torch_dtype=torch.bfloat16)
print(model.instruct_generate(instruct="ขอวิธีลดน้ำหนัก"))
# output: ลดน้ําหนักให้ได้ผล ต้องทําอย่างค่อยเป็นค่อยไป
@@ -166,12 +173,12 @@ def instruct_generate(
"""
if context in (None, ""):
- prompt = self.PROMPT_DICT['prompt_no_input'].format_map(
- {'instruction': instruct, 'input': ''}
+ prompt = self.PROMPT_DICT["prompt_no_input"].format_map(
+ {"instruction": instruct, "input": ""}
)
else:
- prompt = self.PROMPT_DICT['prompt_input'].format_map(
- {'instruction': instruct, 'input': context}
+ prompt = self.PROMPT_DICT["prompt_input"].format_map(
+ {"instruction": instruct, "input": context}
)
result = self.gen_instruct(
prompt,
@@ -182,6 +189,6 @@ def instruct_generate(
no_repeat_ngram_size=no_repeat_ngram_size,
typical_p=typical_p,
thai_only=thai_only,
- skip_special_tokens=skip_special_tokens
+ skip_special_tokens=skip_special_tokens,
)
return result
diff --git a/pythainlp/khavee/__init__.py b/pythainlp/khavee/__init__.py
index b8959d272..e4882747b 100644
--- a/pythainlp/khavee/__init__.py
+++ b/pythainlp/khavee/__init__.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
-# SPDX-FileType: SOURCE
-# SPDX-License-Identifier: Apache-2.0
-
-__all__ = ["KhaveeVerifier"]
-
-from pythainlp.khavee.core import KhaveeVerifier
+# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
+# SPDX-FileType: SOURCE
+# SPDX-License-Identifier: Apache-2.0
+
+__all__ = ["KhaveeVerifier"]
+
+from pythainlp.khavee.core import KhaveeVerifier
diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py
index 81e05b662..c22a8212f 100644
--- a/pythainlp/khavee/core.py
+++ b/pythainlp/khavee/core.py
@@ -1,691 +1,687 @@
-# -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
-# SPDX-FileType: SOURCE
-# SPDX-License-Identifier: Apache-2.0
-# ruff: noqa: C901
-
-from typing import List, Union
-
-from pythainlp import thai_consonants
-from pythainlp.tokenize import subword_tokenize
-from pythainlp.util import remove_tonemark, sound_syllable
-
-
-class KhaveeVerifier:
- def __init__(self):
- """
- KhaveeVerifier: Thai Poetry verifier
- """
-
- def _has_true_final_yl(self, word: str) -> bool:
- """
- Check if ย or ล is a true final consonant
- (not just part of the vowel sound with ไ/ใ)
-
- :param str word: Thai word
- :return: True if ย or ล is a true final consonant
- :rtype: bool
- """
- if len(word) < 2:
- return False
- # Count consonants in the word
- consonant_count = sum(1 for c in word if c in thai_consonants)
- # If there are 2+ consonants and word ends with ย or ล, it's a true final
- return consonant_count >= 2 and word[-1] in ["ย", "ล"]
-
- def check_sara(self, word: str) -> str:
- """
- Check the vowels in the Thai word.
-
- :param str word: Thai word
- :return: vowel name of the word
- :rtype: str
-
- :Example:
- ::
-
- from pythainlp.khavee import KhaveeVerifier
-
- kv = KhaveeVerifier()
-
- print(kv.check_sara("เริง"))
- # output: 'เออ'
- """
- sara = []
- countoa = 0
-
- # In case of การันย์
- if "์" in word[-1]:
- word = word[:-2]
-
- # In case of สระเดี่ยว
- for i in word:
- if i in ("ะ", "ั"):
- sara.append("อะ")
- elif i == "ิ":
- sara.append("อิ")
- elif i == "ุ":
- sara.append("อุ")
- elif i == "ึ":
- sara.append("อึ")
- elif i == "ี":
- sara.append("อี")
- elif i == "ู":
- sara.append("อู")
- elif i == "ื":
- sara.append("อือ")
- elif i == "เ":
- sara.append("เอ")
- elif i == "แ":
- sara.append("แอ")
- elif i == "า":
- sara.append("อา")
- elif i == "โ":
- sara.append("โอ")
- elif i == "ำ":
- sara.append("อำ")
- elif i == "อ":
- countoa += 1
- sara.append("ออ")
- elif i == "ั" and "ว" in word:
- sara.append("อัว")
- elif i in ("ไ", "ใ"):
- sara.append("ไอ")
- elif i == "็":
- sara.append("ออ")
- elif "รร" in word:
- if self.check_marttra(word) == "กม":
- sara.append("อำ")
- else:
- sara.append("อะ")
-
- # In case of ออ
- if countoa == 1 and "อ" in word[-1] and "เ" not in word:
- sara.remove("ออ")
-
- # In case of เอ เอ
- countA = 0
- for i in sara:
- if i == "เอ":
- countA = countA + 1
- if countA > 1:
- sara.remove("เอ")
- sara.remove("เอ")
- sara.append("แ")
-
- # In case of สระประสม
- if "เอ" in sara and "อะ" in sara:
- sara.remove("เอ")
- sara.remove("อะ")
- sara.append("เอะ")
- elif "แอ" in sara and "อะ" in sara:
- sara.remove("แอ")
- sara.remove("อะ")
- sara.append("แอะ")
-
- if "เอะ" in sara and "ออ" in sara:
- sara.remove("เอะ")
- sara.remove("ออ")
- sara.append("เออะ")
- elif "เอ" in sara and "อิ" in sara:
- sara.remove("เอ")
- sara.remove("อิ")
- sara.append("เออ")
- elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]:
- sara.remove("เอ")
- sara.remove("ออ")
- sara.append("เออ")
- elif "โอ" in sara and "อะ" in sara:
- sara.remove("โอ")
- sara.remove("อะ")
- sara.append("โอะ")
- elif "เอ" in sara and "อี" in sara:
- sara.remove("เอ")
- sara.remove("อี")
- sara.append("เอีย")
- elif "เอ" in sara and "อือ" in sara:
- sara.remove("เอ")
- sara.remove("อือ")
- sara.append("อัว")
- elif "เอ" in sara and "อา" in sara:
- sara.remove("เอ")
- sara.remove("อา")
- sara.append("เอา")
- elif "เ" in word and "า" in word and "ะ" in word:
- sara = []
- sara.append("เอาะ")
-
- if "อือ" in sara and "เออ" in sara:
- sara.remove("เออ")
- sara.remove("อือ")
- sara.append("เอือ")
- elif "ออ" in sara and len(sara) > 1:
- sara.remove("ออ")
- elif "ว" in word and len(sara) == 0:
- sara.append("อัว")
-
- if "ั" in word and self.check_marttra(word) == "กา":
- sara = []
- sara.append("ไอ")
-
- # In case of อ
- if word == "เออะ":
- sara = []
- sara.append("เออะ")
- elif word == "เออ":
- sara = []
- sara.append("เออ")
- elif word == "เอ":
- sara = []
- sara.append("เอ")
- elif word == "เอะ":
- sara = []
- sara.append("เอะ")
- elif word == "เอา":
- sara = []
- sara.append("เอา")
- elif word == "เอาะ":
- sara = []
- sara.append("เอาะ")
-
- if "ฤา" in word or "ฦา" in word:
- sara = []
- sara.append("อือ")
- elif "ฤ" in word or "ฦ" in word:
- sara = []
- sara.append("อึ")
-
- # In case of กน
- if not sara and len(word) == 2:
- if word[-1] != "ร":
- sara.append("โอะ")
- else:
- sara.append("ออ")
- elif not sara and len(word) == 3:
- sara.append("ออ")
-
- # In case of บ่
- if word == "บ่":
- sara = []
- sara.append("ออ")
-
- if "ํ" in word:
- sara = []
- sara.append("อำ")
-
- if "เ" in word and "ื" in word and "อ" in word:
- sara = []
- sara.append("เอือ")
-
- if not sara:
- return "Can't find Sara in this word"
-
- return sara[0]
-
- def check_marttra(self, word: str) -> str:
- """
- Check the Thai spelling Section in the Thai word.
-
- :param str word: Thai word
- :return: name of spelling Section of the word.
- :rtype: str
-
- :Example:
- ::
-
- from pythainlp.khavee import KhaveeVerifier
-
- kv = KhaveeVerifier()
-
- print(kv.check_marttra("สาว"))
- # output: 'เกอว'
- """
- # Handle consonant clusters ending with ร
- # ตร, ทร → remove ร (treat as final ต/ท sound)
- # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound)
- # But single syllable words like "กร" should keep ร
- if len(word) >= 3 and word[-1] == "ร":
- if word[-2] in ["ต", "ท"]:
- word = word[:-1]
- elif word[-2] in ["ก", "ข", "ค", "ฆ"]:
- word = word[:-1]
-
- word = self.handle_karun_sound_silence(word)
- word = remove_tonemark(word)
-
- # Check for ำ at the end (represents "am" sound, ends with m)
- if word[-1] == "ำ":
- return "กม"
-
- # Check for vowels and special patterns that indicate open syllables (กา)
- # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel
- if "ไ" in word or "ใ" in word:
- if word[-1] not in ["ย", "ล"]:
- return "กา"
- elif not self._has_true_final_yl(word):
- # ย/ล is part of the vowel sound, not a true final
- return "กา"
- # else: ย/ล is a true final, continue to consonant classification below
-
- if (
- ("ํ" in word and "า" in word)
- ):
- return "กา"
- elif (
- word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"]
- or ("ี" in word and "ย" in word[-1])
- or ("ื" in word and "อ" in word[-1])
- ):
- return "กา"
- elif word[-1] in ["ง"]:
- return "กง"
- elif word[-1] in ["ม"]:
- return "กม"
- elif word[-1] in ["ย"]:
- return "เกย"
- elif word[-1] in ["ล"]:
- return "เกย"
- elif word[-1] in ["ว"]:
- return "เกอว"
- elif word[-1] in ["ก", "ข", "ค", "ฆ"]:
- return "กก"
- elif word[-1] in [
- "จ",
- "ช",
- "ซ",
- "ฎ",
- "ฏ",
- "ฐ",
- "ฑ",
- "ฒ",
- "ด",
- "ต",
- "ถ",
- "ท",
- "ธ",
- "ศ",
- "ษ",
- "ส",
- ]:
- return "กด"
- elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]:
- return "กน"
- elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]:
- return "กบ"
- else:
- if "็" in word:
- return "กา"
- else:
- return "Cant find Marttra in this word"
-
- def is_sumpus(self, word1: str, word2: str) -> bool:
- """
- Check the rhyme between two words.
-
- :param str word1: Thai word
- :param str word2: Thai word
- :return: boolean
- :rtype: bool
-
- :Example:
- ::
-
- from pythainlp.khavee import KhaveeVerifier
-
- kv = KhaveeVerifier()
-
- print(kv.is_sumpus("สรร", "อัน"))
- # output: True
-
- print(kv.is_sumpus("สรร", "แมว"))
- # output: False
- """
- marttra1 = self.check_marttra(word1)
- marttra2 = self.check_marttra(word2)
- sara1 = self.check_sara(word1)
- sara2 = self.check_sara(word2)
- if sara1 == "อะ" and marttra1 == "เกย":
- sara1 = "ไอ"
- marttra1 = "กา"
- elif sara2 == "อะ" and marttra2 == "เกย":
- sara2 = "ไอ"
- marttra2 = "กา"
- if sara1 == "อำ" and marttra1 == "กม":
- sara1 = "อำ"
- marttra1 = "กา"
- elif sara2 == "อำ" and marttra2 == "กม":
- sara2 = "อำ"
- marttra2 = "กา"
- return bool(marttra1 == marttra2 and sara1 == sara2)
-
- def check_karu_lahu(self, text):
- if (
- self.check_marttra(text) != "กา"
- or (
- self.check_marttra(text) == "กา"
- and self.check_sara(text)
- in [
- "อา",
- "อี",
- "อือ",
- "อู",
- "เอ",
- "แอ",
- "โอ",
- "ออ",
- "เออ",
- "เอีย",
- "เอือ",
- "อัว",
- ]
- )
- or self.check_sara(text) in ["อำ", "ไอ", "เอา"]
- ) and text not in ["บ่", "ณ", "ธ", "ก็"]:
- return "karu"
- else:
- return "lahu"
-
- def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]:
- """
- Check the suitability of the poem according to Thai principles.
-
- :param str text: Thai poem
- :param int k_type: type of Thai poem
- :return: the check results of the suitability of the poem according to Thai principles.
- :rtype: Union[List[str], str]
-
- :Example:
- ::
-
- from pythainlp.khavee import KhaveeVerifier
-
- kv = KhaveeVerifier()
-
- print(kv.check_klon(
- 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \
- มีคนจับจอง เขาชื่อน้องเธียร',
- k_type=4
- ))
- # output: The poem is correct according to the principle.
-
- print(kv.check_klon(
- 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \
- เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร',
- k_type=4
- ))
- # output: [
- "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2",
- "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2"
- ]
- """
- if k_type == 8:
- try:
- error = []
- list_sumpus_sent1 = []
- list_sumpus_sent2h = []
- list_sumpus_sent2l = []
- list_sumpus_sent3 = []
- list_sumpus_sent4 = []
- for i, sent in enumerate(text.split()):
- sub_sent = subword_tokenize(sent, engine="dict")
- if len(sub_sent) > 10:
- error.append(
- "In sentence "
- + str(i + 2)
- + ", there are more than 10 words. "
- + str(sub_sent)
- )
- if (i + 1) % 4 == 1:
- list_sumpus_sent1.append(sub_sent[-1])
- elif (i + 1) % 4 == 2:
- list_sumpus_sent2h.append(
- [
- sub_sent[1],
- sub_sent[2],
- sub_sent[3],
- sub_sent[4],
- ]
- )
- list_sumpus_sent2l.append(sub_sent[-1])
- elif (i + 1) % 4 == 3:
- list_sumpus_sent3.append(sub_sent[-1])
- elif (i + 1) % 4 == 0:
- list_sumpus_sent4.append(sub_sent[-1])
- if (
- len(list_sumpus_sent1) != len(list_sumpus_sent2h)
- or len(list_sumpus_sent2h) != len(list_sumpus_sent2l)
- or len(list_sumpus_sent2l) != len(list_sumpus_sent3)
- or len(list_sumpus_sent3) != len(list_sumpus_sent4)
- or len(list_sumpus_sent4) != len(list_sumpus_sent1)
- ):
- return "The poem does not have 4 complete sentences."
- else:
- for i in range(len(list_sumpus_sent1)):
- countwrong = 0
- for j in list_sumpus_sent2h[i]:
- if (
- self.is_sumpus(list_sumpus_sent1[i], j)
- is False
- ):
- countwrong += 1
- if countwrong > 3:
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent1[i],
- list_sumpus_sent2h[i],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if (
- self.is_sumpus(
- list_sumpus_sent2l[i], list_sumpus_sent3[i]
- )
- is False
- ):
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent2l[i],
- list_sumpus_sent3[i],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if i > 0:
- if (
- self.is_sumpus(
- list_sumpus_sent2l[i],
- list_sumpus_sent4[i - 1],
- )
- is False
- ):
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent2l[i],
- list_sumpus_sent4[i - 1],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if not error:
- return (
- "The poem is correct according to the principle."
- )
- else:
- return error
- except:
- return "Something went wrong. Make sure you enter it in the correct form of klon 8."
- elif k_type == 4:
- try:
- error = []
- list_sumpus_sent1 = []
- list_sumpus_sent2h = []
- list_sumpus_sent2l = []
- list_sumpus_sent3 = []
- list_sumpus_sent4 = []
- for i, sent in enumerate(text.split()):
- sub_sent = subword_tokenize(sent, engine="dict")
- if len(sub_sent) > 5:
- error.append(
- "In sentence "
- + str(i + 2)
- + ", there are more than 4 words. "
- + str(sub_sent)
- )
- if (i + 1) % 4 == 1:
- list_sumpus_sent1.append(sub_sent[-1])
- elif (i + 1) % 4 == 2:
- list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]])
- list_sumpus_sent2l.append(sub_sent[-1])
- elif (i + 1) % 4 == 3:
- list_sumpus_sent3.append(sub_sent[-1])
- elif (i + 1) % 4 == 0:
- list_sumpus_sent4.append(sub_sent[-1])
- if (
- len(list_sumpus_sent1) != len(list_sumpus_sent2h)
- or len(list_sumpus_sent2h) != len(list_sumpus_sent2l)
- or len(list_sumpus_sent2l) != len(list_sumpus_sent3)
- or len(list_sumpus_sent3) != len(list_sumpus_sent4)
- or len(list_sumpus_sent4) != len(list_sumpus_sent1)
- ):
- return "The poem does not have 4 complete sentences."
- else:
- for i in range(len(list_sumpus_sent1)):
- countwrong = 0
- for j in list_sumpus_sent2h[i]:
- if (
- self.is_sumpus(list_sumpus_sent1[i], j)
- is False
- ):
- countwrong += 1
- if countwrong > 1:
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent1[i],
- list_sumpus_sent2h[i],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if (
- self.is_sumpus(
- list_sumpus_sent2l[i], list_sumpus_sent3[i]
- )
- is False
- ):
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent2l[i],
- list_sumpus_sent3[i],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if i > 0:
- if (
- self.is_sumpus(
- list_sumpus_sent2l[i],
- list_sumpus_sent4[i - 1],
- )
- is False
- ):
- error.append(
- "Can't find rhyme between paragraphs "
- + str(
- (
- list_sumpus_sent2l[i],
- list_sumpus_sent4[i - 1],
- )
- )
- + " in paragraph "
- + str(i + 1)
- )
- if not error:
- return (
- "The poem is correct according to the principle."
- )
- else:
- return error
- except:
- return "Something went wrong. Make sure you enter it in the correct form."
-
- else:
- return "Something went wrong. Make sure you enter it in the correct form."
-
- def check_aek_too(
- self, text: Union[List[str], str], dead_syllable_as_aek: bool = False
- ) -> Union[List[bool], List[str], bool, str]:
- """
- Checker of Thai tonal words
-
- :param Union[List[str], str] text: Thai word or list of Thai words
- :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek
- :return: the check result if the word is aek or too or False (not both) or list of check results if input is list
- :rtype: Union[List[bool], List[str], bool, str]
-
- :Example:
- ::
-
- from pythainlp.khavee import KhaveeVerifier
-
- kv = KhaveeVerifier()
-
- # การเช็คคำเอกโท
- print(
- kv.check_aek_too("เอง"),
- kv.check_aek_too("เอ่ง"),
- kv.check_aek_too("เอ้ง"),
- )
- # -> False, aek, too
- print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน
- # -> [False, 'aek', 'too']
-
-
- """
- if isinstance(text, list):
- return [self.check_aek_too(t, dead_syllable_as_aek) for t in text]
-
- if not isinstance(text, str):
- raise TypeError("text must be str or iterable list[str]")
-
- word_characters = [*text]
- if "่" in word_characters and "้" not in word_characters:
- return "aek"
- elif "้" in word_characters and "่" not in word_characters:
- return "too"
- if dead_syllable_as_aek and sound_syllable(text) == "dead":
- return "aek"
- else:
- return False
-
- def handle_karun_sound_silence(self, word: str) -> str:
- """
- Handle silent sounds in Thai words using '์' character (Karun)
- by stripping all characters before the 'Karun' character that should be silenced
-
- :param str text: Thai word
- :return: Thai word with silent words stripped
- :rtype: str
- """
- sound_silenced = word.endswith("์")
- if not sound_silenced:
- return word
- # Remove ์ and the silent consonant before it
- # การันต์ (์) marks the consonant immediately before it as silent
- word = word[:-2]
- return word
+# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
+# SPDX-FileType: SOURCE
+# SPDX-License-Identifier: Apache-2.0
+# ruff: noqa: C901
+from __future__ import annotations
+
+from pythainlp import thai_consonants
+from pythainlp.tokenize import subword_tokenize
+from pythainlp.util import remove_tonemark, sound_syllable
+
+
+class KhaveeVerifier:
+ def __init__(self):
+ """
+ KhaveeVerifier: Thai Poetry verifier
+ """
+
+ def _has_true_final_yl(self, word: str) -> bool:
+ """
+ Check if ย or ล is a true final consonant
+ (not just part of the vowel sound with ไ/ใ)
+
+ :param str word: Thai word
+ :return: True if ย or ล is a true final consonant
+ :rtype: bool
+ """
+ if len(word) < 2:
+ return False
+ # Count consonants in the word
+ consonant_count = sum(1 for c in word if c in thai_consonants)
+ # If there are 2+ consonants and word ends with ย or ล, it's a true final
+ return consonant_count >= 2 and word[-1] in ["ย", "ล"]
+
+ def check_sara(self, word: str) -> str:
+ """
+ Check the vowels in the Thai word.
+
+ :param str word: Thai word
+ :return: vowel name of the word
+ :rtype: str
+
+ :Example:
+ ::
+
+ from pythainlp.khavee import KhaveeVerifier
+
+ kv = KhaveeVerifier()
+
+ print(kv.check_sara("เริง"))
+ # output: 'เออ'
+ """
+ sara = []
+ countoa = 0
+
+ # In case of การันย์
+ if "์" in word[-1]:
+ word = word[:-2]
+
+ # In case of สระเดี่ยว
+ for i in word:
+ if i in ("ะ", "ั"):
+ sara.append("อะ")
+ elif i == "ิ":
+ sara.append("อิ")
+ elif i == "ุ":
+ sara.append("อุ")
+ elif i == "ึ":
+ sara.append("อึ")
+ elif i == "ี":
+ sara.append("อี")
+ elif i == "ู":
+ sara.append("อู")
+ elif i == "ื":
+ sara.append("อือ")
+ elif i == "เ":
+ sara.append("เอ")
+ elif i == "แ":
+ sara.append("แอ")
+ elif i == "า":
+ sara.append("อา")
+ elif i == "โ":
+ sara.append("โอ")
+ elif i == "ำ":
+ sara.append("อำ")
+ elif i == "อ":
+ countoa += 1
+ sara.append("ออ")
+ elif i == "ั" and "ว" in word:
+ sara.append("อัว")
+ elif i in ("ไ", "ใ"):
+ sara.append("ไอ")
+ elif i == "็":
+ sara.append("ออ")
+ elif "รร" in word:
+ if self.check_marttra(word) == "กม":
+ sara.append("อำ")
+ else:
+ sara.append("อะ")
+
+ # In case of ออ
+ if countoa == 1 and "อ" in word[-1] and "เ" not in word:
+ sara.remove("ออ")
+
+ # In case of เอ เอ
+ countA = 0
+ for i in sara:
+ if i == "เอ":
+ countA = countA + 1
+ if countA > 1:
+ sara.remove("เอ")
+ sara.remove("เอ")
+ sara.append("แ")
+
+ # In case of สระประสม
+ if "เอ" in sara and "อะ" in sara:
+ sara.remove("เอ")
+ sara.remove("อะ")
+ sara.append("เอะ")
+ elif "แอ" in sara and "อะ" in sara:
+ sara.remove("แอ")
+ sara.remove("อะ")
+ sara.append("แอะ")
+
+ if "เอะ" in sara and "ออ" in sara:
+ sara.remove("เอะ")
+ sara.remove("ออ")
+ sara.append("เออะ")
+ elif "เอ" in sara and "อิ" in sara:
+ sara.remove("เอ")
+ sara.remove("อิ")
+ sara.append("เออ")
+ elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]:
+ sara.remove("เอ")
+ sara.remove("ออ")
+ sara.append("เออ")
+ elif "โอ" in sara and "อะ" in sara:
+ sara.remove("โอ")
+ sara.remove("อะ")
+ sara.append("โอะ")
+ elif "เอ" in sara and "อี" in sara:
+ sara.remove("เอ")
+ sara.remove("อี")
+ sara.append("เอีย")
+ elif "เอ" in sara and "อือ" in sara:
+ sara.remove("เอ")
+ sara.remove("อือ")
+ sara.append("อัว")
+ elif "เอ" in sara and "อา" in sara:
+ sara.remove("เอ")
+ sara.remove("อา")
+ sara.append("เอา")
+ elif "เ" in word and "า" in word and "ะ" in word:
+ sara = []
+ sara.append("เอาะ")
+
+ if "อือ" in sara and "เออ" in sara:
+ sara.remove("เออ")
+ sara.remove("อือ")
+ sara.append("เอือ")
+ elif "ออ" in sara and len(sara) > 1:
+ sara.remove("ออ")
+ elif "ว" in word and len(sara) == 0:
+ sara.append("อัว")
+
+ if "ั" in word and self.check_marttra(word) == "กา":
+ sara = []
+ sara.append("ไอ")
+
+ # In case of อ
+ if word == "เออะ":
+ sara = []
+ sara.append("เออะ")
+ elif word == "เออ":
+ sara = []
+ sara.append("เออ")
+ elif word == "เอ":
+ sara = []
+ sara.append("เอ")
+ elif word == "เอะ":
+ sara = []
+ sara.append("เอะ")
+ elif word == "เอา":
+ sara = []
+ sara.append("เอา")
+ elif word == "เอาะ":
+ sara = []
+ sara.append("เอาะ")
+
+ if "ฤา" in word or "ฦา" in word:
+ sara = []
+ sara.append("อือ")
+ elif "ฤ" in word or "ฦ" in word:
+ sara = []
+ sara.append("อึ")
+
+ # In case of กน
+ if not sara and len(word) == 2:
+ if word[-1] != "ร":
+ sara.append("โอะ")
+ else:
+ sara.append("ออ")
+ elif not sara and len(word) == 3:
+ sara.append("ออ")
+
+ # In case of บ่
+ if word == "บ่":
+ sara = []
+ sara.append("ออ")
+
+ if "ํ" in word:
+ sara = []
+ sara.append("อำ")
+
+ if "เ" in word and "ื" in word and "อ" in word:
+ sara = []
+ sara.append("เอือ")
+
+ if not sara:
+ return "Can't find Sara in this word"
+
+ return sara[0]
+
+ def check_marttra(self, word: str) -> str:
+ """
+ Check the Thai spelling Section in the Thai word.
+
+ :param str word: Thai word
+ :return: name of spelling Section of the word.
+ :rtype: str
+
+ :Example:
+ ::
+
+ from pythainlp.khavee import KhaveeVerifier
+
+ kv = KhaveeVerifier()
+
+ print(kv.check_marttra("สาว"))
+ # output: 'เกอว'
+ """
+ # Handle consonant clusters ending with ร
+ # ตร, ทร → remove ร (treat as final ต/ท sound)
+ # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound)
+ # But single syllable words like "กร" should keep ร
+ if len(word) >= 3 and word[-1] == "ร":
+ if word[-2] in ["ต", "ท"]:
+ word = word[:-1]
+ elif word[-2] in ["ก", "ข", "ค", "ฆ"]:
+ word = word[:-1]
+
+ word = self.handle_karun_sound_silence(word)
+ word = remove_tonemark(word)
+
+ # Check for ำ at the end (represents "am" sound, ends with m)
+ if word[-1] == "ำ":
+ return "กม"
+
+ # Check for vowels and special patterns that indicate open syllables (กา)
+ # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel
+ if "ไ" in word or "ใ" in word:
+ if word[-1] not in ["ย", "ล"]:
+ return "กา"
+ elif not self._has_true_final_yl(word):
+ # ย/ล is part of the vowel sound, not a true final
+ return "กา"
+ # else: ย/ล is a true final, continue to consonant classification below
+
+ if "ํ" in word and "า" in word:
+ return "กา"
+ elif (
+ word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"]
+ or ("ี" in word and "ย" in word[-1])
+ or ("ื" in word and "อ" in word[-1])
+ ):
+ return "กา"
+ elif word[-1] in ["ง"]:
+ return "กง"
+ elif word[-1] in ["ม"]:
+ return "กม"
+ elif word[-1] in ["ย"]:
+ return "เกย"
+ elif word[-1] in ["ล"]:
+ return "เกย"
+ elif word[-1] in ["ว"]:
+ return "เกอว"
+ elif word[-1] in ["ก", "ข", "ค", "ฆ"]:
+ return "กก"
+ elif word[-1] in [
+ "จ",
+ "ช",
+ "ซ",
+ "ฎ",
+ "ฏ",
+ "ฐ",
+ "ฑ",
+ "ฒ",
+ "ด",
+ "ต",
+ "ถ",
+ "ท",
+ "ธ",
+ "ศ",
+ "ษ",
+ "ส",
+ ]:
+ return "กด"
+ elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]:
+ return "กน"
+ elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]:
+ return "กบ"
+ else:
+ if "็" in word:
+ return "กา"
+ else:
+ return "Cant find Marttra in this word"
+
+ def is_sumpus(self, word1: str, word2: str) -> bool:
+ """
+ Check the rhyme between two words.
+
+ :param str word1: Thai word
+ :param str word2: Thai word
+ :return: boolean
+ :rtype: bool
+
+ :Example:
+ ::
+
+ from pythainlp.khavee import KhaveeVerifier
+
+ kv = KhaveeVerifier()
+
+ print(kv.is_sumpus("สรร", "อัน"))
+ # output: True
+
+ print(kv.is_sumpus("สรร", "แมว"))
+ # output: False
+ """
+ marttra1 = self.check_marttra(word1)
+ marttra2 = self.check_marttra(word2)
+ sara1 = self.check_sara(word1)
+ sara2 = self.check_sara(word2)
+ if sara1 == "อะ" and marttra1 == "เกย":
+ sara1 = "ไอ"
+ marttra1 = "กา"
+ elif sara2 == "อะ" and marttra2 == "เกย":
+ sara2 = "ไอ"
+ marttra2 = "กา"
+ if sara1 == "อำ" and marttra1 == "กม":
+ sara1 = "อำ"
+ marttra1 = "กา"
+ elif sara2 == "อำ" and marttra2 == "กม":
+ sara2 = "อำ"
+ marttra2 = "กา"
+ return bool(marttra1 == marttra2 and sara1 == sara2)
+
+ def check_karu_lahu(self, text):
+ if (
+ self.check_marttra(text) != "กา"
+ or (
+ self.check_marttra(text) == "กา"
+ and self.check_sara(text)
+ in [
+ "อา",
+ "อี",
+ "อือ",
+ "อู",
+ "เอ",
+ "แอ",
+ "โอ",
+ "ออ",
+ "เออ",
+ "เอีย",
+ "เอือ",
+ "อัว",
+ ]
+ )
+ or self.check_sara(text) in ["อำ", "ไอ", "เอา"]
+ ) and text not in ["บ่", "ณ", "ธ", "ก็"]:
+ return "karu"
+ else:
+ return "lahu"
+
+ def check_klon(self, text: str, k_type: int = 8) -> list[str] | str:
+ """
+ Check the suitability of the poem according to Thai principles.
+
+ :param str text: Thai poem
+ :param int k_type: type of Thai poem
+ :return: the check results of the suitability of the poem according to Thai principles.
+ :rtype: Union[List[str], str]
+
+ :Example:
+ ::
+
+ from pythainlp.khavee import KhaveeVerifier
+
+ kv = KhaveeVerifier()
+
+ print(kv.check_klon(
+ 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \
+ มีคนจับจอง เขาชื่อน้องเธียร',
+ k_type=4
+ ))
+ # output: The poem is correct according to the principle.
+
+ print(kv.check_klon(
+ 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \
+ เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร',
+ k_type=4
+ ))
+ # output: [
+ "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2",
+ "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2"
+ ]
+ """
+ if k_type == 8:
+ try:
+ error = []
+ list_sumpus_sent1 = []
+ list_sumpus_sent2h = []
+ list_sumpus_sent2l = []
+ list_sumpus_sent3 = []
+ list_sumpus_sent4 = []
+ for i, sent in enumerate(text.split()):
+ sub_sent = subword_tokenize(sent, engine="dict")
+ if len(sub_sent) > 10:
+ error.append(
+ "In sentence "
+ + str(i + 2)
+ + ", there are more than 10 words. "
+ + str(sub_sent)
+ )
+ if (i + 1) % 4 == 1:
+ list_sumpus_sent1.append(sub_sent[-1])
+ elif (i + 1) % 4 == 2:
+ list_sumpus_sent2h.append(
+ [
+ sub_sent[1],
+ sub_sent[2],
+ sub_sent[3],
+ sub_sent[4],
+ ]
+ )
+ list_sumpus_sent2l.append(sub_sent[-1])
+ elif (i + 1) % 4 == 3:
+ list_sumpus_sent3.append(sub_sent[-1])
+ elif (i + 1) % 4 == 0:
+ list_sumpus_sent4.append(sub_sent[-1])
+ if (
+ len(list_sumpus_sent1) != len(list_sumpus_sent2h)
+ or len(list_sumpus_sent2h) != len(list_sumpus_sent2l)
+ or len(list_sumpus_sent2l) != len(list_sumpus_sent3)
+ or len(list_sumpus_sent3) != len(list_sumpus_sent4)
+ or len(list_sumpus_sent4) != len(list_sumpus_sent1)
+ ):
+ return "The poem does not have 4 complete sentences."
+ else:
+ for i in range(len(list_sumpus_sent1)):
+ countwrong = 0
+ for j in list_sumpus_sent2h[i]:
+ if (
+ self.is_sumpus(list_sumpus_sent1[i], j)
+ is False
+ ):
+ countwrong += 1
+ if countwrong > 3:
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent1[i],
+ list_sumpus_sent2h[i],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if (
+ self.is_sumpus(
+ list_sumpus_sent2l[i], list_sumpus_sent3[i]
+ )
+ is False
+ ):
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent2l[i],
+ list_sumpus_sent3[i],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if i > 0:
+ if (
+ self.is_sumpus(
+ list_sumpus_sent2l[i],
+ list_sumpus_sent4[i - 1],
+ )
+ is False
+ ):
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent2l[i],
+ list_sumpus_sent4[i - 1],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if not error:
+ return (
+ "The poem is correct according to the principle."
+ )
+ else:
+ return error
+ except:
+ return "Something went wrong. Make sure you enter it in the correct form of klon 8."
+ elif k_type == 4:
+ try:
+ error = []
+ list_sumpus_sent1 = []
+ list_sumpus_sent2h = []
+ list_sumpus_sent2l = []
+ list_sumpus_sent3 = []
+ list_sumpus_sent4 = []
+ for i, sent in enumerate(text.split()):
+ sub_sent = subword_tokenize(sent, engine="dict")
+ if len(sub_sent) > 5:
+ error.append(
+ "In sentence "
+ + str(i + 2)
+ + ", there are more than 4 words. "
+ + str(sub_sent)
+ )
+ if (i + 1) % 4 == 1:
+ list_sumpus_sent1.append(sub_sent[-1])
+ elif (i + 1) % 4 == 2:
+ list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]])
+ list_sumpus_sent2l.append(sub_sent[-1])
+ elif (i + 1) % 4 == 3:
+ list_sumpus_sent3.append(sub_sent[-1])
+ elif (i + 1) % 4 == 0:
+ list_sumpus_sent4.append(sub_sent[-1])
+ if (
+ len(list_sumpus_sent1) != len(list_sumpus_sent2h)
+ or len(list_sumpus_sent2h) != len(list_sumpus_sent2l)
+ or len(list_sumpus_sent2l) != len(list_sumpus_sent3)
+ or len(list_sumpus_sent3) != len(list_sumpus_sent4)
+ or len(list_sumpus_sent4) != len(list_sumpus_sent1)
+ ):
+ return "The poem does not have 4 complete sentences."
+ else:
+ for i in range(len(list_sumpus_sent1)):
+ countwrong = 0
+ for j in list_sumpus_sent2h[i]:
+ if (
+ self.is_sumpus(list_sumpus_sent1[i], j)
+ is False
+ ):
+ countwrong += 1
+ if countwrong > 1:
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent1[i],
+ list_sumpus_sent2h[i],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if (
+ self.is_sumpus(
+ list_sumpus_sent2l[i], list_sumpus_sent3[i]
+ )
+ is False
+ ):
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent2l[i],
+ list_sumpus_sent3[i],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if i > 0:
+ if (
+ self.is_sumpus(
+ list_sumpus_sent2l[i],
+ list_sumpus_sent4[i - 1],
+ )
+ is False
+ ):
+ error.append(
+ "Can't find rhyme between paragraphs "
+ + str(
+ (
+ list_sumpus_sent2l[i],
+ list_sumpus_sent4[i - 1],
+ )
+ )
+ + " in paragraph "
+ + str(i + 1)
+ )
+ if not error:
+ return (
+ "The poem is correct according to the principle."
+ )
+ else:
+ return error
+ except:
+ return "Something went wrong. Make sure you enter it in the correct form."
+
+ else:
+ return "Something went wrong. Make sure you enter it in the correct form."
+
+ def check_aek_too(
+ self, text: list[str] | str, dead_syllable_as_aek: bool = False
+ ) -> list[bool] | list[str] | bool | str:
+ """
+ Checker of Thai tonal words
+
+ :param Union[List[str], str] text: Thai word or list of Thai words
+ :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek
+ :return: the check result if the word is aek or too or False (not both) or list of check results if input is list
+ :rtype: Union[List[bool], List[str], bool, str]
+
+ :Example:
+ ::
+
+ from pythainlp.khavee import KhaveeVerifier
+
+ kv = KhaveeVerifier()
+
+ # การเช็คคำเอกโท
+ print(
+ kv.check_aek_too("เอง"),
+ kv.check_aek_too("เอ่ง"),
+ kv.check_aek_too("เอ้ง"),
+ )
+ # -> False, aek, too
+ print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน
+ # -> [False, 'aek', 'too']
+
+
+ """
+ if isinstance(text, list):
+ return [self.check_aek_too(t, dead_syllable_as_aek) for t in text]
+
+ if not isinstance(text, str):
+ raise TypeError("text must be str or iterable list[str]")
+
+ word_characters = [*text]
+ if "่" in word_characters and "้" not in word_characters:
+ return "aek"
+ elif "้" in word_characters and "่" not in word_characters:
+ return "too"
+ if dead_syllable_as_aek and sound_syllable(text) == "dead":
+ return "aek"
+ else:
+ return False
+
+ def handle_karun_sound_silence(self, word: str) -> str:
+ """
+ Handle silent sounds in Thai words using '์' character (Karun)
+ by stripping all characters before the 'Karun' character that should be silenced
+
+ :param str text: Thai word
+ :return: Thai word with silent words stripped
+ :rtype: str
+ """
+ sound_silenced = word.endswith("์")
+ if not sound_silenced:
+ return word
+ # Remove ์ and the silent consonant before it
+ # การันต์ (์) marks the consonant immediately before it as silent
+ word = word[:-2]
+ return word
diff --git a/pythainlp/lm/__init__.py b/pythainlp/lm/__init__.py
index 60d8b503c..259f101d2 100644
--- a/pythainlp/lm/__init__.py
+++ b/pythainlp/lm/__init__.py
@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-__all__ = [
- "calculate_ngram_counts",
- "remove_repeated_ngrams"
-]
+__all__ = ["calculate_ngram_counts", "remove_repeated_ngrams"]
-from pythainlp.lm.text_util import calculate_ngram_counts, remove_repeated_ngrams
+from pythainlp.lm.text_util import (
+ calculate_ngram_counts,
+ remove_repeated_ngrams,
+)
diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py
index 946fd8451..35e4b9167 100644
--- a/pythainlp/lm/text_util.py
+++ b/pythainlp/lm/text_util.py
@@ -1,16 +1,13 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
# ruff: noqa: C901
-
-from typing import List, Tuple, Dict
+from __future__ import annotations
def calculate_ngram_counts(
- list_words: List[str],
- n_min: int = 2,
- n_max: int = 4) -> Dict[Tuple[str], int]:
+ list_words: list[str], n_min: int = 2, n_max: int = 4
+) -> dict[tuple[str], int]:
"""
Calculates the counts of n-grams in the list words for the specified range.
@@ -26,13 +23,13 @@ def calculate_ngram_counts(
for n in range(n_min, n_max + 1):
for i in range(len(list_words) - n + 1):
- ngram = tuple(list_words[i:i + n])
+ ngram = tuple(list_words[i : i + n])
ngram_counts[ngram] = ngram_counts.get(ngram, 0) + 1
return ngram_counts
-def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]:
+def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]:
"""
Remove repeated n-grams
@@ -46,7 +43,7 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]:
from pythainlp.lm import remove_repeated_ngrams
- remove_repeated_ngrams(['เอา', 'เอา', 'แบบ', 'ไหน'], n=1)
+ remove_repeated_ngrams(["เอา", "เอา", "แบบ", "ไหน"], n=1)
# output: ['เอา', 'แบบ', 'ไหน']
"""
if not string_list or n <= 0:
@@ -58,12 +55,14 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]:
for i in range(len(string_list)):
if i + n <= len(string_list):
- ngram = tuple(string_list[i:i + n])
+ ngram = tuple(string_list[i : i + n])
if ngram not in unique_ngrams:
unique_ngrams.add(ngram)
- if not output_list or output_list[-(n - 1):] != list(ngram[:-1]):
+ if not output_list or output_list[-(n - 1) :] != list(
+ ngram[:-1]
+ ):
output_list.extend(ngram)
else:
output_list.append(ngram[-1])
diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py
index d40baa777..3f8ed10dc 100644
--- a/pythainlp/morpheme/__init__.py
+++ b/pythainlp/morpheme/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,9 +5,7 @@
"""
PyThaiNLP morpheme
"""
-__all__ = [
- "nighit",
- "is_native_thai"
-]
+
+__all__ = ["nighit", "is_native_thai"]
from pythainlp.morpheme.thaiwordcheck import is_native_thai
from pythainlp.morpheme.word_formation import nighit
diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py
index 61c02baff..9fce45aaf 100644
--- a/pythainlp/morpheme/thaiwordcheck.py
+++ b/pythainlp/morpheme/thaiwordcheck.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -13,6 +12,9 @@
http://www.trueplookpanya.com/learning/detail/30589-043067
- วารุณี บำรุงรส 2010. คำไทยแท้ https://www.gotoknow.org/posts/377619
"""
+
+from __future__ import annotations
+
import re
_THANTHAKHAT_CHAR = "\u0e4c" # Thanthakhat (cancellation of sound)
diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py
index ede6c248d..11b432d93 100644
--- a/pythainlp/morpheme/word_formation.py
+++ b/pythainlp/morpheme/word_formation.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
from pythainlp import thai_consonants
@@ -31,7 +32,7 @@ def nighit(w1: str, w2: str) -> str:
assert nighit("สํ","ปทา")=="สัมปทา"
assert nighit("สํ","โยค")=="สังโยค"
"""
- if not str(w1).endswith('ํ') and len(w1) != 2:
+ if not str(w1).endswith("ํ") and len(w1) != 2:
raise NotImplementedError(f"The function doesn't support {w1}.")
list_w1 = list(w1)
list_w2 = list(w2)
@@ -56,4 +57,4 @@ def nighit(w1: str, w2: str) -> str:
The function doesn't support {w1} and {w2}.
""")
newword.extend(list_w2)
- return ''.join(newword)
+ return "".join(newword)
diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py
index d7aae6f1c..e4afb08e1 100644
--- a/pythainlp/parse/__init__.py
+++ b/pythainlp/parse/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,6 +5,7 @@
"""
PyThaiNLP Parse
"""
+
__all__ = ["dependency_parsing"]
from pythainlp.parse.core import dependency_parsing
diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py
index 2a64b4bd5..6421dd87a 100644
--- a/pythainlp/parse/core.py
+++ b/pythainlp/parse/core.py
@@ -1,9 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-
-from typing import List, Union
+from __future__ import annotations
_tagger = None
_tagger_name = ""
@@ -11,10 +9,10 @@
def dependency_parsing(
text: str,
- model: Union[str, None] = None,
+ model: str | None = None,
tag: str = "str",
engine: str = "esupar",
-) -> Union[List[List[str]], str]:
+) -> list[list[str]] | str:
"""
Dependency Parsing
diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py
index 259cb00b7..06ae3a35e 100644
--- a/pythainlp/parse/esupar_engine.py
+++ b/pythainlp/parse/esupar_engine.py
@@ -1,10 +1,10 @@
-# -*- coding: utf-8 -*-
"""
esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages
GitHub: https://github.com/KoichiYasuoka/esupar
"""
-from typing import List, Union
+
+from __future__ import annotations
try:
import esupar
@@ -18,9 +18,7 @@ def __init__(self, model: str = "th") -> None:
model = "th"
self.nlp = esupar.load(model)
- def __call__(
- self, text: str, tag: str = "str"
- ) -> Union[List[List[str]], str]:
+ def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str:
_data = str(self.nlp(text))
if tag == "list":
_temp = _data.splitlines()
diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py
index 6d0eef7ab..3f44a61a7 100644
--- a/pythainlp/parse/spacy_thai_engine.py
+++ b/pythainlp/parse/spacy_thai_engine.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
"""
spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies.
GitHub: https://github.com/KoichiYasuoka/spacy-thai
"""
-from typing import List, Union
+
+from __future__ import annotations
import spacy_thai
@@ -14,9 +14,7 @@ class Parse:
def __init__(self, model: str = "th") -> None:
self.nlp = spacy_thai.load()
- def __call__(
- self, text: str, tag: str = "str"
- ) -> Union[List[List[str]], str]:
+ def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str:
doc = self.nlp(text)
_text = []
if tag == "list":
diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py
index 9d1ffca5b..2f4aec7b2 100644
--- a/pythainlp/parse/transformers_ud.py
+++ b/pythainlp/parse/transformers_ud.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""
TransformersUD
@@ -10,8 +9,10 @@
GitHub: https://github.com/KoichiYasuoka
"""
+
+from __future__ import annotations
+
import os
-from typing import List, Union
import numpy
import torch
@@ -56,9 +57,7 @@ def __init__(
model=t, tokenizer=self.tokenizer
)
- def __call__(
- self, text: str, tag: str = "str"
- ) -> Union[List[List[str]], str]:
+ def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str:
w = [
(t["start"], t["end"], t["entity_group"])
for t in self.deprel(text)
diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py
index fc258d41d..904c64dd3 100644
--- a/pythainlp/parse/ud_goeswith.py
+++ b/pythainlp/parse/ud_goeswith.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""
UDgoeswith
@@ -10,7 +9,8 @@
GitHub: https://github.com/KoichiYasuoka
"""
-from typing import List, Union
+
+from __future__ import annotations
import numpy as np
import torch
@@ -27,9 +27,7 @@ def __init__(
self.tokenizer = AutoTokenizer.from_pretrained(model)
self.model = AutoModelForTokenClassification.from_pretrained(model)
- def __call__(
- self, text: str, tag: str = "str"
- ) -> Union[List[List[str]], str]:
+ def __call__(self, text: str, tag: str = "str") -> list[list[str]] | str:
w = self.tokenizer(text, return_offsets_mapping=True)
v = w["input_ids"]
x = [
diff --git a/pythainlp/phayathaibert/__init__.py b/pythainlp/phayathaibert/__init__.py
index 2d50efabd..088f8d219 100644
--- a/pythainlp/phayathaibert/__init__.py
+++ b/pythainlp/phayathaibert/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py
index 5631b766c..a73bcdfe9 100644
--- a/pythainlp/phayathaibert/core.py
+++ b/pythainlp/phayathaibert/core.py
@@ -1,12 +1,12 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
import random
import re
import warnings
-from typing import Callable, List, Tuple, Union
+from collections.abc import Callable
from transformers import (
CamembertTokenizer,
@@ -139,7 +139,7 @@ def _replace_rep(m):
re_rep = re.compile(r"(\S)(\1{3,})")
return re_rep.sub(_replace_rep, text)
- def replace_wrep_post(self, toks: List[str]) -> List[str]:
+ def replace_wrep_post(self, toks: list[str]) -> list[str]:
"""
Replace repetitive words post tokenization;
fastai `replace_wrep` does not work well with Thai.
@@ -166,7 +166,7 @@ def replace_wrep_post(self, toks: List[str]) -> List[str]:
return res[1:]
- def remove_space(self, toks: List[str]) -> List[str]:
+ def remove_space(self, toks: list[str]) -> list[str]:
"""
Do not include space for bag-of-word models.
:param List[str] toks: list of tokens
@@ -189,7 +189,7 @@ def remove_space(self, toks: List[str]) -> List[str]:
def preprocess(
self,
text: str,
- pre_rules: List[Callable] = [
+ pre_rules: list[Callable] = [
rm_brackets,
replace_newlines,
rm_useless_spaces,
@@ -253,7 +253,7 @@ def augment(
text: str,
num_augs: int = 3,
sample: bool = False,
- ) -> List[str]:
+ ) -> list[str]:
"""
Text augmentation from PhayaThaiBERT
@@ -315,7 +315,7 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None:
def get_tag(
self, sentence: str, strategy: str = "simple"
- ) -> List[List[Tuple[str, str]]]:
+ ) -> list[list[tuple[str, str]]]:
"""
Marks sentences with part-of-speech (POS) tags.
@@ -363,7 +363,7 @@ def get_ner(
tag: bool = False,
pos: bool = False,
strategy: str = "simple",
- ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]:
+ ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str:
"""
This function tags named entities in text in IOB format.
@@ -435,7 +435,7 @@ def get_ner(
return sample_output
-def segment(sentence: str) -> List[str]:
+def segment(sentence: str) -> list[str]:
"""
Subword tokenize of PhayaThaiBERT, \
sentencepiece from WangchanBERTa model with vocabulary expansion.
diff --git a/pythainlp/soundex/__init__.py b/pythainlp/soundex/__init__.py
index ed924db9f..6ff802204 100644
--- a/pythainlp/soundex/__init__.py
+++ b/pythainlp/soundex/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/soundex/core.py b/pythainlp/soundex/core.py
index 8ecbcb1b7..e0435a92e 100644
--- a/pythainlp/soundex/core.py
+++ b/pythainlp/soundex/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -7,6 +6,9 @@
Has three systems to choose from: Udom83 (default), LK82, and MetaSound
"""
+
+from __future__ import annotations
+
from pythainlp.soundex import DEFAULT_SOUNDEX_ENGINE
from pythainlp.soundex.lk82 import lk82
from pythainlp.soundex.metasound import metasound
diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py
index fd3010e69..1885553a7 100644
--- a/pythainlp/soundex/lk82.py
+++ b/pythainlp/soundex/lk82.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -15,6 +14,9 @@
by Korakot Chaovavanich
https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8
"""
+
+from __future__ import annotations
+
import re
from pythainlp.util import remove_tonemark
diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py
index 7729d520c..924f9b58d 100644
--- a/pythainlp/soundex/metasound.py
+++ b/pythainlp/soundex/metasound.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -12,6 +11,8 @@
https://pdfs.semanticscholar.org/3983/963e87ddc6dfdbb291099aa3927a0e3e4ea6.pdf
"""
+from __future__ import annotations
+
_CONS_THANTHAKHAT = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ์"
_THANTHAKHAT = "์" # \u0e4c
_C1 = "กขฃคฆฅ" # sound K -> coded letter 1
diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py
index 5ba0908d7..ae74895ae 100644
--- a/pythainlp/soundex/prayut_and_somchaip.py
+++ b/pythainlp/soundex/prayut_and_somchaip.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,6 +8,9 @@
References:
Prayut Suwanvisat, Somchai Prasitjutrakul.Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf
"""
+
+from __future__ import annotations
+
from pythainlp import thai_characters
_C0 = "AEIOUHWYอ"
diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py
index 355949d90..a376d0715 100644
--- a/pythainlp/soundex/sound.py
+++ b/pythainlp/soundex/sound.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List
+from __future__ import annotations
import panphon
import panphon.distance
@@ -13,6 +12,7 @@
_ft = panphon.FeatureTable()
_dst = panphon.distance.Distance()
+
def _clean_ipa(ipa: str) -> str:
"""
Clean IPA by removing tones and space between phonetic codes
@@ -21,7 +21,18 @@ def _clean_ipa(ipa: str) -> str:
:return: IPA with tones removed from the text
:rtype: str
"""
- return ipa.replace("˩˩˦","").replace("˥˩","").replace("˨˩","").replace("˦˥","").replace("˧","").replace("˧","").replace(" .",".").replace(". ",".").strip()
+ return (
+ ipa.replace("˩˩˦", "")
+ .replace("˥˩", "")
+ .replace("˨˩", "")
+ .replace("˦˥", "")
+ .replace("˧", "")
+ .replace("˧", "")
+ .replace(" .", ".")
+ .replace(". ", ".")
+ .strip()
+ )
+
def word2audio(word: str) -> str:
"""
@@ -41,10 +52,13 @@ def word2audio(word: str) -> str:
"""
_word = word_tokenize(word)
_phone = [pronunciate(w, engine="w2p") for w in _word]
- _ipa = [_clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone]
- return '.'.join(_ipa)
+ _ipa = [
+ _clean_ipa(transliterate(phone, engine="thaig2p")) for phone in _phone
+ ]
+ return ".".join(_ipa)
-def audio_vector(word: str) -> List[List[int]]:
+
+def audio_vector(word: str) -> list[list[int]]:
"""
Convert audio to vector list
@@ -62,7 +76,8 @@ def audio_vector(word: str) -> List[List[int]]:
"""
return _ft.word_to_vector_list(word2audio(word), numeric=True)
-def word_approximation(word: str, list_word: List[str]) -> List[float]:
+
+def word_approximation(word: str, list_word: list[str]) -> list[float]:
"""
Thai Word Approximation
@@ -81,5 +96,7 @@ def word_approximation(word: str, list_word: List[str]) -> List[float]:
"""
_word = word2audio(word)
_list_word = [word2audio(w) for w in list_word]
- _distance = [_dst.weighted_feature_edit_distance(_word, w) for w in _list_word]
+ _distance = [
+ _dst.weighted_feature_edit_distance(_word, w) for w in _list_word
+ ]
return _distance
diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py
index 7b401bb01..caf6dd45a 100644
--- a/pythainlp/soundex/udom83.py
+++ b/pythainlp/soundex/udom83.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -16,6 +15,9 @@
by Korakot Chaovavanich
https://gist.github.com/korakot/0b772e09340cac2f493868da035597e8
"""
+
+from __future__ import annotations
+
import re
from pythainlp import thai_consonants
diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py
index 54e9af445..611a06230 100644
--- a/pythainlp/spell/__init__.py
+++ b/pythainlp/spell/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -22,4 +21,6 @@
# these imports are placed here to avoid circular imports
from pythainlp.spell.core import correct, correct_sent, spell, spell_sent
-from pythainlp.spell.words_spelling_correction import get_words_spell_suggestion
+from pythainlp.spell.words_spelling_correction import (
+ get_words_spell_suggestion,
+)
diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py
index a9320e756..d99ce5154 100644
--- a/pythainlp/spell/core.py
+++ b/pythainlp/spell/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,18 +5,21 @@
Spell checking functions
"""
-from functools import lru_cache
+from __future__ import annotations
+
import itertools
-from typing import List
+from functools import lru_cache
from pythainlp.spell import DEFAULT_SPELL_CHECKER
+
@lru_cache
def default_spell_checker():
"""Lazy load default spell checker with cache"""
return DEFAULT_SPELL_CHECKER()
-def spell(word: str, engine: str = "pn") -> List[str]:
+
+def spell(word: str, engine: str = "pn") -> list[str]:
"""
Provides a list of possible correct spellings of the given word.
The list of words are from the words in the dictionary
@@ -42,13 +44,13 @@ def spell(word: str, engine: str = "pn") -> List[str]:
from pythainlp.spell import spell
- spell("เส้นตรบ", engine="pn")
+ spell("เส้นตรบ", engine="pn")
# output: ['เส้นตรง']
spell("เส้นตรบ")
# output: ['เส้นตรง']
- spell("เส้นตรบ", engine="tltk")
+ spell("เส้นตรบ", engine="tltk")
# output: ['เส้นตรง']
spell("ครัช")
@@ -125,7 +127,9 @@ def correct(word: str, engine: str = "pn") -> str:
text_correct = SPELL_CHECKER(word)
elif engine == "wanchanberta_thai_grammarly":
- from pythainlp.spell.wanchanberta_thai_grammarly import correct as SPELL_CHECKER
+ from pythainlp.spell.wanchanberta_thai_grammarly import (
+ correct as SPELL_CHECKER,
+ )
text_correct = SPELL_CHECKER(word)
@@ -135,7 +139,7 @@ def correct(word: str, engine: str = "pn") -> str:
return text_correct
-def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]:
+def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]:
"""
Provides a list of possible correct spellings of sentence
@@ -152,7 +156,7 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]:
from pythainlp.spell import spell_sent
- spell_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy')
+ spell_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy")
# output: [['เด็ก', 'อินเทอร์เน็ต', 'แรง']]
"""
if engine == "symspellpy":
@@ -173,7 +177,7 @@ def spell_sent(list_words: List[str], engine: str = "pn") -> List[List[str]]:
return list_new
-def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]:
+def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]:
"""
Corrects and returns the spelling of the given sentence
@@ -191,7 +195,7 @@ def correct_sent(list_words: List[str], engine: str = "pn") -> List[str]:
from pythainlp.spell import correct_sent
- correct_sent(["เด็","อินอร์เน็ต","แรง"],engine='symspellpy')
+ correct_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy")
# output: ['เด็ก', 'อินเทอร์เน็ต', 'แรง']
"""
return spell_sent(list_words, engine=engine)[0]
diff --git a/pythainlp/spell/phunspell.py b/pythainlp/spell/phunspell.py
index 7a293816d..26cb90f1e 100644
--- a/pythainlp/spell/phunspell.py
+++ b/pythainlp/spell/phunspell.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,17 +10,20 @@
* \
https://github.com/dvwright/phunspell
"""
-from typing import List
+
+from __future__ import annotations
try:
import phunspell
except ImportError:
- raise ImportError("Import Error; Install phunspell by pip install phunspell")
+ raise ImportError(
+ "Import Error; Install phunspell by pip install phunspell"
+ )
pspell = phunspell.Phunspell("th_TH")
-def spell(text: str) -> List[str]:
+def spell(text: str) -> list[str]:
return list(pspell.suggest(text))
diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py
index 4cdfab5d7..12d2acc3e 100644
--- a/pythainlp/spell/pn.py
+++ b/pythainlp/spell/pn.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""
Spell checker, using Peter Norvig algorithm.
Spelling dictionary can be customized.
@@ -6,19 +5,12 @@
Based on Peter Norvig's Python code from http://norvig.com/spell-correct.html
"""
+
+from __future__ import annotations
+
from collections import Counter
+from collections.abc import Callable, ItemsView, Iterable
from string import digits
-from typing import (
- Callable,
- Dict,
- ItemsView,
- Iterable,
- List,
- Optional,
- Set,
- Tuple,
- Union,
-)
from pythainlp import thai_digits, thai_letters
from pythainlp.corpus import tnc
@@ -39,7 +31,7 @@ def _is_thai_and_not_num(word: str) -> bool:
def _keep(
- word_freq: Tuple[str, int],
+ word_freq: tuple[str, int],
min_freq: int,
min_len: int,
max_len: int,
@@ -59,7 +51,7 @@ def _keep(
return dict_filter(word)
-def _edits1(word: str) -> Set[str]:
+def _edits1(word: str) -> set[str]:
"""
Returns a set of words with an edit distance of 1 from the input word
"""
@@ -72,7 +64,7 @@ def _edits1(word: str) -> Set[str]:
return set(deletes + transposes + replaces + inserts)
-def _edits2(word: str) -> Set[str]:
+def _edits2(word: str) -> set[str]:
"""
Returns a set of words with an edit distance of 2 from the input word
"""
@@ -80,14 +72,12 @@ def _edits2(word: str) -> Set[str]:
def _convert_custom_dict(
- custom_dict: Union[
- Dict[str, int], Iterable[str], Iterable[Tuple[str, int]]
- ],
+ custom_dict: dict[str, int] | Iterable[str] | Iterable[tuple[str, int]],
min_freq: int,
min_len: int,
max_len: int,
- dict_filter: Optional[Callable[[str], bool]],
-) -> List[Tuple[str, int]]:
+ dict_filter: Callable[[str], bool] | None,
+) -> list[tuple[str, int]]:
"""
Converts a custom dictionary to a list of (str, int) tuples
"""
@@ -123,13 +113,13 @@ def _convert_custom_dict(
class NorvigSpellChecker:
def __init__(
self,
- custom_dict: Union[
- Dict[str, int], Iterable[str], Iterable[Tuple[str, int]]
- ] = None,
+ custom_dict: dict[str, int]
+ | Iterable[str]
+ | Iterable[tuple[str, int]] = None,
min_freq: int = 2,
min_len: int = 2,
max_len: int = 40,
- dict_filter: Optional[Callable[[str], bool]] = _is_thai_and_not_num,
+ dict_filter: Callable[[str], bool] | None = _is_thai_and_not_num,
):
"""
Initializes Peter Norvig's spell checker object.
@@ -191,7 +181,7 @@ def dictionary(self) -> ItemsView[str, int]:
from pythainlp.spell import NorvigSpellChecker
- dictionary= [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)]
+ dictionary = [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)]
checker = NorvigSpellChecker(custom_dict=dictionary)
checker.dictionary()
@@ -199,7 +189,7 @@ def dictionary(self) -> ItemsView[str, int]:
"""
return self.__WORDS.items()
- def known(self, words: Iterable[str]) -> List[str]:
+ def known(self, words: Iterable[str]) -> list[str]:
"""
Returns a list of given words found in the spelling dictionary
@@ -220,7 +210,7 @@ def known(self, words: Iterable[str]) -> List[str]:
checker.known(["เพยน", "เพล", "เพลง"])
# output: ['เพล', 'เพลง']
- checker.known(['ยกไ', 'ไฟล์ม'])
+ checker.known(["ยกไ", "ไฟล์ม"])
# output: []
checker.known([])
@@ -280,7 +270,7 @@ def freq(self, word: str) -> int:
"""
return self.__WORDS[word]
- def spell(self, word: str) -> List[str]:
+ def spell(self, word: str) -> list[str]:
"""
Returns a list of all correctly-spelled words whose spelling
is similar to the given word by edit distance metrics.
diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py
index b3c5abb81..d75b502f3 100644
--- a/pythainlp/spell/symspellpy.py
+++ b/pythainlp/spell/symspellpy.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -13,12 +12,14 @@
https://github.com/mammothb/symspellpy
"""
-from typing import List
+from __future__ import annotations
try:
from symspellpy import SymSpell, Verbosity
except ImportError:
- raise ImportError("Import Error; Install symspellpy by pip install symspellpy")
+ raise ImportError(
+ "Import Error; Install symspellpy by pip install symspellpy"
+ )
from pythainlp.corpus import get_corpus_path, path_pythainlp_corpus
@@ -42,7 +43,7 @@
)
-def spell(text: str, max_edit_distance: int = 2) -> List[str]:
+def spell(text: str, max_edit_distance: int = 2) -> list[str]:
return [
str(i).split(",", maxsplit=1)[0]
for i in list(
@@ -58,8 +59,8 @@ def correct(text: str, max_edit_distance: int = 1) -> str:
def spell_sent(
- list_words: List[str], max_edit_distance: int = 2
-) -> List[List[str]]:
+ list_words: list[str], max_edit_distance: int = 2
+) -> list[list[str]]:
temp = [
str(i).split(",", maxsplit=1)[0].split(" ")
for i in list(
@@ -77,7 +78,7 @@ def spell_sent(
return list_new
-def correct_sent(list_words: List[str], max_edit_distance=1) -> List[str]:
+def correct_sent(list_words: list[str], max_edit_distance=1) -> list[str]:
return [
i[0]
for i in spell_sent(list_words, max_edit_distance=max_edit_distance)
diff --git a/pythainlp/spell/tltk.py b/pythainlp/spell/tltk.py
index dd120aa3d..d21381675 100644
--- a/pythainlp/spell/tltk.py
+++ b/pythainlp/spell/tltk.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,12 +10,16 @@
* \
https://pypi.org/project/tltk/
"""
+
+from __future__ import annotations
+
try:
from tltk.nlp import spell_candidates
except ImportError:
- raise ImportError("Not found tltk! Please install tltk by pip install tltk")
-from typing import List
+ raise ImportError(
+ "Not found tltk! Please install tltk by pip install tltk"
+ )
-def spell(text: str) -> List[str]:
+def spell(text: str) -> list[str]:
return spell_candidates(text)
diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py
index 2707467f8..6cf81f878 100644
--- a/pythainlp/spell/wanchanberta_thai_grammarly.py
+++ b/pythainlp/spell/wanchanberta_thai_grammarly.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,7 +10,9 @@
* GitHub: \
https://github.com/bookpanda/Two-stage-Thai-Misspelling-Correction-Based-on-Pre-trained-Language-Models
"""
-from typing import List
+
+from __future__ import annotations
+
import torch
from transformers import (
AutoModelForMaskedLM,
@@ -21,28 +22,41 @@
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
-tokenizer = AutoTokenizer.from_pretrained("airesearch/wangchanberta-base-att-spm-uncased")
+tokenizer = AutoTokenizer.from_pretrained(
+ "airesearch/wangchanberta-base-att-spm-uncased"
+)
+
class BertModel(torch.nn.Module):
def __init__(self):
super().__init__()
- self.bert = BertForTokenClassification.from_pretrained('bookpanda/wangchanberta-base-att-spm-uncased-tagging')
+ self.bert = BertForTokenClassification.from_pretrained(
+ "bookpanda/wangchanberta-base-att-spm-uncased-tagging"
+ )
def forward(self, input_id, mask, label):
- output = self.bert(input_ids=input_id, attention_mask=mask, labels=label, return_dict=False)
+ output = self.bert(
+ input_ids=input_id,
+ attention_mask=mask,
+ labels=label,
+ return_dict=False,
+ )
return output
+
tagging_model = BertModel()
if use_cuda:
tagging_model = tagging_model.to(device=device)
-ids_to_labels = {0: 'f', 1: 'i'}
+ids_to_labels = {0: "f", 1: "i"}
-def align_word_ids(texts: str) -> List[int]:
- tokenized_inputs = tokenizer(texts, padding='max_length', max_length=512, truncation=True)
+
+def align_word_ids(texts: str) -> list[int]:
+ tokenized_inputs = tokenizer(
+ texts, padding="max_length", max_length=512, truncation=True
+ )
word_ids = tokenized_inputs.word_ids()
label_ids = []
for word_idx in word_ids:
-
if word_idx is None:
label_ids.append(-100)
else:
@@ -53,10 +67,17 @@ def align_word_ids(texts: str) -> List[int]:
return label_ids
+
def evaluate_one_text(model, sentence):
- text = tokenizer(sentence, padding='max_length', max_length = 512, truncation=True, return_tensors="pt")
- mask = text['attention_mask'][0].unsqueeze(0).to(device)
- input_id = text['input_ids'][0].unsqueeze(0).to(device)
+ text = tokenizer(
+ sentence,
+ padding="max_length",
+ max_length=512,
+ truncation=True,
+ return_tensors="pt",
+ )
+ mask = text["attention_mask"][0].unsqueeze(0).to(device)
+ input_id = text["input_ids"][0].unsqueeze(0).to(device)
label_ids = torch.Tensor(align_word_ids(sentence)).unsqueeze(0).to(device)
logits = tagging_model(input_id, mask, None)
@@ -67,39 +88,53 @@ def evaluate_one_text(model, sentence):
return prediction_label
-mlm_model = AutoModelForMaskedLM.from_pretrained("bookpanda/wangchanberta-base-att-spm-uncased-masking")
+mlm_model = AutoModelForMaskedLM.from_pretrained(
+ "bookpanda/wangchanberta-base-att-spm-uncased-masking"
+)
if use_cuda:
mlm_model = mlm_model.to(device=device)
+
def correct(text: str) -> str:
ans = []
i_f = evaluate_one_text(tagging_model, text)
a = tokenizer(text)
i_f_len = len(i_f)
for j in range(i_f_len):
- if i_f[j] == 'i':
- ph = a['input_ids'][j+1]
- a['input_ids'][j+1] = 25004
- b = {'input_ids': torch.Tensor([a['input_ids']]).type(torch.int64).to(device), 'attention_mask': torch.Tensor([a['attention_mask']]).type(torch.int64).to(device)}
+ if i_f[j] == "i":
+ ph = a["input_ids"][j + 1]
+ a["input_ids"][j + 1] = 25004
+ b = {
+ "input_ids": torch.Tensor([a["input_ids"]])
+ .type(torch.int64)
+ .to(device),
+ "attention_mask": torch.Tensor([a["attention_mask"]])
+ .type(torch.int64)
+ .to(device),
+ }
token_logits = mlm_model(**b).logits
- mask_token_index = torch.where(b["input_ids"] == tokenizer.mask_token_id)[1]
+ mask_token_index = torch.where(
+ b["input_ids"] == tokenizer.mask_token_id
+ )[1]
mask_token_logits = token_logits[0, mask_token_index, :]
- top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()
+ top_5_tokens = (
+ torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()
+ )
ans.append((j, top_5_tokens[0]))
- text = ''.join(tokenizer.convert_ids_to_tokens(a['input_ids']))
- a['input_ids'][j+1] = ph
- for x,y in ans:
- a['input_ids'][x+1] = y
- final_output = tokenizer.convert_ids_to_tokens(a['input_ids'])
+ text = "".join(tokenizer.convert_ids_to_tokens(a["input_ids"]))
+ a["input_ids"][j + 1] = ph
+ for x, y in ans:
+ a["input_ids"][x + 1] = y
+ final_output = tokenizer.convert_ids_to_tokens(a["input_ids"])
if "" in final_output:
final_output.remove("")
if "" in final_output:
final_output.remove("")
if "" in final_output:
final_output.remove("")
- if final_output[0] == '▁':
+ if final_output[0] == "▁":
final_output.pop(0)
- final_output = ''.join(final_output)
+ final_output = "".join(final_output)
final_output = final_output.replace("▁", " ")
final_output = final_output.replace("", "")
return final_output
diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py
index 10f0d7991..fa6a27ed6 100644
--- a/pythainlp/spell/words_spelling_correction.py
+++ b/pythainlp/spell/words_spelling_correction.py
@@ -1,24 +1,34 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import os
+
from pythainlp.corpus import get_hf_hub
-from typing import List, Union
class FastTextEncoder:
"""
- A class to load pre-trained FastText-like word embeddings,
- compute word and sentence vectors, and interact with an ONNX
+ A class to load pre-trained FastText-like word embeddings,
+ compute word and sentence vectors, and interact with an ONNX
model for nearest neighbor suggestions.
"""
# --- Initialization and Data Loading ---
-
- def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_words=2000000, minn=5, maxn=5):
+
+ def __init__(
+ self,
+ model_dir,
+ nn_model_path,
+ words_list,
+ bucket=2000000,
+ nb_words=2000000,
+ minn=5,
+ maxn=5,
+ ):
"""
- Initializes the FastTextEncoder, loading embeddings, vocabulary,
+ Initializes the FastTextEncoder, loading embeddings, vocabulary,
nearest neighbor model, and suggestion words list.
Args:
@@ -31,15 +41,16 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word
maxn (int): Maximum character length for subwords.
"""
try:
- import numpy as np # reduce load
+ import numpy as np # reduce load
import onnxruntime
+
self.np = np
except ModuleNotFoundError:
raise ModuleNotFoundError("""
Please installing the package via 'pip install numpy onnxruntime'.
""")
except Exception as e:
- raise Exception(f"An unexpected error occurred: {e}")
+ raise RuntimeError(f"An unexpected error occurred: {e}") from e
self.model_dir = model_dir
self.nn_model_path = nn_model_path
self.bucket = bucket
@@ -55,10 +66,12 @@ def __init__(self, model_dir, nn_model_path, words_list, bucket=2000000, nb_word
def _load_embeddings(self):
"""Loads embeddings matrix and vocabulary list."""
- input_matrix = self.np.load(os.path.join(self.model_dir, "embeddings.npy"))
+ input_matrix = self.np.load(
+ os.path.join(self.model_dir, "embeddings.npy")
+ )
words = []
vocab_path = os.path.join(self.model_dir, "vocabulary.txt")
- with open(vocab_path, "r", encoding='utf-8') as f:
+ with open(vocab_path, encoding="utf-8") as f:
for line in f.readlines():
words.append(line.rstrip())
return words, input_matrix
@@ -72,7 +85,10 @@ def _load_onnx_session(self, onnx_path):
"""Loads the ONNX inference session."""
# Note: Using providers=["CPUExecutionProvider"] for platform independence
import onnxruntime as rt
- sess = rt.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
+
+ sess = rt.InferenceSession(
+ onnx_path, providers=["CPUExecutionProvider"]
+ )
return sess
# --- Helper Methods for Encoding ---
@@ -103,9 +119,11 @@ def _get_subwords(self, word):
for ngram_start in range(0, len(_word)):
for ngram_length in range(self.minn, self.maxn + 1):
if ngram_start + ngram_length <= len(_word):
- _candidate_subword = _word[ngram_start:ngram_start + ngram_length]
+ _candidate_subword = _word[
+ ngram_start : ngram_start + ngram_length
+ ]
# Only append if not already included (e.g., as the full word)
- if _candidate_subword not in _subwords:
+ if _candidate_subword not in _subwords:
_subwords.append(_candidate_subword)
_subword_ids.append(self._get_hash(_candidate_subword))
@@ -115,20 +133,22 @@ def get_word_vector(self, word):
"""Computes the normalized vector for a single word."""
# subword_ids[1] contains the array of indices for the word and its subwords
subword_ids = self._get_subwords(word)[1]
-
+
# Check if the array of subword indices is empty
if subword_ids.size == 0:
# Return a 300-dimensional zero vector if no word/subword is found.
return self.np.zeros(self.embedding_dim)
# Compute the mean of the embeddings for all subword indices
- vector = self.np.mean([self.embeddings[s] for s in subword_ids], axis=0)
-
+ vector = self.np.mean(
+ [self.embeddings[s] for s in subword_ids], axis=0
+ )
+
# Normalize the vector
norm = self.np.linalg.norm(vector)
if norm > 0:
vector /= norm
-
+
return vector
def _tokenize(self, sentence):
@@ -136,11 +156,11 @@ def _tokenize(self, sentence):
tokens = []
word = ""
for c in sentence:
- if c in [' ', '\n', '\r', '\t', '\v', '\f', '\0']:
+ if c in [" ", "\n", "\r", "\t", "\v", "\f", "\0"]:
if word:
tokens.append(word)
word = ""
- if c == '\n':
+ if c == "\n":
tokens.append("")
else:
word += c
@@ -156,7 +176,7 @@ def get_sentence_vector(self, line):
# get_word_vector already handles normalization, so no need to do it again here
vec = self.get_word_vector(t)
vectors.append(vec)
-
+
# If the sentence was empty and resulted in no vectors, return a zero vector
if not vectors:
return self.np.zeros(self.embedding_dim)
@@ -167,15 +187,15 @@ def get_sentence_vector(self, line):
def get_word_suggestion(self, list_word):
"""
- Queries the ONNX model to find the nearest neighbor word(s)
+ Queries the ONNX model to find the nearest neighbor word(s)
for the given word or list of words.
Args:
- list_word (str or list of str): A single word or a list of words
+ list_word (str or list of str): A single word or a list of words
to get suggestions for.
Returns:
- str or list of str: The nearest neighbor word(s) from the
+ str or list of str: The nearest neighbor word(s) from the
pre-loaded suggestion list.
"""
if isinstance(list_word, str):
@@ -184,23 +204,26 @@ def get_word_suggestion(self, list_word):
else:
input_words = list_word
return_single = False
-
+
# Compute sentence vector for each input word/phrase
- # The original code's `get_sentence_vector(' '.join(list(word)))` seems
- # intended to treat a list of characters/tokens as a sentence.
- # I'll stick to a more standard usage: treat each item in `input_words`
+ # The original code's `get_sentence_vector(' '.join(list(word)))` seems
+ # intended to treat a list of characters/tokens as a sentence.
+ # I'll stick to a more standard usage: treat each item in `input_words`
# as a separate phrase/word to encode.
- word_input_vecs = [self.get_sentence_vector(' '.join(list(word))) for word in input_words]
+ word_input_vecs = [
+ self.get_sentence_vector(" ".join(list(word)))
+ for word in input_words
+ ]
# Convert to numpy array for ONNX input (ensure float32)
input_data = self.np.array(word_input_vecs, dtype=self.np.float32)
# Run ONNX inference
indices = self.nn_session.run(None, {"X": input_data})[0]
-
+
# Look up suggestions
suggestions = [self.words_for_suggestion[i].tolist() for i in indices]
-
+
return suggestions[0] if return_single else suggestions
@@ -209,7 +232,11 @@ def __init__(self):
self.model_name = "pythainlp/word-spelling-correction-char2vec"
self.model_path = get_hf_hub(self.model_name)
self.model_onnx = get_hf_hub(self.model_name, "nearest_neighbors.onnx")
- with open(get_hf_hub(self.model_name, "list_word-spelling-correction-char2vec.txt")) as f:
+ with open(
+ get_hf_hub(
+ self.model_name, "list_word-spelling-correction-char2vec.txt"
+ )
+ ) as f:
self.list_word = [i.strip() for i in f.readlines()]
super().__init__(self.model_path, self.model_onnx, self.list_word)
@@ -217,7 +244,9 @@ def __init__(self):
_WSC = None
-def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[str], List[List[str]]]:
+def get_words_spell_suggestion(
+ list_words: str | list[str],
+) -> list[str] | list[list[str]]:
"""
Get words spell suggestion
@@ -243,6 +272,6 @@ def get_words_spell_suggestion(list_words: Union[str, List[str]]) -> Union[List[
# ['กระเพาะ', 'กระพา', 'กะเพรา', 'กระเพาะปลา', 'พระประธาน']]
"""
global _WSC
- if _WSC==None:
+ if _WSC is None:
_WSC = Words_Spelling_Correction()
return _WSC.get_word_suggestion(list_words)
diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py
index 70c1e7e97..b73c8ffff 100644
--- a/pythainlp/summarize/__init__.py
+++ b/pythainlp/summarize/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py
index a72959a85..49cb824da 100644
--- a/pythainlp/summarize/core.py
+++ b/pythainlp/summarize/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,7 +5,9 @@
Text summarization and keyword extraction
"""
-from typing import Iterable, List, Optional, Tuple
+from __future__ import annotations
+
+from collections.abc import Iterable
from pythainlp.summarize import (
CPE_KMUTT_THAI_SENTENCE_SUM,
@@ -22,7 +23,7 @@ def summarize(
n: int = 1,
engine: str = DEFAULT_SUMMARIZE_ENGINE,
tokenizer: str = "newmm",
-) -> List[str]:
+) -> list[str]:
"""
This function summarizes text based on frequency of words.
@@ -119,13 +120,13 @@ def summarize(
def extract_keywords(
text: str,
- keyphrase_ngram_range: Tuple[int, int] = (1, 2),
+ keyphrase_ngram_range: tuple[int, int] = (1, 2),
max_keywords: int = 5,
min_df: int = 1,
engine: str = DEFAULT_KEYWORD_EXTRACTION_ENGINE,
tokenizer: str = "newmm",
- stop_words: Optional[Iterable[str]] = None,
-) -> List[str]:
+ stop_words: Iterable[str] | None = None,
+) -> list[str]:
"""
This function returns most-relevant keywords (and/or keyphrases) from the input document.
Each algorithm may produce completely different keywords from each other,
@@ -197,7 +198,7 @@ def rank_by_frequency(
max_keywords: int = 5,
min_df: int = 5,
tokenizer: str = "newmm",
- stop_words: Optional[Iterable[str]] = None,
+ stop_words: Iterable[str] | None = None,
):
from pythainlp.tokenize import word_tokenize
from pythainlp.util.keywords import rank
diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py
index 3adc828f2..93653274d 100644
--- a/pythainlp/summarize/freq.py
+++ b/pythainlp/summarize/freq.py
@@ -1,14 +1,15 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Summarization by frequency of words
"""
+
+from __future__ import annotations
+
from collections import defaultdict
from heapq import nlargest
from string import punctuation
-from typing import List
from pythainlp.corpus import thai_stopwords
from pythainlp.tokenize import sent_tokenize, word_tokenize
@@ -27,7 +28,7 @@ def __rank(ranking, n: int):
return nlargest(n, ranking, key=ranking.get)
def __compute_frequencies(
- self, word_tokenized_sents: List[List[str]]
+ self, word_tokenized_sents: list[list[str]]
) -> defaultdict:
word_freqs = defaultdict(int)
for sent in word_tokenized_sents:
@@ -48,7 +49,7 @@ def __compute_frequencies(
def summarize(
self, text: str, n: int, tokenizer: str = "newmm"
- ) -> List[str]:
+ ) -> list[str]:
sents = sent_tokenize(text, engine="whitespace+newline")
word_tokenized_sents = [
word_tokenize(sent, engine=tokenizer) for sent in sents
diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py
index acfd981c3..f589e1815 100644
--- a/pythainlp/summarize/keybert.py
+++ b/pythainlp/summarize/keybert.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,8 +10,11 @@
https://github.com/MaartenGr/KeyBERT
"""
+
+from __future__ import annotations
+
from collections import Counter
-from typing import Iterable, List, Optional, Tuple, Union
+from collections.abc import Iterable
import numpy as np
from transformers import pipeline
@@ -35,13 +37,13 @@ def __init__(
def extract_keywords(
self,
text: str,
- keyphrase_ngram_range: Tuple[int, int] = (1, 2),
+ keyphrase_ngram_range: tuple[int, int] = (1, 2),
max_keywords: int = 5,
min_df: int = 1,
tokenizer: str = "newmm",
return_similarity=False,
- stop_words: Optional[Iterable[str]] = None,
- ) -> Union[List[str], List[Tuple[str, float]]]:
+ stop_words: Iterable[str] | None = None,
+ ) -> list[str] | list[tuple[str, float]]:
"""
Extract Thai keywords and/or keyphrases with KeyBERT algorithm.
See https://github.com/MaartenGr/KeyBERT.
@@ -87,7 +89,9 @@ def extract_keywords(
# 'ควบคุมการเปลี่ยนแปลง',
# 'มีพิษ']
- keywords = kb.extract_keyword(text, max_keywords=10, return_similarity=True)
+ keywords = kb.extract_keyword(
+ text, max_keywords=10, return_similarity=True
+ )
# output: [('อวัยวะต่างๆ', 0.3228477063109462),
# ('ซ่อมแซมส่วน', 0.31320597838000375),
@@ -132,7 +136,7 @@ def extract_keywords(
else:
return [kw for kw, _ in keywords]
- def embed(self, docs: Union[str, List[str]]) -> np.ndarray:
+ def embed(self, docs: str | list[str]) -> np.ndarray:
"""
Create an embedding of each input in `docs` by averaging vectors from the last hidden layer.
"""
@@ -152,11 +156,11 @@ def embed(self, docs: Union[str, List[str]]) -> np.ndarray:
def _generate_ngrams(
doc: str,
- keyphrase_ngram_range: Tuple[int, int],
+ keyphrase_ngram_range: tuple[int, int],
min_df: int,
tokenizer_engine: str,
stop_words: Iterable[str],
-) -> List[str]:
+) -> list[str]:
assert keyphrase_ngram_range[0] >= 1, (
f"`keyphrase_ngram_range` must start from 1. "
f"current value={keyphrase_ngram_range}."
@@ -167,7 +171,7 @@ def _generate_ngrams(
f"current value={keyphrase_ngram_range}."
)
- def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]:
+ def _join_ngram(ngrams: list[tuple[str, str]]) -> list[str]:
ngrams_joined = []
for ng in ngrams:
joined = "".join(ng)
@@ -201,18 +205,18 @@ def _join_ngram(ngrams: List[Tuple[str, str]]) -> List[str]:
def _rank_keywords(
doc_vector: np.ndarray,
word_vectors: np.ndarray,
- keywords: List[str],
+ keywords: list[str],
max_keywords: int,
-) -> List[Tuple[str, float]]:
+) -> list[tuple[str, float]]:
def l2_norm(v: np.ndarray) -> np.ndarray:
vec_size = v.shape[1]
result = np.divide(
v,
np.linalg.norm(v, axis=1).reshape(-1, 1).repeat(vec_size, axis=1),
)
- assert np.isclose(
- np.linalg.norm(result, axis=1), 1
- ).all(), "Cannot normalize a vector to unit vector."
+ assert np.isclose(np.linalg.norm(result, axis=1), 1).all(), (
+ "Cannot normalize a vector to unit vector."
+ )
return result
def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray:
diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py
index bc5e572ac..cb0727804 100644
--- a/pythainlp/summarize/mt5.py
+++ b/pythainlp/summarize/mt5.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Summarization by mT5 model
"""
-from typing import List
+
+from __future__ import annotations
from transformers import MT5ForConditionalGeneration, T5Tokenizer
@@ -45,7 +45,7 @@ def __init__(
self.max_length = max_length
self.skip_special_tokens = skip_special_tokens
- def summarize(self, text: str) -> List[str]:
+ def summarize(self, text: str) -> list[str]:
preprocess_text = text.strip().replace("\n", "")
if self.model_name == f"thanathorn/{CPE_KMUTT_THAI_SENTENCE_SUM}":
t5_prepared_Text = "simplify: " + preprocess_text
diff --git a/pythainlp/tag/__init__.py b/pythainlp/tag/__init__.py
index c7ed6412f..891ddd8d2 100644
--- a/pythainlp/tag/__init__.py
+++ b/pythainlp/tag/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py
index 8a6a07701..bf1fdf685 100644
--- a/pythainlp/tag/_tag_perceptron.py
+++ b/pythainlp/tag/_tag_perceptron.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -17,12 +16,15 @@
This tagger is provided under the terms of the MIT License.
"""
+
+from __future__ import annotations
+
import json
from collections import defaultdict
-from typing import Dict, Iterable, List, Tuple, Union
+from collections.abc import Iterable
-class AveragedPerceptron():
+class AveragedPerceptron:
"""
An averaged perceptron, as implemented by Matthew Honnibal.
@@ -45,7 +47,7 @@ def __init__(self) -> None:
# Number of instances seen
self.i = 0
- def predict(self, features: Dict):
+ def predict(self, features: dict):
"""
Dot-product the features and current weights and return the best
label.
@@ -60,7 +62,7 @@ def predict(self, features: Dict):
# Do a secondary alphabetic sort, for stability
return max(self.classes, key=lambda label: (scores[label], label))
- def update(self, truth, guess, features: Dict) -> None:
+ def update(self, truth, guess, features: dict) -> None:
"""Update the feature weights."""
def upd_feat(c, f, w, v):
@@ -128,7 +130,7 @@ def __init__(self, path: str = "") -> None:
self.AP_MODEL_LOC = path
self.load(self.AP_MODEL_LOC)
- def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]:
+ def tag(self, tokens: Iterable[str]) -> list[tuple[str, str]]:
"""Tags a string `tokens`."""
prev, prev2 = self.START
output = []
@@ -146,8 +148,8 @@ def tag(self, tokens: Iterable[str]) -> List[Tuple[str, str]]:
def train(
self,
- sentences: Iterable[Iterable[Tuple[str, str]]],
- save_loc: Union[str, None] = None,
+ sentences: Iterable[Iterable[tuple[str, str]]],
+ save_loc: str | None = None,
nr_iter: int = 5,
) -> None:
"""
@@ -203,11 +205,11 @@ def load(self, loc: str) -> None:
:param str loc: model path
"""
try:
- with open(loc, "r", encoding="utf-8-sig") as f:
+ with open(loc, encoding="utf-8-sig") as f:
w_td_c = json.load(f)
- except IOError:
+ except OSError:
msg = "Missing trontagger.json file."
- raise IOError(msg)
+ raise OSError(msg)
self.model.weights = w_td_c["weights"]
self.tagdict = w_td_c["tagdict"]
self.classes = w_td_c["classes"]
@@ -233,8 +235,8 @@ def _normalize(self, word: str) -> str:
return word.lower()
def _get_features(
- self, i: int, word: str, context: List[str], prev: str, prev2: str
- ) -> Dict:
+ self, i: int, word: str, context: list[str], prev: str, prev2: str
+ ) -> dict:
"""
Map tokens into a feature representation, implemented as a
{hashable: float} dict. If the features change, a new model must be
@@ -265,7 +267,7 @@ def add(name: str, *args):
return features
def _make_tagdict(
- self, sentences: Iterable[Iterable[Tuple[str, str]]]
+ self, sentences: Iterable[Iterable[tuple[str, str]]]
) -> None:
"""Make a tag dictionary for single-tag words."""
counts = defaultdict(lambda: defaultdict(int))
diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py
index afa771087..a4b4b8ee9 100644
--- a/pythainlp/tag/blackboard.py
+++ b/pythainlp/tag/blackboard.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
# defined strings for special characters
CHAR_TO_ESCAPE = {" ": "_"}
@@ -32,7 +31,7 @@
}
-def pre_process(words: List[str]) -> List[str]:
+def pre_process(words: list[str]) -> list[str]:
"""
Convert signs and symbols with their defined strings.
This function is to be used as a preprocessing step,
@@ -44,8 +43,8 @@ def pre_process(words: List[str]) -> List[str]:
def post_process(
- word_tags: List[Tuple[str, str]], to_ud: bool = False
-) -> List[Tuple[str, str]]:
+ word_tags: list[tuple[str, str]], to_ud: bool = False
+) -> list[tuple[str, str]]:
"""
Convert defined strings back to corresponding signs and symbols.
This function is to be used as a post-processing step,
diff --git a/pythainlp/tag/chunk.py b/pythainlp/tag/chunk.py
index cd5722b7f..c950636b0 100644
--- a/pythainlp/tag/chunk.py
+++ b/pythainlp/tag/chunk.py
@@ -1,13 +1,12 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
def chunk_parse(
- sent: List[Tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp"
-) -> List[str]:
+ sent: list[tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp"
+) -> list[str]:
"""
This function parses Thai sentence to phrase structure in IOB format.
diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py
index f8dfbc950..c04a8f52a 100644
--- a/pythainlp/tag/crfchunk.py
+++ b/pythainlp/tag/crfchunk.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import Dict, List, Tuple
+from __future__ import annotations
from pycrfsuite import Tagger as CRFTagger
@@ -13,7 +12,7 @@ def _is_stopword(word: str) -> bool: # check Thai stopword
return word in thai_stopwords()
-def _doc2features(tokens: List[Tuple[str, str]], index: int) -> Dict:
+def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict:
"""
`tokens` = a POS-tagged sentence [(w1, t1), ...]
`index` = the index of the token we want to extract features for
@@ -67,6 +66,6 @@ def load_model(self, corpus: str):
self.path = path_pythainlp_corpus("crfchunk_orchidpp.model")
self.tagger.open(self.path)
- def parse(self, token_pos: List[Tuple[str, str]]) -> List[str]:
+ def parse(self, token_pos: list[tuple[str, str]]) -> list[str]:
self.xseq = extract_features(token_pos)
return self.tagger.tag(self.xseq)
diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py
index b70072ba5..f4753e54a 100644
--- a/pythainlp/tag/locations.py
+++ b/pythainlp/tag/locations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,12 +5,12 @@
Recognizes locations in text
"""
-from typing import List, Tuple
+from __future__ import annotations
from pythainlp.corpus import provinces
-def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]:
+def tag_provinces(tokens: list[str]) -> list[tuple[str, str]]:
"""
This function recognizes Thailand provinces in text.
@@ -26,7 +25,7 @@ def tag_provinces(tokens: List[str]) -> List[Tuple[str, str]]:
from pythainlp.tag import tag_provinces
- text = ['หนองคาย', 'น่าอยู่']
+ text = ["หนองคาย", "น่าอยู่"]
tag_provinces(text)
# output: [('หนองคาย', 'B-LOCATION'), ('น่าอยู่', 'O')]
"""
diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py
index e9628a7b4..4639dafbb 100644
--- a/pythainlp/tag/named_entity.py
+++ b/pythainlp/tag/named_entity.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Named-entity recognizer
"""
-from typing import List, Tuple, Union
+
+from __future__ import annotations
class NER:
@@ -26,7 +26,9 @@ class NER:
**Note**: The tltk engine supports NER models from tltk only.
"""
- def __init__(self, engine: str = "thainer-v2", corpus: str = "thainer") -> None:
+ def __init__(
+ self, engine: str = "thainer-v2", corpus: str = "thainer"
+ ) -> None:
self.load_engine(engine=engine, corpus=corpus)
def load_engine(self, engine: str, corpus: str) -> None:
@@ -38,7 +40,10 @@ def load_engine(self, engine: str, corpus: str) -> None:
self.engine = ThaiNameTagger()
elif engine == "thainer-v2" and corpus == "thainer":
from pythainlp.wangchanberta import NamedEntityRecognition
- self.engine = NamedEntityRecognition(model="pythainlp/thainer-corpus-v2-base-model")
+
+ self.engine = NamedEntityRecognition(
+ model="pythainlp/thainer-corpus-v2-base-model"
+ )
elif engine == "tltk":
from pythainlp.tag import tltk
@@ -53,16 +58,12 @@ def load_engine(self, engine: str, corpus: str) -> None:
self.engine = NamedEntityTagger()
else:
raise ValueError(
- "NER class not support {0} engine or {1} corpus.".format(
- engine, corpus
- )
+ f"NER class not support {engine} engine or {corpus} corpus."
)
- def tag(self,
- text,
- pos=False,
- tag=False
- ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]:
+ def tag(
+ self, text, pos=False, tag=False
+ ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str:
"""
This function tags named entities in text in IOB format.
@@ -116,7 +117,7 @@ def load_engine(self, engine: str = "thai_nner") -> None:
self.engine = Thai_NNER()
- def tag(self, text) -> Tuple[List[str], List[dict]]:
+ def tag(self, text) -> tuple[list[str], list[dict]]:
"""
This function tags nested named entities.
diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py
index 678c7a37e..46e0f1005 100644
--- a/pythainlp/tag/orchid.py
+++ b/pythainlp/tag/orchid.py
@@ -1,11 +1,11 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Data preprocessing for ORCHID corpus
"""
-from typing import List, Tuple
+
+from __future__ import annotations
# defined strings for special characters,
# from Table 4 in ORCHID paper
@@ -126,7 +126,7 @@ def ud_exception(w: str, tag: str) -> str:
return tag
-def pre_process(words: List[str]) -> List[str]:
+def pre_process(words: list[str]) -> list[str]:
"""
Convert signs and symbols with their defined strings.
This function is to be used as a preprocessing step,
@@ -138,8 +138,8 @@ def pre_process(words: List[str]) -> List[str]:
def post_process(
- word_tags: List[Tuple[str, str]], to_ud: bool = False
-) -> List[Tuple[str, str]]:
+ word_tags: list[tuple[str, str]], to_ud: bool = False
+) -> list[tuple[str, str]]:
"""
Convert defined strings back to corresponding signs and symbols.
This function is to be used as a post-processing step,
diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py
index 71e9b720e..2219fdd50 100644
--- a/pythainlp/tag/perceptron.py
+++ b/pythainlp/tag/perceptron.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,8 +5,9 @@
Perceptron part-of-speech tagger
"""
+from __future__ import annotations
+
import os
-from typing import List, Tuple
from pythainlp.corpus import corpus_path, get_corpus_path
from pythainlp.tag import PerceptronTagger, blackboard, orchid
@@ -69,7 +69,7 @@ def _tud_tagger():
return _TUD_TAGGER
-def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
+def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]:
"""
:param list words: a list of tokenized words
:param str corpus: corpus name (orchid, pud)
diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py
index a83a3330f..eb1d8e385 100644
--- a/pythainlp/tag/pos_tag.py
+++ b/pythainlp/tag/pos_tag.py
@@ -1,13 +1,12 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
def pos_tag(
- words: List[str], engine: str = "perceptron", corpus: str = "orchid"
-) -> List[Tuple[str, str]]:
+ words: list[str], engine: str = "perceptron", corpus: str = "orchid"
+) -> list[tuple[str, str]]:
"""
Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'.
@@ -117,9 +116,7 @@ def pos_tag(
from pythainlp.tag.unigram import tag as tag_
else:
raise ValueError(
- "pos_tag not support {0} engine or {1} corpus.".format(
- engine, corpus
- )
+ f"pos_tag not support {engine} engine or {corpus} corpus."
)
word_tags = tag_(words, corpus=corpus)
@@ -128,10 +125,10 @@ def pos_tag(
def pos_tag_sents(
- sentences: List[List[str]],
+ sentences: list[list[str]],
engine: str = "perceptron",
corpus: str = "orchid",
-) -> List[List[Tuple[str, str]]]:
+) -> list[list[tuple[str, str]]]:
"""
Marks sentences with part-of-speech (POS) tags.
@@ -180,7 +177,7 @@ def pos_tag_transformers(
sentence: str,
engine: str = "bert",
corpus: str = "blackboard",
-) -> List[List[Tuple[str, str]]]:
+) -> list[list[tuple[str, str]]]:
"""
Marks sentences with part-of-speech (POS) tags.
@@ -245,15 +242,14 @@ def pos_tag_transformers(
tokenizer = AutoTokenizer.from_pretrained(base_model)
else:
raise ValueError(
- "pos_tag_transformers not support {0} engine or {1} corpus.".format(
- engine, corpus
- )
+ f"pos_tag_transformers not support {engine} engine or {corpus} corpus."
)
- pipeline = TokenClassificationPipeline(model=model,
- tokenizer=tokenizer,
- aggregation_strategy="simple",
- )
+ pipeline = TokenClassificationPipeline(
+ model=model,
+ tokenizer=tokenizer,
+ aggregation_strategy="simple",
+ )
outputs = pipeline(sentence)
word_tags = [[(tag["word"], tag["entity_group"]) for tag in outputs]]
diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py
index ad0a2623a..09432e7f1 100644
--- a/pythainlp/tag/thai_nner.py
+++ b/pythainlp/tag/thai_nner.py
@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
+from __future__ import annotations
from thai_nner import NNER
@@ -13,5 +12,5 @@ class Thai_NNER:
def __init__(self, path_model=get_corpus_path("thai_nner", "1.0")) -> None:
self.model = NNER(path_model=path_model)
- def tag(self, text) -> Tuple[List[str], List[dict]]:
+ def tag(self, text) -> tuple[list[str], list[dict]]:
return self.model.get_tag(text)
diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py
index 8558eef67..c432679d2 100644
--- a/pythainlp/tag/thainer.py
+++ b/pythainlp/tag/thainer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,9 +5,10 @@
Named-entity recognizer
"""
+from __future__ import annotations
+
__all__ = ["ThaiNameTagger"]
-from typing import Dict, List, Tuple, Union
from pythainlp.corpus import get_corpus_path, thai_stopwords
from pythainlp.tag import pos_tag
@@ -22,7 +22,7 @@ def _is_stopword(word: str) -> bool: # เช็คว่าเป็นคำ
return word in thai_stopwords()
-def _doc2features(doc, i) -> Dict:
+def _doc2features(doc, i) -> dict:
word = doc[i][0]
postag = doc[i][1]
@@ -111,7 +111,7 @@ def __init__(self, version: str = "1.4") -> None:
def get_ner(
self, text: str, pos: bool = True, tag: bool = False
- ) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]]]:
+ ) -> list[tuple[str, str]] | list[tuple[str, str, str]]:
"""
This function tags named-entities in text in IOB format.
diff --git a/pythainlp/tag/tltk.py b/pythainlp/tag/tltk.py
index f957a1832..fdb65180d 100644
--- a/pythainlp/tag/tltk.py
+++ b/pythainlp/tag/tltk.py
@@ -1,22 +1,23 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple, Union
+from __future__ import annotations
try:
from tltk import nlp
except ImportError:
- raise ImportError("Not found tltk! Please install tltk by pip install tltk")
+ raise ImportError(
+ "Not found tltk! Please install tltk by pip install tltk"
+ )
from pythainlp.tokenize import word_tokenize
nlp.pos_load()
nlp.ner_load()
-def pos_tag(words: List[str], corpus: str = "tnc") -> List[Tuple[str, str]]:
+def pos_tag(words: list[str], corpus: str = "tnc") -> list[tuple[str, str]]:
if corpus != "tnc":
- raise ValueError("tltk not support {0} corpus.".format(0))
+ raise ValueError(f"tltk not support {0} corpus.")
return nlp.pos_tag_wordlist(words)
@@ -26,7 +27,7 @@ def _post_process(text: str) -> str:
def get_ner(
text: str, pos: bool = True, tag: bool = False
-) -> Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str]:
+) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str:
"""
Named-entity recognizer from **TLTK**
diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py
index 34384072b..23537eeec 100644
--- a/pythainlp/tag/unigram.py
+++ b/pythainlp/tag/unigram.py
@@ -1,13 +1,14 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Unigram Part-Of-Speech tagger
"""
+
+from __future__ import annotations
+
import json
import os
-from typing import List, Tuple
from pythainlp.corpus import corpus_path, get_corpus_path
from pythainlp.tag import blackboard, orchid
@@ -75,8 +76,8 @@ def _tud_tagger():
def _find_tag(
- words: List[str], dictdata: dict, default_tag: str = ""
-) -> List[Tuple[str, str]]:
+ words: list[str], dictdata: dict, default_tag: str = ""
+) -> list[tuple[str, str]]:
keys = list(dictdata.keys())
return [
(word, dictdata[word]) if word in keys else (word, default_tag)
@@ -84,7 +85,7 @@ def _find_tag(
]
-def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
+def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]:
"""
:param list words: a list of tokenized words
:param str corpus: corpus name (orchid or pud)
diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py
index c3744d1a1..ccc12f15a 100644
--- a/pythainlp/tag/wangchanberta_onnx.py
+++ b/pythainlp/tag/wangchanberta_onnx.py
@@ -1,9 +1,9 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
import json
-from typing import List
import numpy as np
@@ -16,7 +16,7 @@ def __init__(
model_name: str,
model_version: str,
file_onnx: str,
- providers: List[str] = ["CPUExecutionProvider"],
+ providers: list[str] = ["CPUExecutionProvider"],
) -> None:
import sentencepiece as spm
from onnxruntime import (
diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py
index 8c64cecc0..865bc0aa9 100644
--- a/pythainlp/tokenize/__init__.py
+++ b/pythainlp/tokenize/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -20,6 +19,7 @@
]
from functools import lru_cache
+
from pythainlp.corpus import thai_syllables, thai_words
from pythainlp.util.trie import Trie
@@ -28,24 +28,27 @@
DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc"
DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo"
+
@lru_cache
def word_dict_trie():
"""Lazy load default word dict trie with cache"""
return Trie(thai_words())
+
@lru_cache
def syllable_dict_trie():
"""Lazy load default syllable dict trie with cache"""
return Trie(thai_syllables())
+
from pythainlp.tokenize.core import (
Tokenizer,
+ display_cell_tokenize,
paragraph_tokenize,
sent_tokenize,
subword_tokenize,
syllable_tokenize,
word_detokenize,
word_tokenize,
- display_cell_tokenize,
)
from pythainlp.tokenize.thai2fit import thai2fit_tokenizer
diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py
index 463b45fb0..8df8f53c0 100644
--- a/pythainlp/tokenize/_utils.py
+++ b/pythainlp/tokenize/_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,15 +5,17 @@
Utility functions for tokenize module.
"""
+from __future__ import annotations
+
import re
-from typing import Callable, List
+from collections.abc import Callable
_DIGITS_WITH_SEPARATOR = re.compile(r"(\d+[\.\,:])+\d+")
def apply_postprocessors(
- segments: List[str], postprocessors: Callable[[List[str]], List[str]]
-) -> List[str]:
+ segments: list[str], postprocessors: Callable[[list[str]], list[str]]
+) -> list[str]:
"""
A list of callables to apply to a raw segmentation result.
"""
@@ -24,7 +25,7 @@ def apply_postprocessors(
return segments
-def rejoin_formatted_num(segments: List[str]) -> List[str]:
+def rejoin_formatted_num(segments: list[str]) -> list[str]:
"""
Rejoin well-known formatted numeric that are over-tokenized.
The formatted numeric are numbers separated by ":", ",", or ".",
@@ -73,7 +74,7 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]:
return tokens_joined
-def strip_whitespace(segments: List[str]) -> List[str]:
+def strip_whitespace(segments: list[str]) -> list[str]:
"""
Strip whitespace(s) off each token and remove whitespace tokens.
:param List[str] segments: result from word tokenizer
diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py
index e5c59c44f..169c750cd 100644
--- a/pythainlp/tokenize/attacut.py
+++ b/pythainlp/tokenize/attacut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -8,7 +7,8 @@
:See Also:
* `GitHub repository `_
"""
-from typing import Dict, List
+
+from __future__ import annotations
from attacut import Tokenizer
@@ -22,14 +22,14 @@ def __init__(self, model="attacut-sc"):
self._tokenizer = Tokenizer(model=self._MODEL_NAME)
- def tokenize(self, text: str) -> List[str]:
+ def tokenize(self, text: str) -> list[str]:
return self._tokenizer.tokenize(text)
-_tokenizers: Dict[str, AttacutTokenizer] = {}
+_tokenizers: dict[str, AttacutTokenizer] = {}
-def segment(text: str, model: str = "attacut-sc") -> List[str]:
+def segment(text: str, model: str = "attacut-sc") -> list[str]:
"""
Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai
:param str text: text to be tokenized to words
diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py
index 5237da2a9..74e46b9c0 100644
--- a/pythainlp/tokenize/budoux.py
+++ b/pythainlp/tokenize/budoux.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,7 +10,8 @@
used and `budoux` is missing, a clear ImportError is raised with an
installation hint.
"""
-from typing import List
+
+from __future__ import annotations
_parser = None
@@ -32,7 +32,7 @@ def _init_parser():
return budoux.load_default_thai_parser()
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""Segment `text` into tokens using budoux.
The function returns a list of strings. If `budoux` is not available
diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py
index a6cdaf737..234be457e 100644
--- a/pythainlp/tokenize/core.py
+++ b/pythainlp/tokenize/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -6,17 +5,19 @@
Generic functions of tokenizers
"""
+from __future__ import annotations
+
import copy
import re
-from typing import Iterable, List, Optional, Union
+from collections.abc import Iterable
from pythainlp.tokenize import (
DEFAULT_SENT_TOKENIZE_ENGINE,
DEFAULT_SUBWORD_TOKENIZE_ENGINE,
- syllable_dict_trie,
DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
- word_dict_trie,
DEFAULT_WORD_TOKENIZE_ENGINE,
+ syllable_dict_trie,
+ word_dict_trie,
)
from pythainlp.tokenize._utils import (
apply_postprocessors,
@@ -27,8 +28,8 @@
def word_detokenize(
- segments: Union[List[List[str]], List[str]], output: str = "str"
-) -> Union[List[str], str]:
+ segments: list[list[str]] | list[str], output: str = "str"
+) -> list[str] | str:
"""
Word detokenizer.
@@ -97,11 +98,11 @@ def word_detokenize(
def word_tokenize(
text: str,
- custom_dict: Optional[Trie] = None,
+ custom_dict: Trie | None = None,
engine: str = DEFAULT_WORD_TOKENIZE_ENGINE,
keep_whitespace: bool = True,
join_broken_num: bool = True,
-) -> List[str]:
+) -> list[str]:
"""
Word tokenizer.
@@ -361,10 +362,10 @@ def map_indices_to_words(index_list, sentences):
def sent_tokenize(
- text: Union[str, List[str]],
+ text: str | list[str],
engine: str = DEFAULT_SENT_TOKENIZE_ENGINE,
keep_whitespace: bool = True,
-) -> List[str]:
+) -> list[str]:
"""
Sentence tokenizer.
@@ -532,7 +533,7 @@ def paragraph_tokenize(
engine: str = "wtp-mini",
paragraph_threshold: float = 0.5,
style: str = "newline",
-) -> List[List[str]]:
+) -> list[list[str]]:
"""
Paragraph tokenizer.
@@ -600,7 +601,7 @@ def subword_tokenize(
text: str,
engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE,
keep_whitespace: bool = True,
-) -> List[str]:
+) -> list[str]:
"""
Subword tokenizer for tokenizing text into units smaller than syllables.
@@ -692,9 +693,7 @@ def subword_tokenize(
words = word_tokenize(text)
for word in words:
segments.extend(
- word_tokenize(
- text=word, custom_dict=syllable_dict_trie()
- )
+ word_tokenize(text=word, custom_dict=syllable_dict_trie())
)
elif engine == "ssg":
from pythainlp.tokenize.ssg import segment
@@ -723,7 +722,7 @@ def syllable_tokenize(
text: str,
engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
keep_whitespace: bool = True,
-) -> List[str]:
+) -> list[str]:
"""
Syllable tokenizer
@@ -755,7 +754,7 @@ def syllable_tokenize(
)
-def display_cell_tokenize(text: str) -> List[str]:
+def display_cell_tokenize(text: str) -> list[str]:
"""
Display cell tokenizer.
@@ -864,7 +863,7 @@ class Tokenizer:
def __init__(
self,
- custom_dict: Union[Trie, Iterable[str], str] = [],
+ custom_dict: Trie | Iterable[str] | str = [],
engine: str = "newmm",
keep_whitespace: bool = True,
join_broken_num: bool = True,
@@ -896,7 +895,7 @@ def __init__(
self.__keep_whitespace = keep_whitespace
self.__join_broken_num = join_broken_num
- def word_tokenize(self, text: str) -> List[str]:
+ def word_tokenize(self, text: str) -> list[str]:
"""
Main tokenization function.
diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py
index 9746d0113..19bf255e6 100644
--- a/pythainlp/tokenize/crfcut.py
+++ b/pythainlp/tokenize/crfcut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -17,8 +16,9 @@
POS features are not used due to unreliable POS tagging available
"""
+from __future__ import annotations
+
import os
-from typing import List
import pycrfsuite
@@ -128,8 +128,8 @@
def extract_features(
- doc: List[str], window: int = 2, max_n_gram: int = 3
-) -> List[List[str]]:
+ doc: list[str], window: int = 2, max_n_gram: int = 3
+) -> list[list[str]]:
"""
Extract features for CRF by sliding `max_n_gram` of tokens
for +/- `window` from the current token
@@ -168,12 +168,12 @@ def extract_features(
# ngram features
for n_gram in range(1, min(max_n_gram + 1, 2 + window * 2)):
for j in range(i - window, i + window + 2 - n_gram):
- feature_position = f"{n_gram}_{j-i}_{j-i+n_gram}"
- word_ = f'{"|".join(doc[j:(j+n_gram)])}'
+ feature_position = f"{n_gram}_{j - i}_{j - i + n_gram}"
+ word_ = f"{'|'.join(doc[j : (j + n_gram)])}"
word_features += [f"word_{feature_position}={word_}"]
- ender_ = f'{"|".join(doc_ender[j:(j+n_gram)])}'
+ ender_ = f"{'|'.join(doc_ender[j : (j + n_gram)])}"
word_features += [f"ender_{feature_position}={ender_}"]
- starter_ = f'{"|".join(doc_starter[j:(j+n_gram)])}'
+ starter_ = f"{'|'.join(doc_starter[j : (j + n_gram)])}"
word_features += [f"starter_{feature_position}={starter_}"]
# append to feature per word
doc_features.append(word_features)
@@ -186,7 +186,7 @@ def extract_features(
_tagger.open(os.path.join(corpus_path(), _CRFCUT_DATA_FILENAME))
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
CRF-based sentence segmentation.
@@ -206,7 +206,7 @@ def segment(text: str) -> List[str]:
if toks[idx].strip().endswith(("!", ".", "?")):
labs[idx] = "E"
# Spaces or empty strings would no longer be treated as end of sentence.
- elif (idx == 0 or labs[idx-1] == "E") and toks[idx].strip() == "":
+ elif (idx == 0 or labs[idx - 1] == "E") and toks[idx].strip() == "":
labs[idx] = "I"
sentences = []
diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py
index 4b008de40..0087fa10f 100644
--- a/pythainlp/tokenize/deepcut.py
+++ b/pythainlp/tokenize/deepcut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -12,7 +11,7 @@
* `GitHub repository `_
"""
-from typing import List, Union
+from __future__ import annotations
try:
from deepcut import tokenize
@@ -21,9 +20,7 @@
from pythainlp.util import Trie
-def segment(
- text: str, custom_dict: Union[Trie, List[str], str] = []
-) -> List[str]:
+def segment(text: str, custom_dict: Trie | list[str] | str = []) -> list[str]:
if not text or not isinstance(text, str):
return []
diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py
index 557be3cef..cb0563ab1 100644
--- a/pythainlp/tokenize/etcc.py
+++ b/pythainlp/tokenize/etcc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -19,24 +18,28 @@
and backward longest matching techniques." In International Symposium on
Communications and Information Technology (ISCIT), pp. 37-40. 2001.
"""
-from functools import lru_cache
+
+from __future__ import annotations
+
import re
-from typing import List
+from functools import lru_cache
from pythainlp import thai_follow_vowels
from pythainlp.corpus import get_corpus
from pythainlp.tokenize import Tokenizer
+
@lru_cache
def _cut_etcc():
"""Lazy load ETCC tokenizer with cache"""
return Tokenizer(get_corpus("etcc.txt"), engine="longest")
+
_PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]"
_RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR)
-def _cut_subword(tokens: List[str]) -> List[str]:
+def _cut_subword(tokens: list[str]) -> list[str]:
len_tokens = len(tokens)
i = 0
while True:
@@ -50,7 +53,7 @@ def _cut_subword(tokens: List[str]) -> List[str]:
return tokens
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
Segmenting text into ETCCs.
diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py
index 30d8180bc..eacfb1494 100644
--- a/pythainlp/tokenize/han_solo.py
+++ b/pythainlp/tokenize/han_solo.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileCopyrightText: Copyright 2019 Ponrawee Prasertsom
# SPDX-License-Identifier: Apache-2.0
@@ -7,7 +6,8 @@
GitHub: https://github.com/PyThaiNLP/Han-solo
"""
-from typing import List
+
+from __future__ import annotations
from pythainlp.corpus import path_pythainlp_corpus
@@ -119,7 +119,7 @@ def featurize(
_to_feature = Featurizer()
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
x = _to_feature.featurize(text)["X"]
y_pred = tagger.tag(x)
list_cut = []
diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py
index 969f47271..b777ca126 100644
--- a/pythainlp/tokenize/longest.py
+++ b/pythainlp/tokenize/longest.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,8 +10,10 @@
`_
"""
+
+from __future__ import annotations
+
import re
-from typing import Dict, List, Optional, Union
from pythainlp import thai_tonemarks
from pythainlp.tokenize import word_dict_trie
@@ -48,7 +49,7 @@ def __init__(self, trie: Trie):
self.__trie = trie
@staticmethod
- def __search_nonthai(text: str) -> Union[None, str]:
+ def __search_nonthai(text: str) -> None | str:
match = _RE_NONTHAI.search(text)
if match.group(0):
return match.group(0).lower()
@@ -137,22 +138,26 @@ def __segment(self, text: str):
# Group consecutive spaces into one token
grouped_tokens = []
for token in tokens:
- if token.isspace() and grouped_tokens and grouped_tokens[-1].isspace():
+ if (
+ token.isspace()
+ and grouped_tokens
+ and grouped_tokens[-1].isspace()
+ ):
grouped_tokens[-1] += token
else:
grouped_tokens.append(token)
return grouped_tokens
- def tokenize(self, text: str) -> List[str]:
+ def tokenize(self, text: str) -> list[str]:
tokens = self.__segment(text)
return tokens
-_tokenizers: Dict[int, LongestMatchTokenizer] = {}
+_tokenizers: dict[int, LongestMatchTokenizer] = {}
-def segment(text: str, custom_dict: Optional[Trie] = None) -> List[str]:
+def segment(text: str, custom_dict: Trie | None = None) -> list[str]:
"""
Dictionary-based longest matching word segmentation.
diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py
index 2ac0b0b49..d6f9c7840 100644
--- a/pythainlp/tokenize/multi_cut.py
+++ b/pythainlp/tokenize/multi_cut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -13,9 +12,11 @@
`_
"""
+from __future__ import annotations
+
import re
from collections import defaultdict
-from typing import Iterator, List, Optional
+from collections.abc import Iterator
from pythainlp.tokenize import word_dict_trie
from pythainlp.util import Trie
@@ -48,7 +49,7 @@ def __init__(self, value, multi=None, in_dict=True):
def _multicut(
- text: str, custom_dict: Optional[Trie] = None
+ text: str, custom_dict: Trie | None = None
) -> Iterator[LatticeString]:
"""Return LatticeString"""
if not custom_dict:
@@ -100,7 +101,7 @@ def serialize(p, p2): # helper function
q.add(i)
-def mmcut(text: str) -> List[str]:
+def mmcut(text: str) -> list[str]:
res = []
for w in _multicut(text):
mm = min(w.multi, key=lambda x: x.count("/"))
@@ -108,7 +109,7 @@ def mmcut(text: str) -> List[str]:
return res
-def _combine(ww: List[LatticeString]) -> Iterator[str]:
+def _combine(ww: list[LatticeString]) -> Iterator[str]:
if ww == []:
yield ""
else:
@@ -121,9 +122,7 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]:
yield m.replace("/", "|") + "|" + tail
-def segment(
- text: str, custom_dict: Optional[Trie] = None
-) -> List[str]:
+def segment(text: str, custom_dict: Trie | None = None) -> list[str]:
"""Dictionary-based maximum matching word segmentation.
:param text: text to be tokenized
@@ -143,9 +142,7 @@ def segment(
return list(_multicut(text, custom_dict=custom_dict))
-def find_all_segment(
- text: str, custom_dict: Optional[Trie] = None
-) -> List[str]:
+def find_all_segment(text: str, custom_dict: Trie | None = None) -> list[str]:
"""Get all possible segment variations.
:param text: input string to be tokenized
diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py
index 1ac0fa8ab..75e42c17f 100644
--- a/pythainlp/tokenize/nercut.py
+++ b/pythainlp/tokenize/nercut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -11,7 +10,10 @@
Code by Wannaphong Phatthiyaphaibun
"""
-from typing import Iterable, List
+
+from __future__ import annotations
+
+from collections.abc import Iterable
from pythainlp.tag.named_entity import NER
@@ -29,7 +31,7 @@ def segment(
"TIME",
],
tagger=_thainer,
-) -> List[str]:
+) -> list[str]:
"""
Dictionary-based maximal matching word segmentation, constrained by
Thai Character Cluster (TCC) boundaries, and combining tokens that are
diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py
index ec30356e5..58dd70080 100644
--- a/pythainlp/tokenize/newmm.py
+++ b/pythainlp/tokenize/newmm.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -15,10 +14,13 @@
* \
https://colab.research.google.com/drive/14Ibg-ngZXj15RKwjNwoZlOT32fQBOrBx#scrollTo=MYZ7NzAR7Dmw
"""
+
+from __future__ import annotations
+
import re
from collections import defaultdict
+from collections.abc import Generator
from heapq import heappop, heappush
-from typing import Generator, List, Optional
from pythainlp.tokenize import word_dict_trie
from pythainlp.tokenize.tcc_p import tcc_pos
@@ -57,7 +59,7 @@
def _bfs_paths_graph(
graph: defaultdict, start: int, goal: int
-) -> Generator[List[int], None, None]:
+) -> Generator[list[int], None, None]:
queue = [(start, [start])]
while queue:
(vertex, path) = queue.pop(0)
@@ -140,9 +142,9 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]:
def segment(
text: str,
- custom_dict: Optional[Trie] = None,
+ custom_dict: Trie | None = None,
safe_mode: bool = False,
-) -> List[str]:
+) -> list[str]:
"""Maximal-matching word segmentation constrained by Thai Character Cluster.
A dictionary-based word segmentation using maximal matching algorithm,
diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py
index c29dc0874..310928ee1 100644
--- a/pythainlp/tokenize/nlpo3.py
+++ b/pythainlp/tokenize/nlpo3.py
@@ -1,9 +1,9 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
from sys import stderr
-from typing import List
from nlpo3 import load_dict as nlpo3_load_dict
from nlpo3 import segment as nlpo3_segment
@@ -45,7 +45,7 @@ def segment(
custom_dict: str = _NLPO3_DEFAULT_DICT_NAME,
safe_mode: bool = False,
parallel_mode: bool = False,
-) -> List[str]:
+) -> list[str]:
"""Break text into tokens.
Python binding for nlpO3. It is newmm engine in Rust.
diff --git a/pythainlp/tokenize/oskut.py b/pythainlp/tokenize/oskut.py
index 3c6037a15..58cb96344 100644
--- a/pythainlp/tokenize/oskut.py
+++ b/pythainlp/tokenize/oskut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -10,7 +9,8 @@
:See Also:
* `GitHub repository `_
"""
-from typing import List
+
+from __future__ import annotations
import oskut
@@ -18,7 +18,7 @@
oskut.load_model(engine=DEFAULT_ENGINE)
-def segment(text: str, engine: str = "ws") -> List[str]:
+def segment(text: str, engine: str = "ws") -> list[str]:
global DEFAULT_ENGINE
if not text or not isinstance(text, str):
return []
diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py
index 5e27116f6..217577a64 100644
--- a/pythainlp/tokenize/pyicu.py
+++ b/pythainlp/tokenize/pyicu.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -10,13 +9,16 @@
:See Also:
* `GitHub repository `_
"""
+
+from __future__ import annotations
+
import re
-from typing import List
from icu import BreakIterator, Locale
bd = BreakIterator.createWordInstance(Locale("th"))
+
def _gen_words(text: str) -> str:
global bd
bd.setText(text)
@@ -26,7 +28,7 @@ def _gen_words(text: str) -> str:
p = q
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
:param str text: text to be tokenized into words
:return: list of words, tokenized from the text
@@ -34,6 +36,6 @@ def segment(text: str) -> List[str]:
if not text or not isinstance(text, str):
return []
- text = re.sub("([^\u0E00-\u0E7F\n ]+)", " \\1 ", text)
+ text = re.sub("([^\u0e00-\u0e7f\n ]+)", " \\1 ", text)
return list(_gen_words(text))
diff --git a/pythainlp/tokenize/sefr_cut.py b/pythainlp/tokenize/sefr_cut.py
index e8434ba49..7a123e541 100644
--- a/pythainlp/tokenize/sefr_cut.py
+++ b/pythainlp/tokenize/sefr_cut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -9,7 +8,8 @@
:See Also:
* `GitHub repository `_
"""
-from typing import List
+
+from __future__ import annotations
import sefr_cut
@@ -17,7 +17,7 @@
sefr_cut.load_model(engine=DEFAULT_ENGINE)
-def segment(text: str, engine: str = "ws1000") -> List[str]:
+def segment(text: str, engine: str = "ws1000") -> list[str]:
global DEFAULT_ENGINE
if not text or not isinstance(text, str):
return []
diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py
index 6ea6daade..e7abeabc3 100644
--- a/pythainlp/tokenize/ssg.py
+++ b/pythainlp/tokenize/ssg.py
@@ -1,13 +1,12 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List
+from __future__ import annotations
from ssg import syllable_tokenize
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
Syllable tokenizer using ssg
"""
diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py
index 81a92b30c..786050397 100644
--- a/pythainlp/tokenize/tcc.py
+++ b/pythainlp/tokenize/tcc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -13,8 +12,10 @@
`_)
* Python code: Korakot Chaovavanich
"""
+
+from __future__ import annotations
+
import re
-from typing import List, Set
_RE_TCC = (
"""\
@@ -48,9 +49,7 @@
ก็
อึ
หึ
-""".replace(
- "k", "(cc?[d|ิ]?[์])?"
- )
+""".replace("k", "(cc?[d|ิ]?[์])?")
.replace("c", "[ก-ฮ]")
.replace("t", "[่-๋]?")
.replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel
@@ -83,7 +82,7 @@ def tcc(text: str) -> str:
p += n
-def tcc_pos(text: str) -> Set[int]:
+def tcc_pos(text: str) -> set[int]:
"""
TCC positions
@@ -103,7 +102,7 @@ def tcc_pos(text: str) -> Set[int]:
return p_set
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
Subword segmentation
diff --git a/pythainlp/tokenize/tcc_p.py b/pythainlp/tokenize/tcc_p.py
index fe4376bbb..1d7fd7325 100644
--- a/pythainlp/tokenize/tcc_p.py
+++ b/pythainlp/tokenize/tcc_p.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
@@ -14,8 +13,10 @@
`_)
* Python code: Korakot Chaovavanich
"""
+
+from __future__ import annotations
+
import re
-from typing import List, Set
_RE_TCC = (
"""\
@@ -48,9 +49,7 @@
ก็
อึ
หึ
-""".replace(
- "k", "(cc?[dิ]?[์])?"
- )
+""".replace("k", "(cc?[dิ]?[์])?")
.replace("c", "[ก-ฮ]")
.replace("t", "[่-๋]?")
.replace("d", "อูอุ".replace("อ", "")) # DSara: lower vowel
@@ -83,7 +82,7 @@ def tcc(text: str) -> str:
p += n
-def tcc_pos(text: str) -> Set[int]:
+def tcc_pos(text: str) -> set[int]:
"""
TCC positions
@@ -103,7 +102,7 @@ def tcc_pos(text: str) -> Set[int]:
return p_set
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
"""
Subword segmentation
diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py
index 31556ae2e..e6f1059f3 100644
--- a/pythainlp/tokenize/thai2fit.py
+++ b/pythainlp/tokenize/thai2fit.py
@@ -1,12 +1,13 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
from functools import lru_cache
+
from pythainlp.corpus import get_corpus
from pythainlp.tokenize import Tokenizer
+
@lru_cache
def thai2fit_tokenizer():
"""Lazy load Thai2Fit tokenizer with cache"""
diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py
index 2c74f8875..15bc321e7 100644
--- a/pythainlp/tokenize/thaisumcut.py
+++ b/pythainlp/tokenize/thaisumcut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileCopyrightText: Copyright 2020 Nakhun Chumpolsathien
# SPDX-License-Identifier: Apache-2.0
@@ -15,21 +14,22 @@
school={Beijing Institute of Technology}
"""
+from __future__ import annotations
+
import math
import operator
import re
-from typing import List
from pythainlp.tokenize import word_tokenize
-def list_to_string(list: List[str]) -> str:
+def list_to_string(list: list[str]) -> str:
string = "".join(list)
string = " ".join(string.split())
return string
-def middle_cut(sentences: List[str]) -> List[str]:
+def middle_cut(sentences: list[str]) -> list[str]:
new_text = ""
for sentence in sentences:
sentence_size = len(word_tokenize(sentence, keep_whitespace=False))
@@ -86,7 +86,7 @@ def middle_cut(sentences: List[str]) -> List[str]:
class ThaiSentenceSegmentor:
def split_into_sentences(
self, text: str, isMiddleCut: bool = False
- ) -> List[str]:
+ ) -> list[str]:
# Declare Variables
th_alphabets = "([ก-๙])"
th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)"
diff --git a/pythainlp/tokenize/tltk.py b/pythainlp/tokenize/tltk.py
index fb044caf0..90b1ab4b2 100644
--- a/pythainlp/tokenize/tltk.py
+++ b/pythainlp/tokenize/tltk.py
@@ -1,17 +1,18 @@
-# -*- coding: utf-8 -*-
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
-from typing import List
+from __future__ import annotations
try:
from tltk.nlp import syl_segment
from tltk.nlp import word_segment as tltk_segment
except ImportError:
- raise ImportError("Not found tltk! Please install tltk by pip install tltk")
+ raise ImportError(
+ "Not found tltk! Please install tltk by pip install tltk"
+ )
-def segment(text: str) -> List[str]:
+def segment(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
text = text.replace(" ", "")
@@ -22,7 +23,7 @@ def segment(text: str) -> List[str]:
return _temp
-def syllable_tokenize(text: str) -> List[str]:
+def syllable_tokenize(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
_temp = syl_segment(text)
@@ -32,7 +33,7 @@ def syllable_tokenize(text: str) -> List[str]:
return _temp
-def sent_tokenize(text: str) -> List[str]:
+def sent_tokenize(text: str) -> list[str]:
text = text.replace(" ", "")
_temp = tltk_segment(text).replace("", " ").replace("|", "")
_temp = _temp.split("