Skip to content

Commit 1146389

Browse files
wannaphongwannaphong
authored
Merge branch 'dev' into copilot/add-module-th-pron
2 parents 48d61a9 + 3705800 commit 1146389

12 files changed

Lines changed: 205 additions & 5 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ and this project adheres to
1717
- Full release notes: <https://github.com/PyThaiNLP/pythainlp/releases>
1818
- Commit history: <https://github.com/PyThaiNLP/pythainlp/compare/v5.3.3...v5.3.4>
1919

20+
## [Unreleased]
21+
22+
## Changed
23+
24+
- Improve guardrails in `check_sara()` and `nighit()`
25+
2026
## [5.3.4] - 2026-04-02
2127

2228
### Fixed

‎docs/api/transliterate.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ Modules
2525

2626
This function provides assistance in generating phonetic representations of Thai words, which is particularly useful for language learning and pronunciation practice.
2727

28+
.. autofunction:: pronunciate_pali
29+
:noindex:
30+
2831
.. autofunction:: puan
2932
:noindex:
3033

‎pythainlp/khavee/core.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,15 @@ def check_sara(self, word: str) -> str:
5353
sara = []
5454
countoa = 0
5555

56+
if not word:
57+
return ""
58+
5659
# In case of การันย์
5760
if "์" in word[-1]:
5861
word = word[:-2]
62+
# After removing the karun, the word may become empty (e.g. "ก์")
63+
if not word:
64+
return ""
5965

6066
# In case of สระเดี่ยว
6167
for i in word:
@@ -217,7 +223,7 @@ def check_sara(self, word: str) -> str:
217223
sara.append("เอือ")
218224

219225
if not sara:
220-
return "Can't find Sara in this word"
226+
return ""
221227

222228
return sara[0]
223229

@@ -251,6 +257,9 @@ def check_marttra(self, word: str) -> str:
251257
word = self.handle_karun_sound_silence(word)
252258
word = remove_tonemark(word)
253259

260+
if not word:
261+
return ""
262+
254263
# Check for ำ at the end (represents "am" sound, ends with m)
255264
if word[-1] == "ำ":
256265
return "กม"

‎pythainlp/morpheme/word_formation.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,28 @@ def nighit(w1: str, w2: str) -> str:
3535
>>> nighit("สํ", "โยค")
3636
'สังโยค'
3737
"""
38+
if not isinstance(w1, str) or not isinstance(w2, str):
39+
raise TypeError("Both w1 and w2 must be strings.")
40+
w1 = w1.strip()
41+
w2 = w2.strip()
42+
if not w1:
43+
return w2
44+
if not w2:
45+
return w1
3846
if not str(w1).endswith("ํ") and len(w1) != 2:
3947
raise NotImplementedError(f"The function doesn't support {w1}.")
4048
list_w1 = list(w1)
4149
list_w2 = list(w2)
4250
newword = []
4351
newword.append(list_w1[0])
4452
newword.append("ั")
45-
consonant_start = [i for i in list_w2 if i in set(thai_consonants)][0]
53+
_consonants = set(thai_consonants)
54+
consonants_in_w2 = [i for i in list_w2 if i in _consonants]
55+
if not consonants_in_w2:
56+
raise ValueError(
57+
f"w2 {w2!r} contains no Thai consonants."
58+
)
59+
consonant_start = consonants_in_w2[0]
4660
if consonant_start in ["ก", "ช", "ค", "ข", "ง"]:
4761
newword.append("ง")
4862
elif consonant_start in ["จ", "ฉ", "ช", "ฌ"]:

‎pythainlp/tokenize/core.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ def sent_tokenize(
452452
if isinstance(text, list):
453453
try:
454454
original_text = "".join(text)
455-
except ValueError:
455+
except TypeError:
456456
return []
457457
else:
458458
original_text = str(text)

‎pythainlp/transliterate/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
"romanize",
1111
"transliterate",
1212
"transliterate_wiktionary",
13+
"pronunciate_pali",
1314
]
1415

1516
from pythainlp.transliterate.core import pronunciate, romanize, transliterate
17+
from pythainlp.transliterate.pali import pronunciate_pali
1618
from pythainlp.transliterate.spoonerism import puan
1719
from pythainlp.transliterate.wiktionary import (
1820
get_word_dict,

‎pythainlp/transliterate/pali.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
2+
# SPDX-FileType: SOURCE
3+
# SPDX-License-Identifier: Apache-2.0
4+
import re
5+
6+
7+
def pronunciate_pali(word: str) -> str:
8+
"""
9+
Convert pali word to Thai pronunciation
10+
11+
:param str word: Pali word (written in Thai script) to be pronunciated.
12+
13+
:return: A string of Thai letters indicating
14+
how the input text should be pronounced.
15+
16+
:Example:
17+
18+
>>> from pythainlp.transliterate import pronunciate_pali
19+
>>> pronunciate_pali('ภควา')
20+
'ภะคะวา'
21+
>>> pronunciate_pali('สมฺมา')
22+
'สัมมา'
23+
>>> pronunciate_pali('มยํ')
24+
'มะยัง'
25+
>>> pronunciate_pali('สฺวากฺขา')
26+
'สวากขาโต'
27+
28+
"""
29+
# 1. จัดการช่องว่างที่อาจพิมพ์เกินมาก่อนนฤคหิต
30+
word = word.replace(" ํ", "ํ").replace("ฺ ", "ฺ")
31+
32+
# --- กฎข้อ 3: นฤคหิต (ํ) ---
33+
# แบบมีสระหน้า (เ-ไ) เช่น โกํ -> โกง
34+
word = re.sub(r'([เ-ไ][ก-ฮ])ํ', r'\1ง_F_', word)
35+
# แบบมีสระบน/ล่าง (ะ-ู) เช่น สุ ํ -> สุง
36+
word = re.sub(r'([ก-ฮ][ะ-ู])ํ', r'\1ง_F_', word)
37+
# แบบพยัญชนะเดี่ยว เช่น หํ -> หัง
38+
word = re.sub(r'([ก-ฮ])ํ', r'\1ัง_F_', word)
39+
40+
# --- กฎข้อ 4 & 5: พินทุ (ฺ) เป็นตัวควบกล้ำ (ต้นคำ) ---
41+
# ถ้าพินทุอยู่พยัญชนะตัวแรกของคำ (ไม่มีตัวอื่นนำหน้า) ให้มาร์คเป็นตัวควบกล้ำ (_C_)
42+
word = re.sub(r'(^|\s)([ก-ฮ])ฺ', r'\1\2_C_', word)
43+
44+
# --- กฎข้อ 2: พินทุ (ฺ) เป็นตัวสะกด ---
45+
# แบบมีสระนำหน้า เช่น เนยฺ -> เนย
46+
word = re.sub(r'([เ-ไ][ก-ฮ])([ก-ฮ])ฺ', r'\1\2_F_', word)
47+
# แบบมีสระบน/ล่าง เช่น ทิฏฺ -> ทิฏ
48+
word = re.sub(r'([ก-ฮ][ะ-ู])([ก-ฮ])ฺ', r'\1\2_F_', word)
49+
# แบบพยัญชนะเดี่ยว (สะกด) เช่น สมฺ -> สัม (เพิ่มไม้หันอากาศ)
50+
word = re.sub(r'([ก-ฮ])([ก-ฮ])ฺ', r'\1ั\2_F_', word)
51+
52+
# --- กฎข้อ 1: พยัญชนะที่ไม่มีเครื่องหมาย ให้อ่านเสียง "อะ" ---
53+
# หาพยัญชนะที่: ไม่ได้ตามหลัง เ-ไ และ ไม่ได้นำหน้าสระ, ไม่ใช่ตัวสะกด(_F_), ไม่ใช่ตัวควบ(_C_)
54+
word = re.sub(r'(?<![เ-ไ])([ก-ฮ])(?![ะ-ู็-์]|_F_|_C_)', r'\1ะ', word)
55+
56+
# ทำความสะอาด Marker ที่เราใช้ทดไว้ตอนประมวลผล (_F_ = Final, _C_ = Cluster)
57+
word = word.replace('_F_', '').replace('_C_', '')
58+
59+
return word

‎pythainlp/ulmfit/preprocess.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def replace_url(text: str) -> str:
3535
>>> replace_url("go to github.com")
3636
'go to xxurl'
3737
"""
38-
URL_PATTERN = r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))"""
38+
URL_PATTERN = r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))"""
3939
return re.sub(URL_PATTERN, _TK_URL, text)
4040

4141

‎tests/core/test_khavee.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,3 +258,14 @@ def test_เอือ_sara(self):
258258

259259
def test_returns_string(self):
260260
self.assertIsInstance(self.kv.check_sara("เริง"), str)
261+
262+
def test_empty_string_returns_empty(self):
263+
self.assertEqual(self.kv.check_sara(""), "")
264+
265+
def test_empty_string_after_removing_karun_returns_empty(self):
266+
self.assertEqual(self.kv.check_sara("ก์"), "")
267+
268+
def test_empty_string_after_removing_tone_marks_returns_empty(self):
269+
self.assertEqual(
270+
self.kv.check_sara("\u0e48"), ""
271+
) # The string contains only Thai Mai Ek tone mark

‎tests/core/test_morpheme.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,26 @@
1010
class MorphemeTestCase(unittest.TestCase):
1111
def test_nighit(self):
1212
self.assertEqual(nighit("สํ", "คีต"), "สังคีต")
13+
self.assertEqual(
14+
nighit("สํ", "คีต "), "สังคีต"
15+
) # w2 has trailing space, should still work
16+
self.assertEqual(
17+
nighit("สํ ", "คีต"), "สังคีต"
18+
) # w1 has trailing space, should still work
1319
self.assertEqual(nighit("สํ", "จร"), "สัญจร")
1420
self.assertEqual(nighit("สํ", "ฐาน"), "สัณฐาน")
1521
self.assertEqual(nighit("สํ", "นิษฐาน"), "สันนิษฐาน")
1622
self.assertEqual(nighit("สํ", "ปทา"), "สัมปทา")
1723
self.assertEqual(nighit("สํ", "โยค"), "สังโยค")
24+
self.assertEqual(nighit("", "คีต"), "คีต") # w1 is empty, should return w2
25+
self.assertEqual(nighit("สํ", ""), "สํ") # w2 is empty, should return w1
26+
1827
with self.assertRaises(NotImplementedError):
1928
nighit("abc", "คีต") # w1 does not end with ํ and len > 2
2029
with self.assertRaises(NotImplementedError):
2130
nighit("สํ", "มาร") # consonant ม is not in any supported group
31+
with self.assertRaises(ValueError):
32+
nighit("สํ", "123") # w2 does not contain any Thai consonant
2233

2334
def test_is_native_thai(self):
2435
self.assertFalse(is_native_thai(None)) # type: ignore[arg-type]

0 commit comments

Comments
 (0)