From bec3c1f09c5b0242f099f3b4582fd4883073fc20 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 2 Feb 2026 08:42:16 +0000 Subject: [PATCH 1/4] Add more ner/tokenize/util tests --- pythainlp/tag/named_entity.py | 45 ++++++++++++++++++++--------------- tests/core/test_tag.py | 7 ++++-- tests/core/test_tokenize.py | 5 ++++ tests/core/test_util.py | 10 ++++++++ 4 files changed, 46 insertions(+), 21 deletions(-) diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index d3a0d303f..98b715207 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -16,12 +16,15 @@ class NER: :param str corpus: corpus **Options for engine** - * *thainer-v2* - Thai NER engine v2.0 for Thai NER 2.0 (default) + * *phayathaibert* - PhayaThaiBERT-based Thai NER engine * *thainer* - Thai NER engine + * *thainer-v2* - Thai NER engine v2.0 for Thai NER 2.0 (default) * *tltk* - wrapper for `TLTK `_. + * *wangchanberta* - WangchanBERTa-based Thai NER engine **Options for corpus** * *thainer* - Thai NER corpus (default) + * *thainer-v2* - Thai NER v2 corpus **Note**: The tltk engine supports NER models from tltk only. """ @@ -34,29 +37,33 @@ def __init__( def load_engine(self, engine: str, corpus: str) -> None: self.name_engine = engine self.engine: Any = None - if engine == "thainer" and corpus == "thainer": - from pythainlp.tag.thainer import ThaiNameTagger + if corpus == "thainer": + if engine == "thainer": + from pythainlp.tag.thainer import ThaiNameTagger - self.engine = ThaiNameTagger() - elif engine == "thainer-v2" and corpus == "thainer": - from pythainlp.wangchanberta import NamedEntityRecognition + self.engine = ThaiNameTagger() + elif engine == "thainer-v2": + from pythainlp.wangchanberta import NamedEntityRecognition - self.engine = NamedEntityRecognition( - model="pythainlp/thainer-corpus-v2-base-model" - ) - elif engine == "tltk": - from pythainlp.tag import tltk + self.engine = NamedEntityRecognition( + model="pythainlp/thainer-corpus-v2-base-model" + ) + elif engine == "wangchanberta": + from pythainlp.wangchanberta import ThaiNameTagger as WangchanbertaThaiNameTagger # type: ignore[assignment] # noqa: I001,E501 + + self.engine = WangchanbertaThaiNameTagger(dataset_name=corpus) # type: ignore[call-arg] + elif corpus == "thainer-v2": + if engine == "phayathaibert": + from pythainlp.phayathaibert.core import NamedEntityTagger - self.engine = tltk - elif engine == "wangchanberta" and corpus == "thainer": - from pythainlp.wangchanberta import ThaiNameTagger # type: ignore[assignment] # noqa: I001 + self.engine = NamedEntityTagger() + else: # No corpus matched + if engine == "tltk": + from pythainlp.tag import tltk - self.engine = ThaiNameTagger(dataset_name=corpus) # type: ignore[call-arg] - elif engine == "phayathaibert" and corpus == "thainer-v2": - from pythainlp.phayathaibert.core import NamedEntityTagger + self.engine = tltk - self.engine = NamedEntityTagger() - else: + if self.engine is None: raise ValueError( f"NER class not support {engine} engine or {corpus} corpus." ) diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 089fb2f6b..8bbfe309c 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -90,9 +90,12 @@ def test_pos_tag(self): ) def test_NER_error_handling(self): - # Test error handling for invalid engine/corpus combination with self.assertRaises(ValueError): - NER(engine="thainer", corpus="cat") + NER(engine="xx_non_existing", corpus="thainer") + with self.assertRaises(ValueError): + NER(engine="xx_non_existing", corpus="thainer-v2") + with self.assertRaises(ValueError): + NER(engine="xx_non_existing", corpus="xx_non_existing") class PerceptronTaggerTestCase(unittest.TestCase): diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index 35ea68889..0135acf0f 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -333,6 +333,11 @@ def test_sent_tokenize(self): with self.assertRaises(ValueError): sent_tokenize("ฉันไป กิน", engine="XX") # engine does not exist + self.assertIsNotNone( + sent_tokenize(SENT_1, engine="crfcut") + ) + self.assertIsInstance(sent_tokenize(SENT_1, engine="crfcut"), list) + def test_subword_tokenize(self): self.assertEqual(subword_tokenize(None), []) self.assertEqual(subword_tokenize(""), []) diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 209d0521e..5bc6800a3 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -46,6 +46,8 @@ remove_zw, sound_syllable, spelling, + spell_word, + spell_syllable, syllable_length, syllable_open_close_detector, text_to_arabic_digit, @@ -984,6 +986,14 @@ def test_spelling(self): spelling("กั้น"), ['กอ', 'อะ', 'นอ', 'กัน', 'ไม้โท', 'กั้น'] ) + def test_spell_words(self): + self.assertEqual(spell_syllable("แมว"), ['มอ', 'วอ', 'แอ', 'แมว']) + self.assertEqual(spell_syllable("คน"), ['คอ', 'นอ', 'คน']) + + self.assertEqual(spell_word("คนดี"), ['คอ', 'นอ', 'คน', 'ดอ', 'อี', 'ดี', 'คนดี']) + self.assertEqual(spell_word("แมว"), ['มอ', 'วอ', 'แอ', 'แมว']) + self.assertEqual(spell_word(""), []) + def test_longest_common_subsequence(self): self.assertEqual(longest_common_subsequence("ABCBDAB", "BDCAB"), "BDAB") self.assertEqual(longest_common_subsequence("AGGTAB", "GXTXAYB"), "GTAB") From 144e0d9dc300a625ec6f0a2c6b76b2631999cbe8 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 2 Feb 2026 08:49:09 +0000 Subject: [PATCH 2/4] Add missing export --- pythainlp/util/__init__.py | 3 +++ tests/core/test_util.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index eb65b23f4..068e51623 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -45,6 +45,8 @@ "reorder_vowels", "rhyme", "sound_syllable", + "spell_syllable", + "spell_word", "spelling", "spell_words", "syllable_length", @@ -121,6 +123,7 @@ from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) +from pythainlp.util.spell_words import spell_syllable, spell_word from pythainlp.util.strftime import thai_strftime from pythainlp.util.thai import ( diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 5bc6800a3..2d4238086 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -45,9 +45,9 @@ remove_trailing_repeat_consonants, remove_zw, sound_syllable, - spelling, - spell_word, spell_syllable, + spell_word, + spelling, syllable_length, syllable_open_close_detector, text_to_arabic_digit, From 95ea7a791190307dd164d1063c8ed56e3adc1f6f Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 2 Feb 2026 08:59:42 +0000 Subject: [PATCH 3/4] Move spell_words test to testc --- tests/compact/testc_util.py | 3 +-- tests/core/test_util.py | 10 ---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/compact/testc_util.py b/tests/compact/testc_util.py index fa7b4cdf9..7ff9915e5 100644 --- a/tests/compact/testc_util.py +++ b/tests/compact/testc_util.py @@ -6,8 +6,7 @@ import unittest -from pythainlp.util import rhyme, thai_word_tone_detector -from pythainlp.util.spell_words import spell_word +from pythainlp.util import rhyme, spell_word, thai_word_tone_detector class SpellWordTestCaseC(unittest.TestCase): diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 2d4238086..209d0521e 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -45,8 +45,6 @@ remove_trailing_repeat_consonants, remove_zw, sound_syllable, - spell_syllable, - spell_word, spelling, syllable_length, syllable_open_close_detector, @@ -986,14 +984,6 @@ def test_spelling(self): spelling("กั้น"), ['กอ', 'อะ', 'นอ', 'กัน', 'ไม้โท', 'กั้น'] ) - def test_spell_words(self): - self.assertEqual(spell_syllable("แมว"), ['มอ', 'วอ', 'แอ', 'แมว']) - self.assertEqual(spell_syllable("คน"), ['คอ', 'นอ', 'คน']) - - self.assertEqual(spell_word("คนดี"), ['คอ', 'นอ', 'คน', 'ดอ', 'อี', 'ดี', 'คนดี']) - self.assertEqual(spell_word("แมว"), ['มอ', 'วอ', 'แอ', 'แมว']) - self.assertEqual(spell_word(""), []) - def test_longest_common_subsequence(self): self.assertEqual(longest_common_subsequence("ABCBDAB", "BDCAB"), "BDAB") self.assertEqual(longest_common_subsequence("AGGTAB", "GXTXAYB"), "GTAB") From 85dbcefc1cf09b2782ab5c05ec6ee824800cfa40 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 2 Feb 2026 09:04:27 +0000 Subject: [PATCH 4/4] Remove crfcut from core test --- tests/core/test_tokenize.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index 0135acf0f..35ea68889 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -333,11 +333,6 @@ def test_sent_tokenize(self): with self.assertRaises(ValueError): sent_tokenize("ฉันไป กิน", engine="XX") # engine does not exist - self.assertIsNotNone( - sent_tokenize(SENT_1, engine="crfcut") - ) - self.assertIsInstance(sent_tokenize(SENT_1, engine="crfcut"), list) - def test_subword_tokenize(self): self.assertEqual(subword_tokenize(None), []) self.assertEqual(subword_tokenize(""), [])