diff --git a/pyproject.toml b/pyproject.toml index d93d8f9ae..5114d4089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Intended Audience :: Developers", "Natural Language :: Thai", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -185,6 +186,7 @@ extra = [ "pandas>=0.24", "ssg>=0.0.8", "symspellpy>=6.7.6", + "tltk>=1.10", ] # Full dependencies - pinned where available diff --git a/tests/compact/testc_tokenize.py b/tests/compact/testc_tokenize.py index 5ccc90fd7..cc6abcd03 100644 --- a/tests/compact/testc_tokenize.py +++ b/tests/compact/testc_tokenize.py @@ -23,6 +23,7 @@ SENT_4, TEXT_1, ) +from ..test_helpers import assert_segment_handles_none_and_empty class SentTokenizeCRFCutTestCaseC(unittest.TestCase): @@ -79,8 +80,7 @@ def test_subword_tokenize(self): class WordTokenizeICUTestCaseC(unittest.TestCase): def test_icu(self): - self.assertEqual(pyicu.segment(None), []) - self.assertEqual(pyicu.segment(""), []) + assert_segment_handles_none_and_empty(self, pyicu.segment) self.assertEqual( word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"), ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], diff --git a/tests/compact/testc_util.py b/tests/compact/testc_util.py index 33a684549..fa7b4cdf9 100644 --- a/tests/compact/testc_util.py +++ b/tests/compact/testc_util.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for pythainlp.util module. -""" +"""Unit tests for pythainlp.util module.""" import unittest diff --git a/tests/core/test_robustness.py b/tests/core/test_robustness.py index 4b764c24c..7970157dc 100644 --- a/tests/core/test_robustness.py +++ b/tests/core/test_robustness.py @@ -253,4 +253,3 @@ def test_word_tokenize_with_very_long_strings(self): f"word_tokenize (engine={engine}) failed with " f"very long string (index={i}): {e}" ) - diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 375bddddb..089fb2f6b 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -6,6 +6,7 @@ from os import path from pythainlp.tag import ( + NER, PerceptronTagger, perceptron, pos_tag, @@ -88,6 +89,11 @@ def test_pos_tag(self): ], ) + def test_NER_error_handling(self): + # Test error handling for invalid engine/corpus combination + with self.assertRaises(ValueError): + NER(engine="thainer", corpus="cat") + class PerceptronTaggerTestCase(unittest.TestCase): """Test pythainlp.tag.PerceptronTagger diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index ecbcc3721..35ea68889 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -11,6 +11,7 @@ longest, multi_cut, newmm, + paragraph_tokenize, sent_tokenize, subword_tokenize, syllable_tokenize, @@ -22,6 +23,8 @@ ) from pythainlp.util import dict_trie +from ..test_helpers import assert_segment_handles_none_and_empty + TEXT_1 = "หมอนทองตากลมหูว์MBK39 :.ฉฺ๐๐๓-#™±" TEXT_2 = "ทดสอบ" @@ -231,7 +234,7 @@ def test_word_detokenize(self): ) def test_numeric_data_format(self): - engines = ["newmm"] + engines = ["newmm", "longest"] for engine in engines: self.assertIn( @@ -257,6 +260,35 @@ def test_numeric_data_format(self): self.assertIn("2.5:1", tokens) self.assertIn("5:2", tokens) + # Test join_broken_num parameter (defaults to True) + # When True, numeric data should be preserved + engine = "longest" + self.assertIn( + "127.0.0.1", + word_tokenize( + "ไอพีของคุณคือ 127.0.0.1 ครับ", + engine=engine, + join_broken_num=True, + ), + ) + # When False, numbers may be broken up + self.assertNotIn( + "127.0.0.1", + word_tokenize( + "ไอพีของคุณคือ 127.0.0.1 ครับ", + engine=engine, + join_broken_num=False, + ), + ) + self.assertNotIn( + "1,234,567.89", + word_tokenize( + "รางวัลมูลค่า 1,234,567.89 บาท", + engine=engine, + join_broken_num=False, + ), + ) + class TokenizeTestCase(unittest.TestCase): def test_Tokenizer(self): @@ -361,8 +393,7 @@ def test_word_tokenize(self): ) def test_etcc(self): - self.assertEqual(etcc.segment(None), []) - self.assertEqual(etcc.segment(""), []) + assert_segment_handles_none_and_empty(self, etcc.segment) self.assertIsInstance(etcc.segment("คืนความสุข"), list) self.assertEqual( etcc.segment("หาเงินเพื่อเรียน"), @@ -377,8 +408,7 @@ def test_etcc(self): ) def test_longest(self): - self.assertEqual(longest.segment(None), []) - self.assertEqual(longest.segment(""), []) + assert_segment_handles_none_and_empty(self, longest.segment) self.assertIsInstance( longest.segment("กรุงเทพฯมากๆเพราโพาง BKKฯ"), list ) @@ -430,8 +460,7 @@ def test_longest_custom_dict(self): ) def test_mm(self): - self.assertEqual(multi_cut.segment(None), []) - self.assertEqual(multi_cut.segment(""), []) + assert_segment_handles_none_and_empty(self, multi_cut.segment) self.assertIsNotNone(multi_cut.segment("ตัด", dict_trie([""]))) self.assertEqual(word_tokenize("", engine="mm"), []) @@ -468,8 +497,7 @@ def test_mm(self): self.assertEqual(multi_cut.find_all_segment(None), []) def test_newmm(self): - self.assertEqual(newmm.segment(None), []) - self.assertEqual(newmm.segment(""), []) + assert_segment_handles_none_and_empty(self, newmm.segment) self.assertEqual( word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="newmm"), ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"], @@ -556,8 +584,7 @@ def test_newmm_dangertext(self): ) def test_tcc(self): - self.assertEqual(tcc.segment(None), []) - self.assertEqual(tcc.segment(""), []) + assert_segment_handles_none_and_empty(self, tcc.segment) self.assertEqual( tcc.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"] ) @@ -616,8 +643,7 @@ def test_tcc(self): self.assertEqual(tcc.tcc_pos(""), set()) def test_tcc_p(self): - self.assertEqual(tcc_p.segment(None), []) - self.assertEqual(tcc_p.segment(""), []) + assert_segment_handles_none_and_empty(self, tcc_p.segment) self.assertEqual( tcc_p.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"] ) @@ -652,3 +678,12 @@ def test_display_cell_tokenize(self): self.assertEqual(display_cell_tokenize("สวัสดี"), ['ส', 'วั', 'ส', 'ดี']) self.assertEqual(display_cell_tokenize("ทดสอบ"), ["ท", "ด", "ส", "อ", "บ"]) self.assertEqual(display_cell_tokenize("ภาษาไทย"), ["ภ", "า", "ษ", "า", "ไ", "ท", "ย"]) + + def test_paragraph_tokenize(self): + # Test error handling for invalid engine + text = ( + "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" + "จากผลงานวิจัยที่เคยทำมาในอดีต" + ) + with self.assertRaises(ValueError): + paragraph_tokenize(text, engine="non-existent-engine") diff --git a/tests/extra/testx_augment.py b/tests/extra/testx_augment.py index 69594c42d..175310f04 100644 --- a/tests/extra/testx_augment.py +++ b/tests/extra/testx_augment.py @@ -22,13 +22,13 @@ def setUp(self): self.text2 = "เราอยู่ที่มหาวิทยาลัยขอนแก่น" def test_WordNetAug(self): - nltk.download('omw-1.4', force=True) # load wordnet + nltk.download("omw-1.4", force=True) # load wordnet wordnetaug = WordNetAug() self.assertIsNotNone(wordnetaug.augment(self.text)) self.assertIsNotNone(wordnetaug.find_synonyms("ผม", pos=None)) self.assertIsNotNone(wordnetaug.augment(self.text, postag=False)) - self.assertIsNone(postype2wordnet('n', 'abc')) - self.assertIsNotNone(postype2wordnet('NOUN', 'orchid')) + self.assertIsNone(postype2wordnet("n", "abc")) + self.assertIsNotNone(postype2wordnet("NOUN", "orchid")) # def test_Thai2fitAug(self): # _aug = Thai2fitAug() diff --git a/tests/extra/testx_spell.py b/tests/extra/testx_spell.py index f932880df..48d43dc5f 100644 --- a/tests/extra/testx_spell.py +++ b/tests/extra/testx_spell.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 # Tests for spell functions that need extra dependencies -# Note: Tests requiring phunspell/tltk/torch/HuggingFace Hub have been moved to tests.noauto import unittest @@ -20,7 +19,6 @@ class SpellTestCaseX(unittest.TestCase): def test_spell(self): - # Tests for symspellpy only (phunspell and tltk moved to noauto) result = spell("เน้ร", engine="symspellpy") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) @@ -30,16 +28,26 @@ def test_spell(self): self.assertGreater(len(result), 0) def test_word_correct(self): - # Tests for symspellpy only (phunspell and wanchanberta moved to noauto) result = correct("ทดสอง", engine="symspellpy") self.assertIsInstance(result, str) self.assertNotEqual(result, "") def test_spell_sent(self): - # Tests for symspellpy only (phunspell moved to noauto) self.assertIsNotNone(spell_sent(SENT_TOKS, engine="symspellpy")) def test_correct_sent(self): - # Tests for symspellpy only (phunspell and wanchanberta moved to noauto) self.assertIsNotNone(correct_sent(SENT_TOKS, engine="symspellpy")) self.assertIsNotNone(symspellpy.correct_sent(SENT_TOKS)) + + +class SpellTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine spell checking""" + + def test_spell_tltk(self): + result = spell("เน้ร", engine="tltk") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = spell("เดก", engine="tltk") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) diff --git a/tests/extra/testx_tag.py b/tests/extra/testx_tag.py index 024661ddb..1373d9528 100644 --- a/tests/extra/testx_tag.py +++ b/tests/extra/testx_tag.py @@ -3,16 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 # Tests for tag functions that need extra dependencies -# Note: Tests requiring transformers/tltk have been moved to tests.noautotest import unittest +from pythainlp.tag import pos_tag, tltk from pythainlp.tag.thainer import ThaiNameTagger class TagTestCaseX(unittest.TestCase): - # Tests for ThaiNameTagger (doesn't require transformers or tltk) - # All tltk and transformers-based tests have been moved to tests.noautotest def test_thai_name_tagger_1_5(self): ner = ThaiNameTagger(version="1.5") @@ -117,3 +115,39 @@ def test_thai_name_tagger_1_4(self): ) ) + +class TagTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine POS tagging and NER""" + + def test_pos_tag_tltk(self): + tokens = ["ผม", "รัก", "คุณ"] + self.assertIsNotNone(pos_tag(tokens, engine="tltk")) + with self.assertRaises(ValueError): + tltk.pos_tag(tokens, corpus="blackboard") + + def test_tltk_ner(self): + self.assertEqual(tltk.get_ner(""), []) + self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone( + tltk.get_ner("พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า") + ) + self.assertIsNotNone( + tltk.get_ner( + "พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า", + tag=True, + ) + ) + self.assertIsNotNone( + tltk.get_ner( + """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น + จังหวัดหนองคาย 43000""" + ) + ) + self.assertIsNotNone( + tltk.get_ner( + """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น + จังหวัดหนองคาย 43000""", + tag=True, + ) + ) diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index d76f3e0bb..d081a6049 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 # Tests for tokenize functions that need extra dependencies -# Note: Tests requiring TensorFlow/Keras/tltk/torch/transformers have been moved to tests.noauto import unittest @@ -12,6 +11,7 @@ sent_tokenize, ssg, subword_tokenize, + tltk, word_tokenize, ) @@ -22,9 +22,13 @@ SENT_4, TEXT_1, ) +from ..test_helpers import ( + assert_segment_handles_none_and_empty, + assert_subword_tokenize_basic, +) -class SentTokenizeThaiSumTestCase(unittest.TestCase): +class SentTokenizeThaiSumTestCaseX(unittest.TestCase): def test_sent_tokenize_thaisum(self): self.assertIsNotNone( sent_tokenize( @@ -50,10 +54,9 @@ def test_sent_tokenize_thaisum(self): ) -class SubwordTokenizeSSGTestCase(unittest.TestCase): +class SubwordTokenizeSSGTestCaseX(unittest.TestCase): def test_subword_tokenize_ssg(self): - self.assertEqual(ssg.segment(None), []) - self.assertEqual(ssg.segment(""), []) + assert_segment_handles_none_and_empty(self, ssg.segment) self.assertEqual(subword_tokenize(None, engine="ssg"), []) self.assertEqual( subword_tokenize("แมวกินปลา", engine="ssg"), ["แมว", "กิน", "ปลา"] @@ -62,13 +65,12 @@ def test_subword_tokenize_ssg(self): self.assertNotIn("า", subword_tokenize("สวัสดีดาวอังคาร", engine="ssg")) -class WordTokenizeNERCutTestCase(unittest.TestCase): +class WordTokenizeNERCutTestCaseX(unittest.TestCase): def test_word_tokenize_nercut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="nercut")) def test_nercut(self): - self.assertEqual(nercut.segment(None), []) - self.assertEqual(nercut.segment(""), []) + assert_segment_handles_none_and_empty(self, nercut.segment) self.assertIsNotNone(nercut.segment("ทดสอบ")) self.assertEqual(nercut.segment("ทันแน่ๆ"), ["ทัน", "แน่ๆ"]) self.assertEqual(nercut.segment("%1ครั้ง"), ["%", "1", "ครั้ง"]) @@ -77,7 +79,68 @@ def test_nercut(self): self.assertIsNotNone(word_tokenize("ทดสอบ", engine="nercut")) -class WordTokenizeBudouxTestCase(unittest.TestCase): +class WordTokenizeBudouxTestCaseX(unittest.TestCase): def test_word_tokenize_budoux(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="budoux")) + +class SentTokenizeTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine sent tokenization""" + + def test_sent_tokenize_tltk(self): + self.assertIsNotNone( + sent_tokenize( + SENT_1, + engine="tltk", + ), + ) + self.assertIsNotNone( + sent_tokenize( + SENT_2, + engine="tltk", + ), + ) + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="tltk", + ), + ) + + +class SubwordTokenizeTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine subword tokenization""" + + def test_subword_tokenize_tltk(self): + assert_subword_tokenize_basic(self, "tltk") + + +class SyllableTokenizeTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine syllable tokenization""" + + def test_tltk(self): + assert_segment_handles_none_and_empty(self, tltk.segment) + self.assertEqual( + tltk.syllable_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + [ + "ฉัน", + "รัก", + "ภา", + "ษา", + "ไทย", + "เพราะ", + "ฉัน", + "เป็น", + "คน", + "ไทย", + ], + ) + self.assertEqual(tltk.syllable_tokenize(None), []) + self.assertEqual(tltk.syllable_tokenize(""), []) + + +class WordTokenizeTLTKTestCaseX(unittest.TestCase): + """Tests for tltk engine word tokenization""" + + def test_word_tokenize_tltk(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="tltk")) diff --git a/tests/extra/testx_translate.py b/tests/extra/testx_translate.py index 542b8432e..6d5fcaebd 100644 --- a/tests/extra/testx_translate.py +++ b/tests/extra/testx_translate.py @@ -71,7 +71,7 @@ def test_translate(self): # ) # ) with self.assertRaises(ValueError): - self.th_cat_translator = Translate('th', 'cat', engine="fkfj") + self.th_cat_translator = Translate("th", "cat", engine="fkfj") def test_word_translate(self): self.assertIsNone(word_translate("cat", src="en", target="th")) diff --git a/tests/extra/testx_util.py b/tests/extra/testx_util.py index f5a69d1db..e7db2650e 100644 --- a/tests/extra/testx_util.py +++ b/tests/extra/testx_util.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for pythainlp.util module. -""" +"""Unit tests for pythainlp.util module.""" import unittest diff --git a/tests/noauto/testn_spell.py b/tests/noauto/testn_spell.py index b9fbc4668..58667a3e3 100644 --- a/tests/noauto/testn_spell.py +++ b/tests/noauto/testn_spell.py @@ -2,10 +2,10 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -# Tests for spell functions that require phunspell (Cython) or tltk +# Tests for spell functions that require phunspell (Cython) or torch # These tests are NOT run in automated CI workflows due to: # - Compilation issues (phunspell requires Cython) -# - Compilation issues (tltk) +# - Large dependencies (torch ~800MB) # - Python 3.13+ compatibility issues import unittest @@ -45,19 +45,6 @@ def test_correct_sent_phunspell(self): self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) -class SpellTLTKTestCaseN(unittest.TestCase): - """Tests for tltk engine (requires tltk with compilation issues)""" - - def test_spell_tltk(self): - result = spell("เน้ร", engine="tltk") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - result = spell("เดก", engine="tltk") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - class SpellWanchanbertaTestCaseN(unittest.TestCase): """Tests for wanchanberta_thai_grammarly engine (requires torch)""" @@ -68,6 +55,7 @@ def test_word_correct_wanchanberta(self): def test_correct_sent_wanchanberta(self): from ..core.test_spell import SENT_TOKS + self.assertIsNotNone( correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") ) diff --git a/tests/noauto/testn_tag.py b/tests/noauto/testn_tag.py index 1397b09e3..99333397d 100644 --- a/tests/noauto/testn_tag.py +++ b/tests/noauto/testn_tag.py @@ -2,10 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -# Tests for tag functions that require transformers or tltk +# Tests for tag functions that require transformers or torch # These tests are NOT run in automated CI workflows due to: # - Large dependencies (transformers, torch) -# - Compilation issues (tltk) # - Python 3.13+ compatibility issues import unittest @@ -13,49 +12,10 @@ from pythainlp.tag import ( NER, NNER, - pos_tag, pos_tag_transformers, - tltk, ) -class TagTLTKTestCaseN(unittest.TestCase): - """Tests for tltk engine (requires tltk with compilation issues)""" - - def test_pos_tag_tltk(self): - tokens = ["ผม", "รัก", "คุณ"] - self.assertIsNotNone(pos_tag(tokens, engine="tltk")) - with self.assertRaises(ValueError): - tltk.pos_tag(tokens, corpus="blackboard") - - def test_tltk_ner(self): - self.assertEqual(tltk.get_ner(""), []) - self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone( - tltk.get_ner("พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า") - ) - self.assertIsNotNone( - tltk.get_ner( - "พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า", - tag=True, - ) - ) - self.assertIsNotNone( - tltk.get_ner( - """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น - จังหวัดหนองคาย 43000""" - ) - ) - self.assertIsNotNone( - tltk.get_ner( - """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น - จังหวัดหนองคาย 43000""", - tag=True, - ) - ) - - class TagTransformersTestCaseN(unittest.TestCase): """Tests for transformers-based engines (requires transformers, torch)""" diff --git a/tests/noauto/testn_tokenize.py b/tests/noauto/testn_tokenize.py index 597d940e6..a2918a180 100644 --- a/tests/noauto/testn_tokenize.py +++ b/tests/noauto/testn_tokenize.py @@ -2,10 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -# Tests for tokenize functions that require TensorFlow, Keras, or tltk +# Tests for tokenize functions that require TensorFlow, Keras, or transformers # These tests are NOT run in automated CI workflows due to: -# - Large dependencies (TensorFlow, Keras) -# - Compilation issues (tltk) +# - Large dependencies (TensorFlow, Keras, transformers, torch) # - Python 3.13+ compatibility issues import unittest @@ -17,18 +16,18 @@ paragraph_tokenize, sefr_cut, sent_tokenize, - subword_tokenize, - tltk, word_dict_trie, word_tokenize, ) from ..core.test_tokenize import ( - SENT_1, - SENT_2, SENT_3, TEXT_1, ) +from ..test_helpers import ( + assert_segment_handles_none_and_empty, + assert_subword_tokenize_basic, +) class DetokenizeTestCaseN(unittest.TestCase): @@ -79,69 +78,12 @@ def test_numeric_data_format(self): ) -class SentTokenizeTLTKTestCaseN(unittest.TestCase): - def test_sent_tokenize_tltk(self): - self.assertIsNotNone( - sent_tokenize( - SENT_1, - engine="tltk", - ), - ) - self.assertIsNotNone( - sent_tokenize( - SENT_2, - engine="tltk", - ), - ) - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="tltk", - ), - ) - - -class SubwordTokenizeTLTKTestCaseN(unittest.TestCase): - def test_subword_tokenize_tltk(self): - self.assertEqual(subword_tokenize(None, engine="tltk"), []) - self.assertEqual(subword_tokenize("", engine="tltk"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="tltk"), list - ) - self.assertNotIn("า", subword_tokenize("สวัสดีดาวอังคาร", engine="tltk")) - self.assertIsInstance(subword_tokenize("โควิด19", engine="tltk"), list) - - -class SyllableTokenizeTLTKTestCaseN(unittest.TestCase): - def test_tltk(self): - self.assertEqual(tltk.segment(None), []) - self.assertEqual(tltk.segment(""), []) - self.assertEqual( - tltk.syllable_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - [ - "ฉัน", - "รัก", - "ภา", - "ษา", - "ไทย", - "เพราะ", - "ฉัน", - "เป็น", - "คน", - "ไทย", - ], - ) - self.assertEqual(tltk.syllable_tokenize(None), []) - self.assertEqual(tltk.syllable_tokenize(""), []) - - class WordTokenizeAttacutTestCaseN(unittest.TestCase): def test_word_tokenize_attacut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) def test_attacut(self): - self.assertEqual(attacut.segment(None), []) - self.assertEqual(attacut.segment(""), []) + assert_segment_handles_none_and_empty(self, attacut.segment) self.assertEqual( word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], @@ -160,8 +102,7 @@ def test_word_tokenize_deepcut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) def test_deepcut(self): - self.assertEqual(deepcut.segment(None), []) - self.assertEqual(deepcut.segment(""), []) + assert_segment_handles_none_and_empty(self, deepcut.segment) self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) @@ -177,8 +118,7 @@ def test_word_tokenize_oskut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) def test_oskut(self): - self.assertEqual(oskut.segment(None), []) - self.assertEqual(oskut.segment(""), []) + assert_segment_handles_none_and_empty(self, oskut.segment) self.assertIsNotNone( oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), ) @@ -192,8 +132,7 @@ def test_word_tokenize_sefr_cut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) def test_sefr_cut(self): - self.assertEqual(sefr_cut.segment(None), []) - self.assertEqual(sefr_cut.segment(""), []) + assert_segment_handles_none_and_empty(self, sefr_cut.segment) self.assertIsNotNone( sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), ) @@ -202,11 +141,6 @@ def test_sefr_cut(self): ) -class WordTokenizeTLTKTestCaseN(unittest.TestCase): - def test_word_tokenize_tltk(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="tltk")) - - class ParagraphTokenizeTestCaseN(unittest.TestCase): def test_paragraph_tokenize(self): sent = ( @@ -242,29 +176,9 @@ def test_sent_tokenize_wtp_tiny(self): class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): def test_subword_tokenize_phayathai(self): - self.assertEqual(subword_tokenize(None, engine="phayathai"), []) - self.assertEqual(subword_tokenize("", engine="phayathai"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="phayathai"), list - ) - self.assertNotIn( - "า", subword_tokenize("สวัสดีดาวอังคาร", engine="phayathai") - ) - self.assertIsInstance( - subword_tokenize("โควิด19", engine="phayathai"), list - ) + assert_subword_tokenize_basic(self, "phayathai") class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): def test_subword_tokenize_wangchanberta(self): - self.assertEqual(subword_tokenize(None, engine="wangchanberta"), []) - self.assertEqual(subword_tokenize("", engine="wangchanberta"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="wangchanberta"), list - ) - self.assertNotIn( - "า", subword_tokenize("สวัสดีดาวอังคาร", engine="wangchanberta") - ) - self.assertIsInstance( - subword_tokenize("โควิด19", engine="wangchanberta"), list - ) + assert_subword_tokenize_basic(self, "wangchanberta") diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 000000000..939bfca38 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test helper functions to reduce code duplication in tests. + +This module provides common test assertions and patterns used across +multiple test files. +""" + +import unittest + + +def assert_segment_handles_none_and_empty(test_case: unittest.TestCase, segment_func): + """Test that a segment function properly handles None and empty string inputs. + + :param unittest.TestCase test_case: The unittest.TestCase instance (typically 'self') + :param callable segment_func: The segment function to test (e.g., attacut.segment) + + :Example: + :: + + assert_segment_handles_none_and_empty(self, attacut.segment) + """ + test_case.assertEqual(segment_func(None), []) + test_case.assertEqual(segment_func(""), []) + + +def assert_subword_tokenize_handles_none_and_empty( + test_case: unittest.TestCase, engine: str +): + """Test that subword_tokenize properly handles None and empty string inputs. + + :param unittest.TestCase test_case: The unittest.TestCase instance (typically 'self') + :param str engine: The engine name to test (e.g., "phayathai") + + :Example: + :: + + assert_subword_tokenize_handles_none_and_empty(self, "phayathai") + """ + from pythainlp.tokenize import subword_tokenize + + test_case.assertEqual(subword_tokenize(None, engine=engine), []) + test_case.assertEqual(subword_tokenize("", engine=engine), []) + + +def assert_subword_tokenize_basic(test_case: unittest.TestCase, engine: str): + """Run basic subword tokenize tests with common test cases. + + This helper function runs a standard set of tests for subword tokenization: + + - None input returns empty list + - Empty string returns empty list + - Returns list type for sample text + - Does not produce standalone vowels + + :param unittest.TestCase test_case: The unittest.TestCase instance (typically 'self') + :param str engine: The engine name to test + + :Example: + :: + + assert_subword_tokenize_basic(self, "phayathai") + """ + from pythainlp.tokenize import subword_tokenize + + # Test None and empty + assert_subword_tokenize_handles_none_and_empty(test_case, engine) + + # Test with sample text + test_case.assertIsInstance( + subword_tokenize("สวัสดีดาวอังคาร", engine=engine), list + ) + + # Should not produce standalone vowels + test_case.assertNotIn( + "า", subword_tokenize("สวัสดีดาวอังคาร", engine=engine) + ) + + # Test with mixed Thai-numeric + test_case.assertIsInstance(subword_tokenize("โควิด19", engine=engine), list)