diff --git a/CHANGELOG.md b/CHANGELOG.md index d0448996e..b722340b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ See PR for prompt and details. - Make package zip-safe #1212 - Ensure thread-safety for tokenizers #1213 - Add Thai-NNER integration with top-level entity filtering #1221 +- Reorganize noauto test suite by dependency groups (torch, tensorflow, onnx, cython, network) #935 - Improved documentation; code cleanup; more tests ## Version 5.1.2 -> 5.2.0 diff --git a/pyproject.toml b/pyproject.toml index d7d905cdb..696a444b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,8 @@ dependencies = [ [project.optional-dependencies] -# Development and testing dependencies +## 1) Development ######################################## + dev = [ "black>=25.11.0", "build>=1.0.0", @@ -79,14 +80,13 @@ dev = [ "tox>=4.30.3", ] -# Documentation dependencies docs = [ "Sphinx>=6.2", "sphinx-copybutton>=0.5.2", "sphinx-rtd-theme>=3.1.0", ] -# Below are dependencies for optional features +## 2) Feature sets ####################################### abbreviation = ["khamyo>=0.2.0"] @@ -166,9 +166,13 @@ wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"] wunsen = ["wunsen>=0.0.3"] -# Below are defined groups of CI testing dependencies +## 3) Testing ############################################ + +# Below are dependency groups for each defined test suites -# Compact dependencies - safe small set of optional dependencies +# Compact dependencies - for tests.compact modules +# Safe small set of optional dependencies +# for common tasks like tokenization and tagging. compact = [ "nlpo3>=1.4.0", "numpy>=1.26.0", @@ -178,7 +182,6 @@ compact = [ ] # Extra dependencies - for tests.extra modules -# Note: Some packages excluded due to Python 3.13 compatibility issues extra = [ "bpemb>=0.3.2", "budoux>=0.7.0", @@ -190,6 +193,46 @@ extra = [ "tltk>=1.10", ] +# Noauto test dependencies - for tests.noauto_* modules +# These are further grouped by dependency framework to avoid conflicts + +# PyTorch-based dependencies - for tests.noauto_torch +noauto-torch = [ + "attacut>=1.0.6", + "numpy>=1.26.0", + "sentencepiece>=0.1.91", + "thai-nner>=0.3", + "tltk>=1.10", + "torch>=1.13.1", + "transformers>=4.22.1", + "wtpsplit>=1.0.1", +] + +# TensorFlow-based dependencies - for tests.noauto_tensorflow +noauto-tensorflow = [ + "deepcut>=0.7.0", + "numpy>=1.26.0", +] + +# ONNX Runtime-based dependencies - for tests.noauto_onnx +noauto-onnx = [ + "numpy>=1.26.0", + "onnxruntime>=1.10.0", + "oskut>=1.3", + "sefr_cut>=1.1", +] + +# Cython-based dependencies - for tests.noauto_cython +noauto-cython = [ + "phunspell>=0.1.6", +] + +# Network-dependent tests - for tests.noauto_network +# These tests require network access but minimal dependencies +noauto-network = [ + "huggingface-hub>=0.16.0", +] + # Full dependencies - pinned where available full = [ "attacut==1.0.6", diff --git a/tests/README.md b/tests/README.md index 02af5fd8f..af26a0fff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -74,16 +74,88 @@ The CI/CD test workflow is at ## Noauto tests (testn_*.py) -- These dependencies might include huge libraries like `tensorflow`. -- Due to dependency complexities, these functionalities may not be tested - in the CI/CD pipeline. - - In the future, we might create a separate - step or workflow to run this test suite. - It will be triggered manually. - We may also need to group test cases by - a non-conflicting set of dependencies. +The noauto (no-automated) test suite contains tests for functionalities +that require heavy dependencies which are not feasible to run in automated +CI/CD pipelines. These tests are organized into specialized suites based +on their dependency requirements. + +### Why separate noauto test suites? + +Different ML/AI frameworks often have conflicting version requirements for +their dependencies. For example: +- PyTorch and TensorFlow may require different versions of numpy or protobuf +- Large frameworks take significant time to install (~1-3 GB each) +- Some packages require Cython compilation or system libraries + +By separating tests by dependency group, we can: +- Test each framework independently without conflicts +- Optimize CI/CD resources by running only relevant test groups +- Make it easier for developers to test specific functionality + +### Noauto test suites + +#### Umbrella suite: tests.noauto + +- Run `unittest tests.noauto` +- Includes all modular noauto test suites +- Use this for comprehensive testing when all dependencies are available - Test case class suffix: `TestCaseN` +#### Modular suites by dependency: + +**PyTorch-based: tests.noauto_torch** + +- Run `unittest tests.noauto_torch` + - Need dependencies from `pip install "pythainlp[noauto-torch]"` +- Tests requiring PyTorch and its ecosystem: + - torch, transformers (PyTorch backend) + - attacut, thai-nner, wtpsplit, tltk +- Tests: spell correction (wanchanberta), NER/POS tagging (transformers-based), + tokenization (attacut), subword tokenization (phayathai, wangchanberta), + sentence tokenization (wtp) +- Dependencies: ~2-3 GB +- Test case class suffix: `TestCaseN` + +**TensorFlow-based: tests.noauto_tensorflow** + +- Run `unittest tests.noauto_tensorflow` + - Need dependencies from `pip install "pythainlp[noauto-tensorflow]"` +- Tests requiring TensorFlow: + - deepcut tokenizer +- Dependencies: ~1-2 GB +- Note: May conflict with PyTorch dependencies +- Test case class suffix: `TestCaseN` + +**ONNX Runtime-based: tests.noauto_onnx** + +- Run `unittest tests.noauto_onnx` + - Need dependencies from `pip install "pythainlp[noauto-onnx]"` +- Tests requiring ONNX Runtime: + - oskut, sefr_cut tokenizers +- Dependencies: ~200-500 MB +- Test case class suffix: `TestCaseN` + +**Cython-compiled: tests.noauto_cython** + +- Run `unittest tests.noauto_cython` + - Need dependencies from `pip install "pythainlp[noauto-cython]"` +- Tests requiring Cython-compiled packages: + - phunspell spell checker +- Requires: Cython, C compiler, system libraries (hunspell) +- Platform-specific build requirements +- Test case class suffix: `TestCaseN` + +**Network-dependent: tests.noauto_network** + +- Run `unittest tests.noauto_network` + - Need dependencies from `pip install "pythainlp[noauto-network]"` +- Tests requiring network access: + - HuggingFace Hub model downloads + - External API calls +- Requires: Internet connection, may involve large downloads +- Test case class suffix: `TestCaseN` + + ## Robustness tests (test_robustness.py) A comprehensive test suite within core tests that tests edge cases important diff --git a/tests/noauto/__init__.py b/tests/noauto/__init__.py index d7cf7bb3d..78c8b340d 100644 --- a/tests/noauto/__init__.py +++ b/tests/noauto/__init__.py @@ -11,6 +11,16 @@ These tests are NOT run in automated CI workflows but are kept for manual testing and future re-enabling when dependencies improve. + +This test suite serves as an umbrella that includes all specialized +noauto test suites: +- noauto_torch: PyTorch and transformers-based tests +- noauto_tensorflow: TensorFlow-based tests +- noauto_onnx: ONNX Runtime-based tests +- noauto_cython: Cython-compiled package tests +- noauto_network: Network-dependent tests + +For targeted testing, use the specific test suites instead of this umbrella. """ from unittest import TestLoader, TestSuite @@ -18,9 +28,11 @@ # Names of module to be tested # Note: These tests are NOT included in automated CI runs test_packages: list[str] = [ - "tests.noauto.testn_spell", - "tests.noauto.testn_tag", - "tests.noauto.testn_tokenize", + "tests.noauto_torch", + "tests.noauto_tensorflow", + "tests.noauto_onnx", + "tests.noauto_cython", + "tests.noauto_network", ] @@ -29,6 +41,14 @@ def load_tests( ) -> TestSuite: """Load test protocol See: https://docs.python.org/3/library/unittest.html#id1 + + This loads all modular test suites. + For targeted testing, use specific test suites directly: + - unittest tests.noauto_torch + - unittest tests.noauto_tensorflow + - unittest tests.noauto_onnx + - unittest tests.noauto_cython + - unittest tests.noauto_network """ suite = TestSuite() for test_package in test_packages: diff --git a/tests/noauto/testn_tokenize.py b/tests/noauto/testn_tokenize.py deleted file mode 100644 index a2918a180..000000000 --- a/tests/noauto/testn_tokenize.py +++ /dev/null @@ -1,184 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -# Tests for tokenize functions that require TensorFlow, Keras, or transformers -# These tests are NOT run in automated CI workflows due to: -# - Large dependencies (TensorFlow, Keras, transformers, torch) -# - Python 3.13+ compatibility issues - -import unittest - -from pythainlp.tokenize import ( - attacut, - deepcut, - oskut, - paragraph_tokenize, - sefr_cut, - sent_tokenize, - word_dict_trie, - word_tokenize, -) - -from ..core.test_tokenize import ( - SENT_3, - TEXT_1, -) -from ..test_helpers import ( - assert_segment_handles_none_and_empty, - assert_subword_tokenize_basic, -) - - -class DetokenizeTestCaseN(unittest.TestCase): - def test_numeric_data_format(self): - engines = ["attacut", "deepcut", "sefr_cut"] - - for engine in engines: - self.assertIn( - "127.0.0.1", - word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine=engine), - ) - - tokens = word_tokenize( - "เวลา 12:12pm มีโปรโมชั่น 11.11", engine=engine - ) - self.assertTrue( - any(value in tokens for value in ["12:12pm", "12:12"]), - msg=f"{engine}: {tokens}", - ) - self.assertIn("11.11", tokens) - - self.assertIn( - "1,234,567.89", - word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine=engine), - ) - - tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine=engine) - self.assertIn("2.5:1", tokens) - self.assertIn("5:2", tokens) - - # try turning off `join_broken_num` - engine = "attacut" - self.assertNotIn( - "127.0.0.1", - word_tokenize( - "ไอพีของคุณคือ 127.0.0.1 ครับ", - engine=engine, - join_broken_num=False, - ), - ) - self.assertNotIn( - "1,234,567.89", - word_tokenize( - "รางวัลมูลค่า 1,234,567.89 บาท", - engine=engine, - join_broken_num=False, - ), - ) - - -class WordTokenizeAttacutTestCaseN(unittest.TestCase): - def test_word_tokenize_attacut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) - - def test_attacut(self): - assert_segment_handles_none_and_empty(self, attacut.segment) - self.assertEqual( - word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertEqual( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertIsNotNone( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") - ) - - -class WordTokenizeDeepcutTestCaseN(unittest.TestCase): - def test_word_tokenize_deepcut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) - - def test_deepcut(self): - assert_segment_handles_none_and_empty(self, deepcut.segment) - self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) - self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) - self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) - self.assertIsNotNone( - word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() - ) - ) - - -class WordTokenizeOSKutTestCaseN(unittest.TestCase): - def test_word_tokenize_oskut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) - - def test_oskut(self): - assert_segment_handles_none_and_empty(self, oskut.segment) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), - ) - - -class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): - def test_word_tokenize_sefr_cut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) - - def test_sefr_cut(self): - assert_segment_handles_none_and_empty(self, sefr_cut.segment) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), - ) - - -class ParagraphTokenizeTestCaseN(unittest.TestCase): - def test_paragraph_tokenize(self): - sent = ( - "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" - "จากผลงานวิจัยที่เคยทำมาในอดีต" - " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" - " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" - ) - self.assertIsNotNone(paragraph_tokenize(sent)) - with self.assertRaises(ValueError): - paragraph_tokenize( - sent, engine="ai2+2thai" - ) # engine does not exist - - -class SentTokenizeWTPTestCaseN(unittest.TestCase): - def test_sent_tokenize_wtp(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp", - ), - ) - - def test_sent_tokenize_wtp_tiny(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp-tiny", - ), - ) - - -class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): - def test_subword_tokenize_phayathai(self): - assert_subword_tokenize_basic(self, "phayathai") - - -class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): - def test_subword_tokenize_wangchanberta(self): - assert_subword_tokenize_basic(self, "wangchanberta") diff --git a/tests/noauto_cython/__init__.py b/tests/noauto_cython/__init__.py new file mode 100644 index 000000000..92e348e7d --- /dev/null +++ b/tests/noauto_cython/__init__.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for Cython-compiled package functionalities. + +Test functions that require packages that need Cython compilation: +- phunspell (requires Cython and hunspell C library) + +These tests are NOT run in automated CI workflows due to: +- Compilation requirements (Cython, C compiler) +- System library dependencies +- Platform-specific build issues + +These tests are kept for manual testing and may be run in separate CI +workflows with appropriate build environments. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto_cython.testn_spell_cython", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + from unittest import main + + main() diff --git a/tests/noauto/testn_spell.py b/tests/noauto_cython/testn_spell_cython.py similarity index 51% rename from tests/noauto/testn_spell.py rename to tests/noauto_cython/testn_spell_cython.py index 58667a3e3..67f2d7a19 100644 --- a/tests/noauto/testn_spell.py +++ b/tests/noauto_cython/testn_spell_cython.py @@ -2,18 +2,17 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -# Tests for spell functions that require phunspell (Cython) or torch +# Tests for spell functions that require phunspell (Cython) # These tests are NOT run in automated CI workflows due to: -# - Compilation issues (phunspell requires Cython) -# - Large dependencies (torch ~800MB) -# - Python 3.13+ compatibility issues +# - Compilation requirements (Cython, C compiler) +# - System library dependencies (hunspell) +# - Platform-specific build issues import unittest from pythainlp.spell import ( correct, correct_sent, - get_words_spell_suggestion, spell, spell_sent, ) @@ -43,27 +42,3 @@ def test_spell_sent_phunspell(self): def test_correct_sent_phunspell(self): self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) - - -class SpellWanchanbertaTestCaseN(unittest.TestCase): - """Tests for wanchanberta_thai_grammarly engine (requires torch)""" - - def test_word_correct_wanchanberta(self): - result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") - self.assertIsInstance(result, str) - self.assertNotEqual(result, "") - - def test_correct_sent_wanchanberta(self): - from ..core.test_spell import SENT_TOKS - - self.assertIsNotNone( - correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") - ) - - -class SpellHuggingFaceTestCaseN(unittest.TestCase): - """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access)""" - - def test_get_words_spell_suggestion(self): - self.assertIsNotNone(get_words_spell_suggestion("คมดี")) - self.assertIsNotNone(get_words_spell_suggestion(["คมดี", "มะนา"])) diff --git a/tests/noauto_network/__init__.py b/tests/noauto_network/__init__.py new file mode 100644 index 000000000..57b6322ca --- /dev/null +++ b/tests/noauto_network/__init__.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for network-dependent functionalities. + +Test functions that require network access: +- HuggingFace Hub downloads +- Model downloads from remote servers +- API calls to external services + +These tests are NOT run in automated CI workflows due to: +- Network dependency +- Potential for large downloads +- External service availability +- Rate limiting concerns + +These tests are kept for manual testing and may be run in environments +with appropriate network access and caching. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto_network.testn_spell_network", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + from unittest import main + + main() diff --git a/tests/noauto_network/testn_spell_network.py b/tests/noauto_network/testn_spell_network.py new file mode 100644 index 000000000..4f7fc642e --- /dev/null +++ b/tests/noauto_network/testn_spell_network.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require network access +# These tests are NOT run in automated CI workflows due to: +# - Network dependency (HuggingFace Hub downloads) +# - External service availability +# - Rate limiting concerns +# - Potential for large model downloads + +import unittest + +from pythainlp.spell import get_words_spell_suggestion + + +class SpellHuggingFaceTestCaseN(unittest.TestCase): + """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access)""" + + def test_get_words_spell_suggestion(self): + self.assertIsNotNone(get_words_spell_suggestion("คมดี")) + self.assertIsNotNone(get_words_spell_suggestion(["คมดี", "มะนา"])) diff --git a/tests/noauto_onnx/__init__.py b/tests/noauto_onnx/__init__.py new file mode 100644 index 000000000..5f27180ca --- /dev/null +++ b/tests/noauto_onnx/__init__.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for ONNX Runtime-based functionalities. + +Test functions that require ONNX Runtime and its ecosystem dependencies: +- onnxruntime +- oskut +- sefr_cut + +These tests are NOT run in automated CI workflows due to: +- Large dependencies +- Potential compatibility issues across platforms +- Version constraints + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to ONNX Runtime-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto_onnx.testn_tokenize_onnx", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + from unittest import main + + main() diff --git a/tests/noauto_onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py new file mode 100644 index 000000000..fd5bc3748 --- /dev/null +++ b/tests/noauto_onnx/testn_tokenize_onnx.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require ONNX Runtime +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (onnxruntime) +# - Platform-specific compatibility issues +# - Version constraints + +import unittest + +from pythainlp.tokenize import ( + oskut, + sefr_cut, + word_tokenize, +) + +from ..core.test_tokenize import TEXT_1 +from ..test_helpers import assert_segment_handles_none_and_empty + + +class DetokenizeSEFRCutTestCaseN(unittest.TestCase): + """Tests for sefr_cut tokenizer numeric handling (requires onnxruntime)""" + + def test_numeric_data_format_sefr_cut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="sefr_cut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="sefr_cut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"sefr_cut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="sefr_cut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="sefr_cut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + +class WordTokenizeOSKutTestCaseN(unittest.TestCase): + """Tests for oskut tokenizer (requires onnxruntime)""" + + def test_word_tokenize_oskut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) + + def test_oskut(self): + assert_segment_handles_none_and_empty(self, oskut.segment) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), + ) + + +class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): + """Tests for sefr_cut tokenizer (requires onnxruntime)""" + + def test_word_tokenize_sefr_cut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) + + def test_sefr_cut(self): + assert_segment_handles_none_and_empty(self, sefr_cut.segment) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), + ) + + +class TransliterateONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based transliteration (requires onnxruntime)""" + + def test_thai2rom_onnx(self): + from pythainlp.transliterate.thai2rom_onnx import romanize + + result = romanize("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + +class TagONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based POS tagging (requires onnxruntime)""" + + def test_pos_tag_wangchanberta_onnx(self): + from pythainlp.tag import pos_tag + + result = pos_tag( + ["แมว", "กิน", "ปลา"], + engine="wangchanberta_onnx" + ) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + self.assertEqual(len(result), 3) + + +class SpellONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based spell correction (requires onnxruntime)""" + + def test_words_spelling_correction(self): + from pythainlp.spell.words_spelling_correction import ( + get_words_spell_suggestion, + ) + + result = get_words_spell_suggestion("สวัสดี") + self.assertIsInstance(result, list) + diff --git a/tests/noauto_tensorflow/__init__.py b/tests/noauto_tensorflow/__init__.py new file mode 100644 index 000000000..dd71f2b28 --- /dev/null +++ b/tests/noauto_tensorflow/__init__.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for TensorFlow-based functionalities. + +Test functions that require TensorFlow and its ecosystem dependencies: +- tensorflow +- keras +- deepcut + +These tests are NOT run in automated CI workflows due to: +- Very large dependencies (~1-2 GB for tensorflow) +- Potential version conflicts with PyTorch +- Long installation time + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to TensorFlow-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto_tensorflow.testn_tokenize_tensorflow", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + from unittest import main + + main() diff --git a/tests/noauto_tensorflow/testn_tokenize_tensorflow.py b/tests/noauto_tensorflow/testn_tokenize_tensorflow.py new file mode 100644 index 000000000..03d7155c8 --- /dev/null +++ b/tests/noauto_tensorflow/testn_tokenize_tensorflow.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require TensorFlow +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (TensorFlow ~1-2 GB) +# - Potential version conflicts with PyTorch +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tokenize import ( + deepcut, + word_dict_trie, + word_tokenize, +) + +from ..core.test_tokenize import TEXT_1 +from ..test_helpers import assert_segment_handles_none_and_empty + + +class DetokenizeDeepcutTestCaseN(unittest.TestCase): + """Tests for deepcut tokenizer numeric handling (requires tensorflow)""" + + def test_numeric_data_format_deepcut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="deepcut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="deepcut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"deepcut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="deepcut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="deepcut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + +class WordTokenizeDeepcutTestCaseN(unittest.TestCase): + """Tests for deepcut tokenizer (requires tensorflow)""" + + def test_word_tokenize_deepcut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) + + def test_deepcut(self): + assert_segment_handles_none_and_empty(self, deepcut.segment) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) + self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) + self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) + self.assertIsNotNone( + word_tokenize( + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() + ) + ) diff --git a/tests/noauto_torch/__init__.py b/tests/noauto_torch/__init__.py new file mode 100644 index 000000000..675181b6c --- /dev/null +++ b/tests/noauto_torch/__init__.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for PyTorch-based functionalities. + +Test functions that require PyTorch and its ecosystem dependencies: +- torch +- transformers (when using PyTorch backend) +- attacut +- thai-nner +- wtpsplit + +These tests are NOT run in automated CI workflows due to: +- Very large dependencies (~2-3 GB for torch) +- Potential version conflicts with other frameworks +- Long installation time + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to PyTorch-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto_torch.testn_spell_torch", + "tests.noauto_torch.testn_tag_torch", + "tests.noauto_torch.testn_tokenize_torch", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + from unittest import main + + main() diff --git a/tests/noauto_torch/testn_spell_torch.py b/tests/noauto_torch/testn_spell_torch.py new file mode 100644 index 000000000..d1c8378b4 --- /dev/null +++ b/tests/noauto_torch/testn_spell_torch.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require torch and transformers +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (torch ~800MB, transformers) +# - Python 3.13+ compatibility issues +# - Long installation time + +import unittest + +from pythainlp.spell import ( + correct, + correct_sent, +) + +from ..core.test_spell import SENT_TOKS + + +class SpellWanchanbertaTestCaseN(unittest.TestCase): + """Tests for wanchanberta_thai_grammarly engine (requires torch and transformers)""" + + def test_word_correct_wanchanberta(self): + result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") + self.assertIsInstance(result, str) + self.assertNotEqual(result, "") + + def test_correct_sent_wanchanberta(self): + self.assertIsNotNone( + correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") + ) diff --git a/tests/noauto/testn_tag.py b/tests/noauto_torch/testn_tag_torch.py similarity index 97% rename from tests/noauto/testn_tag.py rename to tests/noauto_torch/testn_tag_torch.py index a5825e86a..1c9dbab06 100644 --- a/tests/noauto/testn_tag.py +++ b/tests/noauto_torch/testn_tag_torch.py @@ -2,9 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -# Tests for tag functions that require transformers or torch +# Tests for tag functions that require torch and transformers # These tests are NOT run in automated CI workflows due to: -# - Large dependencies (transformers, torch) +# - Large dependencies (torch, transformers) # - Python 3.13+ compatibility issues import unittest @@ -17,7 +17,7 @@ class TagTransformersTestCaseN(unittest.TestCase): - """Tests for transformers-based engines (requires transformers, torch)""" + """Tests for transformers-based engines (requires transformers and torch)""" def test_NER_class(self): with self.assertRaises(ValueError): diff --git a/tests/noauto_torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py new file mode 100644 index 000000000..ae3591253 --- /dev/null +++ b/tests/noauto_torch/testn_tokenize_torch.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require torch and transformers +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (torch, transformers, attacut) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tokenize import ( + attacut, + paragraph_tokenize, + sent_tokenize, + word_tokenize, +) + +from ..core.test_tokenize import ( + SENT_3, + TEXT_1, +) +from ..test_helpers import ( + assert_segment_handles_none_and_empty, + assert_subword_tokenize_basic, +) + + +class DetokenizeAttacutTestCaseN(unittest.TestCase): + """Tests for attacut tokenizer numeric handling (requires torch)""" + + def test_numeric_data_format_attacut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="attacut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="attacut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"attacut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="attacut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="attacut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + # try turning off `join_broken_num` + self.assertNotIn( + "127.0.0.1", + word_tokenize( + "ไอพีของคุณคือ 127.0.0.1 ครับ", + engine="attacut", + join_broken_num=False, + ), + ) + self.assertNotIn( + "1,234,567.89", + word_tokenize( + "รางวัลมูลค่า 1,234,567.89 บาท", + engine="attacut", + join_broken_num=False, + ), + ) + + +class WordTokenizeAttacutTestCaseN(unittest.TestCase): + """Tests for attacut tokenizer (requires torch)""" + + def test_word_tokenize_attacut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) + + def test_attacut(self): + assert_segment_handles_none_and_empty(self, attacut.segment) + self.assertEqual( + word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertEqual( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertIsNotNone( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") + ) + + +class ParagraphTokenizeTestCaseN(unittest.TestCase): + """Tests for paragraph tokenization (requires transformers)""" + + def test_paragraph_tokenize(self): + sent = ( + "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" + "จากผลงานวิจัยที่เคยทำมาในอดีต" + " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" + " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" + ) + self.assertIsNotNone(paragraph_tokenize(sent)) + with self.assertRaises(ValueError): + paragraph_tokenize( + sent, engine="ai2+2thai" + ) # engine does not exist + + +class SentTokenizeWTPTestCaseN(unittest.TestCase): + """Tests for WTP sentence tokenizer (requires transformers and torch)""" + + def test_sent_tokenize_wtp(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp", + ), + ) + + def test_sent_tokenize_wtp_tiny(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp-tiny", + ), + ) + + +class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): + """Tests for phayathai subword tokenizer (requires transformers)""" + + def test_subword_tokenize_phayathai(self): + assert_subword_tokenize_basic(self, "phayathai") + + +class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): + """Tests for wangchanberta subword tokenizer (requires transformers)""" + + def test_subword_tokenize_wangchanberta(self): + assert_subword_tokenize_basic(self, "wangchanberta") + + +class TransliterateTestCaseN(unittest.TestCase): + """Tests for transliteration functions (requires torch)""" + + def test_thai2rom(self): + from pythainlp.transliterate.thai2rom import romanize + + result = romanize("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_thaig2p(self): + from pythainlp.transliterate.thaig2p import transliterate + + result = transliterate("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_thaig2p_v2(self): + from pythainlp.transliterate.thaig2p_v2 import transliterate + + result = transliterate("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_umt5_thaig2p(self): + from pythainlp.transliterate.umt5_thaig2p import transliterate + + result = transliterate("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + +class ParseTestCaseN(unittest.TestCase): + """Tests for parsing functions (requires torch and transformers)""" + + def test_dependency_parsing(self): + from pythainlp.parse import dependency_parsing + + result = dependency_parsing("แมวกินปลา") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + +class SummarizeTestCaseN(unittest.TestCase): + """Tests for summarization functions (requires transformers)""" + + def test_summarize_keybert(self): + from pythainlp.summarize.keybert import KeyBERT + + text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" + keybert = KeyBERT() + result = keybert.extract_keywords(text, max_keywords=2) + self.assertIsInstance(result, list) + + def test_summarize_mt5(self): + from pythainlp.summarize.mt5 import mT5Summarizer + + text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" + summarizer = mT5Summarizer() + result = summarizer.summarize(text) + self.assertIsInstance(result, list) + + +class AugmentTestCaseN(unittest.TestCase): + """Tests for augmentation functions (requires transformers)""" + + def test_augment_wangchanberta(self): + from pythainlp.augment.lm import Thai2transformersAug + + augmenter = Thai2transformersAug() + result = augmenter.augment("แมวกิน") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_augment_phayathaibert(self): + from pythainlp.augment.lm import ThaiTextAugmenter + + augmenter = ThaiTextAugmenter() + result = augmenter.augment("แมวกิน") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) +