From 68c18ea030559cbdb71b02852cd8200dbfa45b5a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:17:08 +0000 Subject: [PATCH 01/17] Initial plan From a8ed98578e084af6490282b07b33e215d75c4207 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:22:17 +0000 Subject: [PATCH 02/17] Create modular noauto test suites by dependency groups Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 41 +++++ tests/README.md | 99 +++++++++++- tests/noauto-cython/__init__.py | 42 +++++ tests/noauto-cython/testn_spell_cython.py | 44 ++++++ tests/noauto-network/__init__.py | 45 ++++++ tests/noauto-network/testn_spell_network.py | 22 +++ tests/noauto-onnx/__init__.py | 44 ++++++ tests/noauto-onnx/testn_tokenize_onnx.py | 80 ++++++++++ tests/noauto-tensorflow/__init__.py | 44 ++++++ .../testn_tokenize_tensorflow.py | 66 ++++++++ tests/noauto-torch/__init__.py | 49 ++++++ tests/noauto-torch/testn_spell_torch.py | 32 ++++ tests/noauto-torch/testn_tag_torch.py | 99 ++++++++++++ tests/noauto-torch/testn_tokenize_torch.py | 145 ++++++++++++++++++ tests/noauto/__init__.py | 36 +++++ 15 files changed, 880 insertions(+), 8 deletions(-) create mode 100644 tests/noauto-cython/__init__.py create mode 100644 tests/noauto-cython/testn_spell_cython.py create mode 100644 tests/noauto-network/__init__.py create mode 100644 tests/noauto-network/testn_spell_network.py create mode 100644 tests/noauto-onnx/__init__.py create mode 100644 tests/noauto-onnx/testn_tokenize_onnx.py create mode 100644 tests/noauto-tensorflow/__init__.py create mode 100644 tests/noauto-tensorflow/testn_tokenize_tensorflow.py create mode 100644 tests/noauto-torch/__init__.py create mode 100644 tests/noauto-torch/testn_spell_torch.py create mode 100644 tests/noauto-torch/testn_tag_torch.py create mode 100644 tests/noauto-torch/testn_tokenize_torch.py diff --git a/pyproject.toml b/pyproject.toml index d7d905cdb..8d58da7a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,6 +190,47 @@ extra = [ "tltk>=1.10", ] +# Noauto test dependencies - for tests.noauto-* modules +# These are grouped by dependency framework to avoid conflicts + +# PyTorch-based dependencies - for tests.noauto-torch +noauto-torch = [ + "attacut>=1.0.6", + "numpy>=1.26.0", + "sentencepiece>=0.1.91", + "thai-nner>=0.3", + "tltk>=1.10", + "torch>=1.13.1", + "transformers>=4.22.1", + "wtpsplit>=1.0.1", +] + +# TensorFlow-based dependencies - for tests.noauto-tensorflow +noauto-tensorflow = [ + "deepcut>=0.7.0", + "numpy>=1.26.0", +] + +# ONNX Runtime-based dependencies - for tests.noauto-onnx +noauto-onnx = [ + "numpy>=1.26.0", + "onnxruntime>=1.10.0", + "oskut>=1.3", + "sefr_cut>=1.1", +] + +# Cython-based dependencies - for tests.noauto-cython +noauto-cython = [ + "phunspell>=0.1.6", +] + +# Network-dependent tests - for tests.noauto-network +# These tests require network access but minimal dependencies +noauto-network = [ + "huggingface-hub>=0.16.0", +] + + # Full dependencies - pinned where available full = [ "attacut==1.0.6", diff --git a/tests/README.md b/tests/README.md index 02af5fd8f..54b0211fe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -74,16 +74,99 @@ The CI/CD test workflow is at ## Noauto tests (testn_*.py) -- These dependencies might include huge libraries like `tensorflow`. -- Due to dependency complexities, these functionalities may not be tested - in the CI/CD pipeline. - - In the future, we might create a separate - step or workflow to run this test suite. - It will be triggered manually. - We may also need to group test cases by - a non-conflicting set of dependencies. +The noauto (no-automated) test suite contains tests for functionalities +that require heavy dependencies which are not feasible to run in automated +CI/CD pipelines. These tests are organized into specialized suites based +on their dependency requirements. + +### Why separate noauto test suites? + +Different ML/AI frameworks often have conflicting version requirements for +their dependencies. For example: +- PyTorch and TensorFlow may require different versions of numpy or protobuf +- Large frameworks take significant time to install (~1-3 GB each) +- Some packages require Cython compilation or system libraries + +By separating tests by dependency group, we can: +- Test each framework independently without conflicts +- Optimize CI/CD resources by running only relevant test groups +- Make it easier for developers to test specific functionality + +### Noauto test suites + +#### Umbrella suite: tests.noauto + +- Run `unittest tests.noauto` +- Includes all noauto test suites (legacy and new modular suites) +- Use this for comprehensive testing when all dependencies are available +- Test case class suffix: `TestCaseN` + +#### Modular suites by dependency: + +**PyTorch-based: tests.noauto-torch** + +- Run `unittest tests.noauto-torch` + - Need dependencies from `pip install "pythainlp[noauto-torch]"` +- Tests requiring PyTorch and its ecosystem: + - torch, transformers (PyTorch backend), sentence-transformers + - attacut, thai-nner, wtpsplit, tltk +- Tests: spell correction (wanchanberta), NER/POS tagging (transformers-based), + tokenization (attacut), subword tokenization (phayathai, wangchanberta), + sentence tokenization (wtp) +- Dependencies: ~2-3 GB +- Test case class suffix: `TestCaseN` + +**TensorFlow-based: tests.noauto-tensorflow** + +- Run `unittest tests.noauto-tensorflow` + - Need dependencies from `pip install "pythainlp[noauto-tensorflow]"` +- Tests requiring TensorFlow: + - deepcut tokenizer +- Dependencies: ~1-2 GB +- Note: May conflict with PyTorch dependencies - Test case class suffix: `TestCaseN` +**ONNX Runtime-based: tests.noauto-onnx** + +- Run `unittest tests.noauto-onnx` + - Need dependencies from `pip install "pythainlp[noauto-onnx]"` +- Tests requiring ONNX Runtime: + - oskut, sefr_cut tokenizers +- Dependencies: ~200-500 MB +- Test case class suffix: `TestCaseN` + +**Cython-compiled: tests.noauto-cython** + +- Run `unittest tests.noauto-cython` + - Need dependencies from `pip install "pythainlp[noauto-cython]"` +- Tests requiring Cython-compiled packages: + - phunspell spell checker +- Requires: Cython, C compiler, system libraries (hunspell) +- Platform-specific build requirements +- Test case class suffix: `TestCaseN` + +**Network-dependent: tests.noauto-network** + +- Run `unittest tests.noauto-network` + - Need dependencies from `pip install "pythainlp[noauto-network]"` +- Tests requiring network access: + - HuggingFace Hub model downloads + - External API calls +- Requires: Internet connection, may involve large downloads +- Test case class suffix: `TestCaseN` + +### Legacy noauto tests (deprecated) + +The original test files in `tests/noauto/` directory are kept for backward +compatibility but are deprecated: +- `testn_spell.py` +- `testn_tag.py` +- `testn_tokenize.py` + +New tests should be added to the appropriate modular test suite +(`tests.noauto-torch/`, `tests.noauto-tensorflow/`, etc.) instead of +the legacy files. + ## Robustness tests (test_robustness.py) A comprehensive test suite within core tests that tests edge cases important diff --git a/tests/noauto-cython/__init__.py b/tests/noauto-cython/__init__.py new file mode 100644 index 000000000..2757d4405 --- /dev/null +++ b/tests/noauto-cython/__init__.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for Cython-compiled package functionalities. + +Test functions that require packages that need Cython compilation: +- phunspell (requires Cython and hunspell C library) + +These tests are NOT run in automated CI workflows due to: +- Compilation requirements (Cython, C compiler) +- System library dependencies +- Platform-specific build issues + +These tests are kept for manual testing and may be run in separate CI +workflows with appropriate build environments. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto-cython.testn_spell_cython", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto-cython/testn_spell_cython.py b/tests/noauto-cython/testn_spell_cython.py new file mode 100644 index 000000000..67f2d7a19 --- /dev/null +++ b/tests/noauto-cython/testn_spell_cython.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require phunspell (Cython) +# These tests are NOT run in automated CI workflows due to: +# - Compilation requirements (Cython, C compiler) +# - System library dependencies (hunspell) +# - Platform-specific build issues + +import unittest + +from pythainlp.spell import ( + correct, + correct_sent, + spell, + spell_sent, +) + +from ..core.test_spell import SENT_TOKS + + +class SpellPhunspellTestCaseN(unittest.TestCase): + """Tests for phunspell engine (requires Cython compilation)""" + + def test_spell_phunspell(self): + result = spell("เน้ร", engine="phunspell") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = spell("เกสมร์", engine="phunspell") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_word_correct_phunspell(self): + result = correct("ทดสอง", engine="phunspell") + self.assertIsInstance(result, str) + self.assertNotEqual(result, "") + + def test_spell_sent_phunspell(self): + self.assertIsNotNone(spell_sent(SENT_TOKS, engine="phunspell")) + + def test_correct_sent_phunspell(self): + self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) diff --git a/tests/noauto-network/__init__.py b/tests/noauto-network/__init__.py new file mode 100644 index 000000000..2795f2311 --- /dev/null +++ b/tests/noauto-network/__init__.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for network-dependent functionalities. + +Test functions that require network access: +- HuggingFace Hub downloads +- Model downloads from remote servers +- API calls to external services + +These tests are NOT run in automated CI workflows due to: +- Network dependency +- Potential for large downloads +- External service availability +- Rate limiting concerns + +These tests are kept for manual testing and may be run in environments +with appropriate network access and caching. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto-network.testn_spell_network", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto-network/testn_spell_network.py b/tests/noauto-network/testn_spell_network.py new file mode 100644 index 000000000..4f7fc642e --- /dev/null +++ b/tests/noauto-network/testn_spell_network.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require network access +# These tests are NOT run in automated CI workflows due to: +# - Network dependency (HuggingFace Hub downloads) +# - External service availability +# - Rate limiting concerns +# - Potential for large model downloads + +import unittest + +from pythainlp.spell import get_words_spell_suggestion + + +class SpellHuggingFaceTestCaseN(unittest.TestCase): + """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access)""" + + def test_get_words_spell_suggestion(self): + self.assertIsNotNone(get_words_spell_suggestion("คมดี")) + self.assertIsNotNone(get_words_spell_suggestion(["คมดี", "มะนา"])) diff --git a/tests/noauto-onnx/__init__.py b/tests/noauto-onnx/__init__.py new file mode 100644 index 000000000..84e670ecc --- /dev/null +++ b/tests/noauto-onnx/__init__.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for ONNX Runtime-based functionalities. + +Test functions that require ONNX Runtime and its ecosystem dependencies: +- onnxruntime +- oskut +- sefr_cut + +These tests are NOT run in automated CI workflows due to: +- Large dependencies +- Potential compatibility issues across platforms +- Version constraints + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to ONNX Runtime-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto-onnx.testn_tokenize_onnx", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto-onnx/testn_tokenize_onnx.py b/tests/noauto-onnx/testn_tokenize_onnx.py new file mode 100644 index 000000000..d581b0b79 --- /dev/null +++ b/tests/noauto-onnx/testn_tokenize_onnx.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require ONNX Runtime +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (onnxruntime) +# - Platform-specific compatibility issues +# - Version constraints + +import unittest + +from pythainlp.tokenize import ( + oskut, + sefr_cut, + word_tokenize, +) + +from ..core.test_tokenize import TEXT_1 +from ..test_helpers import assert_segment_handles_none_and_empty + + +class DetokenizeSEFRCutTestCaseN(unittest.TestCase): + """Tests for sefr_cut tokenizer numeric handling (requires onnxruntime)""" + + def test_numeric_data_format_sefr_cut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="sefr_cut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="sefr_cut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"sefr_cut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="sefr_cut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="sefr_cut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + +class WordTokenizeOSKutTestCaseN(unittest.TestCase): + """Tests for oskut tokenizer (requires onnxruntime)""" + + def test_word_tokenize_oskut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) + + def test_oskut(self): + assert_segment_handles_none_and_empty(self, oskut.segment) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), + ) + + +class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): + """Tests for sefr_cut tokenizer (requires onnxruntime)""" + + def test_word_tokenize_sefr_cut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) + + def test_sefr_cut(self): + assert_segment_handles_none_and_empty(self, sefr_cut.segment) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), + ) diff --git a/tests/noauto-tensorflow/__init__.py b/tests/noauto-tensorflow/__init__.py new file mode 100644 index 000000000..81bc46445 --- /dev/null +++ b/tests/noauto-tensorflow/__init__.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for TensorFlow-based functionalities. + +Test functions that require TensorFlow and its ecosystem dependencies: +- tensorflow +- keras +- deepcut + +These tests are NOT run in automated CI workflows due to: +- Very large dependencies (~1-2 GB for tensorflow) +- Potential version conflicts with PyTorch +- Long installation time + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to TensorFlow-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto-tensorflow.testn_tokenize_tensorflow", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto-tensorflow/testn_tokenize_tensorflow.py b/tests/noauto-tensorflow/testn_tokenize_tensorflow.py new file mode 100644 index 000000000..03d7155c8 --- /dev/null +++ b/tests/noauto-tensorflow/testn_tokenize_tensorflow.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require TensorFlow +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (TensorFlow ~1-2 GB) +# - Potential version conflicts with PyTorch +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tokenize import ( + deepcut, + word_dict_trie, + word_tokenize, +) + +from ..core.test_tokenize import TEXT_1 +from ..test_helpers import assert_segment_handles_none_and_empty + + +class DetokenizeDeepcutTestCaseN(unittest.TestCase): + """Tests for deepcut tokenizer numeric handling (requires tensorflow)""" + + def test_numeric_data_format_deepcut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="deepcut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="deepcut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"deepcut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="deepcut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="deepcut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + +class WordTokenizeDeepcutTestCaseN(unittest.TestCase): + """Tests for deepcut tokenizer (requires tensorflow)""" + + def test_word_tokenize_deepcut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) + + def test_deepcut(self): + assert_segment_handles_none_and_empty(self, deepcut.segment) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) + self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) + self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) + self.assertIsNotNone( + word_tokenize( + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() + ) + ) diff --git a/tests/noauto-torch/__init__.py b/tests/noauto-torch/__init__.py new file mode 100644 index 000000000..de452d11b --- /dev/null +++ b/tests/noauto-torch/__init__.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for PyTorch-based functionalities. + +Test functions that require PyTorch and its ecosystem dependencies: +- torch +- transformers (when using PyTorch backend) +- sentence-transformers +- attacut +- thai_nner +- wtpsplit + +These tests are NOT run in automated CI workflows due to: +- Very large dependencies (~2-3 GB for torch) +- Potential version conflicts with other frameworks +- Long installation time + +These tests are kept for manual testing and may be run in separate CI +workflows dedicated to PyTorch-based features. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +test_packages: list[str] = [ + "tests.noauto-torch.testn_spell_torch", + "tests.noauto-torch.testn_tag_torch", + "tests.noauto-torch.testn_tokenize_torch", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto-torch/testn_spell_torch.py b/tests/noauto-torch/testn_spell_torch.py new file mode 100644 index 000000000..d1c8378b4 --- /dev/null +++ b/tests/noauto-torch/testn_spell_torch.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require torch and transformers +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (torch ~800MB, transformers) +# - Python 3.13+ compatibility issues +# - Long installation time + +import unittest + +from pythainlp.spell import ( + correct, + correct_sent, +) + +from ..core.test_spell import SENT_TOKS + + +class SpellWanchanbertaTestCaseN(unittest.TestCase): + """Tests for wanchanberta_thai_grammarly engine (requires torch and transformers)""" + + def test_word_correct_wanchanberta(self): + result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") + self.assertIsInstance(result, str) + self.assertNotEqual(result, "") + + def test_correct_sent_wanchanberta(self): + self.assertIsNotNone( + correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") + ) diff --git a/tests/noauto-torch/testn_tag_torch.py b/tests/noauto-torch/testn_tag_torch.py new file mode 100644 index 000000000..1c9dbab06 --- /dev/null +++ b/tests/noauto-torch/testn_tag_torch.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tag functions that require torch and transformers +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (torch, transformers) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tag import ( + NER, + NNER, + pos_tag_transformers, +) + + +class TagTransformersTestCaseN(unittest.TestCase): + """Tests for transformers-based engines (requires transformers and torch)""" + + def test_NER_class(self): + with self.assertRaises(ValueError): + NER(engine="thainer", corpus="cat") + + ner = NER(engine="thainer") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="thainer-v2") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="wangchanberta") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="tltk") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + # Test thai-nner engine + ner = NER(engine="thai-nner") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + def test_NNER_class(self): + nner = NNER() + # Test basic tagging + self.assertIsNotNone(nner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + + # Test with top_level_only parameter + tokens, entities = nner.tag("แมวทำอะไรตอนห้าโมงเช้า") + self.assertIsInstance(tokens, list) + self.assertIsInstance(entities, list) + + tokens_top, entities_top = nner.tag("แมวทำอะไรตอนห้าโมงเช้า", top_level_only=True) + self.assertIsInstance(tokens_top, list) + self.assertIsInstance(entities_top, list) + # Top-level entities should be less than or equal to all entities + self.assertLessEqual(len(entities_top), len(entities)) + + def test_pos_tag_transformers(self): + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="bert", + corpus="blackboard", + ) + ) + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="mdeberta", + corpus="pud", + ) + ) + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="wangchanberta", + corpus="pud", + ) + ) + with self.assertRaises(ValueError): + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", engine="non-existing-engine" + ) + with self.assertRaises(ValueError): + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="bert", + corpus="non-existing corpus", + ) diff --git a/tests/noauto-torch/testn_tokenize_torch.py b/tests/noauto-torch/testn_tokenize_torch.py new file mode 100644 index 000000000..0829b74fa --- /dev/null +++ b/tests/noauto-torch/testn_tokenize_torch.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require torch and transformers +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (torch, transformers, attacut) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tokenize import ( + attacut, + paragraph_tokenize, + sent_tokenize, + word_dict_trie, + word_tokenize, +) + +from ..core.test_tokenize import ( + SENT_3, + TEXT_1, +) +from ..test_helpers import ( + assert_segment_handles_none_and_empty, + assert_subword_tokenize_basic, +) + + +class DetokenizeAttacutTestCaseN(unittest.TestCase): + """Tests for attacut tokenizer numeric handling (requires torch)""" + + def test_numeric_data_format_attacut(self): + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine="attacut"), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine="attacut" + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"attacut: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine="attacut"), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine="attacut") + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + # try turning off `join_broken_num` + self.assertNotIn( + "127.0.0.1", + word_tokenize( + "ไอพีของคุณคือ 127.0.0.1 ครับ", + engine="attacut", + join_broken_num=False, + ), + ) + self.assertNotIn( + "1,234,567.89", + word_tokenize( + "รางวัลมูลค่า 1,234,567.89 บาท", + engine="attacut", + join_broken_num=False, + ), + ) + + +class WordTokenizeAttacutTestCaseN(unittest.TestCase): + """Tests for attacut tokenizer (requires torch)""" + + def test_word_tokenize_attacut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) + + def test_attacut(self): + assert_segment_handles_none_and_empty(self, attacut.segment) + self.assertEqual( + word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertEqual( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertIsNotNone( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") + ) + + +class ParagraphTokenizeTestCaseN(unittest.TestCase): + """Tests for paragraph tokenization (requires transformers)""" + + def test_paragraph_tokenize(self): + sent = ( + "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" + "จากผลงานวิจัยที่เคยทำมาในอดีต" + " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" + " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" + ) + self.assertIsNotNone(paragraph_tokenize(sent)) + with self.assertRaises(ValueError): + paragraph_tokenize( + sent, engine="ai2+2thai" + ) # engine does not exist + + +class SentTokenizeWTPTestCaseN(unittest.TestCase): + """Tests for WTP sentence tokenizer (requires transformers and torch)""" + + def test_sent_tokenize_wtp(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp", + ), + ) + + def test_sent_tokenize_wtp_tiny(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp-tiny", + ), + ) + + +class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): + """Tests for phayathai subword tokenizer (requires transformers)""" + + def test_subword_tokenize_phayathai(self): + assert_subword_tokenize_basic(self, "phayathai") + + +class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): + """Tests for wangchanberta subword tokenizer (requires transformers)""" + + def test_subword_tokenize_wangchanberta(self): + assert_subword_tokenize_basic(self, "wangchanberta") diff --git a/tests/noauto/__init__.py b/tests/noauto/__init__.py index d7cf7bb3d..c8ae65499 100644 --- a/tests/noauto/__init__.py +++ b/tests/noauto/__init__.py @@ -11,29 +11,65 @@ These tests are NOT run in automated CI workflows but are kept for manual testing and future re-enabling when dependencies improve. + +This test suite serves as an umbrella that includes all specialized +noauto test suites: +- noauto-torch: PyTorch and transformers-based tests +- noauto-tensorflow: TensorFlow-based tests +- noauto-onnx: ONNX Runtime-based tests +- noauto-cython: Cython-compiled package tests +- noauto-network: Network-dependent tests + +For targeted testing, use the specific test suites instead of this umbrella. """ from unittest import TestLoader, TestSuite # Names of module to be tested # Note: These tests are NOT included in automated CI runs +# Legacy test files (deprecated, kept for backward compatibility) test_packages: list[str] = [ "tests.noauto.testn_spell", "tests.noauto.testn_tag", "tests.noauto.testn_tokenize", ] +# New modular test suites by dependency group +test_packages_modular: list[str] = [ + "tests.noauto-torch", + "tests.noauto-tensorflow", + "tests.noauto-onnx", + "tests.noauto-cython", + "tests.noauto-network", +] + def load_tests( loader: TestLoader, standard_tests: TestSuite, pattern: str ) -> TestSuite: """Load test protocol See: https://docs.python.org/3/library/unittest.html#id1 + + This loads both legacy and new modular test suites. + For targeted testing, use specific test suites directly: + - unittest tests.noauto-torch + - unittest tests.noauto-tensorflow + - unittest tests.noauto-onnx + - unittest tests.noauto-cython + - unittest tests.noauto-network """ suite = TestSuite() + + # Load legacy test files (for backward compatibility) for test_package in test_packages: tests = loader.loadTestsFromName(test_package) suite.addTests(tests) + + # Load new modular test suites + for test_package in test_packages_modular: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite From e74b91221520aac8eadb3665050d196cb5f30fb4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:24:31 +0000 Subject: [PATCH 03/17] Add deprecation notices to legacy noauto test files Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/noauto/testn_spell.py | 21 ++++++++++++++++++--- tests/noauto/testn_tag.py | 8 +++++++- tests/noauto/testn_tokenize.py | 24 ++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/tests/noauto/testn_spell.py b/tests/noauto/testn_spell.py index 58667a3e3..9bdcae616 100644 --- a/tests/noauto/testn_spell.py +++ b/tests/noauto/testn_spell.py @@ -2,6 +2,12 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +# DEPRECATED: This file is kept for backward compatibility only. +# New tests should be added to the appropriate modular test suite: +# - tests.noauto-cython for phunspell tests +# - tests.noauto-torch for wanchanberta tests +# - tests.noauto-network for HuggingFace Hub tests + # Tests for spell functions that require phunspell (Cython) or torch # These tests are NOT run in automated CI workflows due to: # - Compilation issues (phunspell requires Cython) @@ -22,7 +28,10 @@ class SpellPhunspellTestCaseN(unittest.TestCase): - """Tests for phunspell engine (requires Cython compilation)""" + """Tests for phunspell engine (requires Cython compilation) + + DEPRECATED: Moved to tests.noauto-cython.testn_spell_cython + """ def test_spell_phunspell(self): result = spell("เน้ร", engine="phunspell") @@ -46,7 +55,10 @@ def test_correct_sent_phunspell(self): class SpellWanchanbertaTestCaseN(unittest.TestCase): - """Tests for wanchanberta_thai_grammarly engine (requires torch)""" + """Tests for wanchanberta_thai_grammarly engine (requires torch) + + DEPRECATED: Moved to tests.noauto-torch.testn_spell_torch + """ def test_word_correct_wanchanberta(self): result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") @@ -62,7 +74,10 @@ def test_correct_sent_wanchanberta(self): class SpellHuggingFaceTestCaseN(unittest.TestCase): - """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access)""" + """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access) + + DEPRECATED: Moved to tests.noauto-network.testn_spell_network + """ def test_get_words_spell_suggestion(self): self.assertIsNotNone(get_words_spell_suggestion("คมดี")) diff --git a/tests/noauto/testn_tag.py b/tests/noauto/testn_tag.py index a5825e86a..c6c73b3d0 100644 --- a/tests/noauto/testn_tag.py +++ b/tests/noauto/testn_tag.py @@ -2,6 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +# DEPRECATED: This file is kept for backward compatibility only. +# New tests should be added to tests.noauto-torch for transformers-based tests. + # Tests for tag functions that require transformers or torch # These tests are NOT run in automated CI workflows due to: # - Large dependencies (transformers, torch) @@ -17,7 +20,10 @@ class TagTransformersTestCaseN(unittest.TestCase): - """Tests for transformers-based engines (requires transformers, torch)""" + """Tests for transformers-based engines (requires transformers, torch) + + DEPRECATED: Moved to tests.noauto-torch.testn_tag_torch + """ def test_NER_class(self): with self.assertRaises(ValueError): diff --git a/tests/noauto/testn_tokenize.py b/tests/noauto/testn_tokenize.py index a2918a180..e7b8ac2cc 100644 --- a/tests/noauto/testn_tokenize.py +++ b/tests/noauto/testn_tokenize.py @@ -2,6 +2,12 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +# DEPRECATED: This file is kept for backward compatibility only. +# New tests should be added to the appropriate modular test suite: +# - tests.noauto-torch for attacut, wtp, subword tokenizers +# - tests.noauto-tensorflow for deepcut +# - tests.noauto-onnx for oskut, sefr_cut + # Tests for tokenize functions that require TensorFlow, Keras, or transformers # These tests are NOT run in automated CI workflows due to: # - Large dependencies (TensorFlow, Keras, transformers, torch) @@ -31,6 +37,8 @@ class DetokenizeTestCaseN(unittest.TestCase): + """DEPRECATED: Tests split across noauto-torch, noauto-tensorflow, and noauto-onnx""" + def test_numeric_data_format(self): engines = ["attacut", "deepcut", "sefr_cut"] @@ -79,6 +87,8 @@ def test_numeric_data_format(self): class WordTokenizeAttacutTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" + def test_word_tokenize_attacut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) @@ -98,6 +108,8 @@ def test_attacut(self): class WordTokenizeDeepcutTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-tensorflow.testn_tokenize_tensorflow""" + def test_word_tokenize_deepcut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) @@ -114,6 +126,8 @@ def test_deepcut(self): class WordTokenizeOSKutTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-onnx.testn_tokenize_onnx""" + def test_word_tokenize_oskut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) @@ -128,6 +142,8 @@ def test_oskut(self): class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-onnx.testn_tokenize_onnx""" + def test_word_tokenize_sefr_cut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) @@ -142,6 +158,8 @@ def test_sefr_cut(self): class ParagraphTokenizeTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" + def test_paragraph_tokenize(self): sent = ( "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" @@ -157,6 +175,8 @@ def test_paragraph_tokenize(self): class SentTokenizeWTPTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" + def test_sent_tokenize_wtp(self): self.assertIsNotNone( sent_tokenize( @@ -175,10 +195,14 @@ def test_sent_tokenize_wtp_tiny(self): class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" + def test_subword_tokenize_phayathai(self): assert_subword_tokenize_basic(self, "phayathai") class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): + """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" + def test_subword_tokenize_wangchanberta(self): assert_subword_tokenize_basic(self, "wangchanberta") From 91523b5b6a5f11b7b48ba7ff83ce3e867602ce22 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:26:03 +0000 Subject: [PATCH 04/17] Address code review comments and add CHANGELOG entry Co-authored-by: bact <128572+bact@users.noreply.github.com> --- CHANGELOG.md | 1 + tests/noauto/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0448996e..b722340b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ See PR for prompt and details. - Make package zip-safe #1212 - Ensure thread-safety for tokenizers #1213 - Add Thai-NNER integration with top-level entity filtering #1221 +- Reorganize noauto test suite by dependency groups (torch, tensorflow, onnx, cython, network) #935 - Improved documentation; code cleanup; more tests ## Version 5.1.2 -> 5.2.0 diff --git a/tests/noauto/__init__.py b/tests/noauto/__init__.py index c8ae65499..83075a552 100644 --- a/tests/noauto/__init__.py +++ b/tests/noauto/__init__.py @@ -35,7 +35,7 @@ ] # New modular test suites by dependency group -test_packages_modular: list[str] = [ +modular_test_packages: list[str] = [ "tests.noauto-torch", "tests.noauto-tensorflow", "tests.noauto-onnx", @@ -66,7 +66,7 @@ def load_tests( suite.addTests(tests) # Load new modular test suites - for test_package in test_packages_modular: + for test_package in modular_test_packages: tests = loader.loadTestsFromName(test_package) suite.addTests(tests) From e70a20c72973a451d5f0dd183f17aea106b7076a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:30:10 +0000 Subject: [PATCH 05/17] Fix: Use underscores instead of hyphens in test directory names and remove legacy test files Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/README.md | 41 ++-- tests/noauto/__init__.py | 48 ++-- tests/noauto/testn_spell.py | 84 ------- tests/noauto/testn_tag.py | 105 --------- tests/noauto/testn_tokenize.py | 208 ------------------ .../__init__.py | 2 +- .../testn_spell_cython.py | 0 .../__init__.py | 2 +- .../testn_spell_network.py | 0 .../{noauto-onnx => noauto_onnx}/__init__.py | 2 +- .../testn_tokenize_onnx.py | 0 .../__init__.py | 2 +- .../testn_tokenize_tensorflow.py | 0 .../__init__.py | 6 +- .../testn_spell_torch.py | 0 .../testn_tag_torch.py | 0 .../testn_tokenize_torch.py | 0 17 files changed, 38 insertions(+), 462 deletions(-) delete mode 100644 tests/noauto/testn_spell.py delete mode 100644 tests/noauto/testn_tag.py delete mode 100644 tests/noauto/testn_tokenize.py rename tests/{noauto-cython => noauto_cython}/__init__.py (96%) rename tests/{noauto-cython => noauto_cython}/testn_spell_cython.py (100%) rename tests/{noauto-network => noauto_network}/__init__.py (95%) rename tests/{noauto-network => noauto_network}/testn_spell_network.py (100%) rename tests/{noauto-onnx => noauto_onnx}/__init__.py (96%) rename tests/{noauto-onnx => noauto_onnx}/testn_tokenize_onnx.py (100%) rename tests/{noauto-tensorflow => noauto_tensorflow}/__init__.py (95%) rename tests/{noauto-tensorflow => noauto_tensorflow}/testn_tokenize_tensorflow.py (100%) rename tests/{noauto-torch => noauto_torch}/__init__.py (89%) rename tests/{noauto-torch => noauto_torch}/testn_spell_torch.py (100%) rename tests/{noauto-torch => noauto_torch}/testn_tag_torch.py (100%) rename tests/{noauto-torch => noauto_torch}/testn_tokenize_torch.py (100%) diff --git a/tests/README.md b/tests/README.md index 54b0211fe..753dbdecb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -103,10 +103,10 @@ By separating tests by dependency group, we can: #### Modular suites by dependency: -**PyTorch-based: tests.noauto-torch** +**PyTorch-based: tests.noauto_torch** -- Run `unittest tests.noauto-torch` - - Need dependencies from `pip install "pythainlp[noauto-torch]"` +- Run `unittest tests.noauto_torch` + - Need dependencies from `pip install "pythainlp[noauto_torch]"` - Tests requiring PyTorch and its ecosystem: - torch, transformers (PyTorch backend), sentence-transformers - attacut, thai-nner, wtpsplit, tltk @@ -116,56 +116,45 @@ By separating tests by dependency group, we can: - Dependencies: ~2-3 GB - Test case class suffix: `TestCaseN` -**TensorFlow-based: tests.noauto-tensorflow** +**TensorFlow-based: tests.noauto_tensorflow** -- Run `unittest tests.noauto-tensorflow` - - Need dependencies from `pip install "pythainlp[noauto-tensorflow]"` +- Run `unittest tests.noauto_tensorflow` + - Need dependencies from `pip install "pythainlp[noauto_tensorflow]"` - Tests requiring TensorFlow: - deepcut tokenizer - Dependencies: ~1-2 GB - Note: May conflict with PyTorch dependencies - Test case class suffix: `TestCaseN` -**ONNX Runtime-based: tests.noauto-onnx** +**ONNX Runtime-based: tests.noauto_onnx** -- Run `unittest tests.noauto-onnx` - - Need dependencies from `pip install "pythainlp[noauto-onnx]"` +- Run `unittest tests.noauto_onnx` + - Need dependencies from `pip install "pythainlp[noauto_onnx]"` - Tests requiring ONNX Runtime: - oskut, sefr_cut tokenizers - Dependencies: ~200-500 MB - Test case class suffix: `TestCaseN` -**Cython-compiled: tests.noauto-cython** +**Cython-compiled: tests.noauto_cython** -- Run `unittest tests.noauto-cython` - - Need dependencies from `pip install "pythainlp[noauto-cython]"` +- Run `unittest tests.noauto_cython` + - Need dependencies from `pip install "pythainlp[noauto_cython]"` - Tests requiring Cython-compiled packages: - phunspell spell checker - Requires: Cython, C compiler, system libraries (hunspell) - Platform-specific build requirements - Test case class suffix: `TestCaseN` -**Network-dependent: tests.noauto-network** +**Network-dependent: tests.noauto_network** -- Run `unittest tests.noauto-network` - - Need dependencies from `pip install "pythainlp[noauto-network]"` +- Run `unittest tests.noauto_network` + - Need dependencies from `pip install "pythainlp[noauto_network]"` - Tests requiring network access: - HuggingFace Hub model downloads - External API calls - Requires: Internet connection, may involve large downloads - Test case class suffix: `TestCaseN` -### Legacy noauto tests (deprecated) - -The original test files in `tests/noauto/` directory are kept for backward -compatibility but are deprecated: -- `testn_spell.py` -- `testn_tag.py` -- `testn_tokenize.py` - -New tests should be added to the appropriate modular test suite -(`tests.noauto-torch/`, `tests.noauto-tensorflow/`, etc.) instead of -the legacy files. ## Robustness tests (test_robustness.py) diff --git a/tests/noauto/__init__.py b/tests/noauto/__init__.py index 83075a552..78c8b340d 100644 --- a/tests/noauto/__init__.py +++ b/tests/noauto/__init__.py @@ -14,11 +14,11 @@ This test suite serves as an umbrella that includes all specialized noauto test suites: -- noauto-torch: PyTorch and transformers-based tests -- noauto-tensorflow: TensorFlow-based tests -- noauto-onnx: ONNX Runtime-based tests -- noauto-cython: Cython-compiled package tests -- noauto-network: Network-dependent tests +- noauto_torch: PyTorch and transformers-based tests +- noauto_tensorflow: TensorFlow-based tests +- noauto_onnx: ONNX Runtime-based tests +- noauto_cython: Cython-compiled package tests +- noauto_network: Network-dependent tests For targeted testing, use the specific test suites instead of this umbrella. """ @@ -27,20 +27,12 @@ # Names of module to be tested # Note: These tests are NOT included in automated CI runs -# Legacy test files (deprecated, kept for backward compatibility) test_packages: list[str] = [ - "tests.noauto.testn_spell", - "tests.noauto.testn_tag", - "tests.noauto.testn_tokenize", -] - -# New modular test suites by dependency group -modular_test_packages: list[str] = [ - "tests.noauto-torch", - "tests.noauto-tensorflow", - "tests.noauto-onnx", - "tests.noauto-cython", - "tests.noauto-network", + "tests.noauto_torch", + "tests.noauto_tensorflow", + "tests.noauto_onnx", + "tests.noauto_cython", + "tests.noauto_network", ] @@ -50,26 +42,18 @@ def load_tests( """Load test protocol See: https://docs.python.org/3/library/unittest.html#id1 - This loads both legacy and new modular test suites. + This loads all modular test suites. For targeted testing, use specific test suites directly: - - unittest tests.noauto-torch - - unittest tests.noauto-tensorflow - - unittest tests.noauto-onnx - - unittest tests.noauto-cython - - unittest tests.noauto-network + - unittest tests.noauto_torch + - unittest tests.noauto_tensorflow + - unittest tests.noauto_onnx + - unittest tests.noauto_cython + - unittest tests.noauto_network """ suite = TestSuite() - - # Load legacy test files (for backward compatibility) for test_package in test_packages: tests = loader.loadTestsFromName(test_package) suite.addTests(tests) - - # Load new modular test suites - for test_package in modular_test_packages: - tests = loader.loadTestsFromName(test_package) - suite.addTests(tests) - return suite diff --git a/tests/noauto/testn_spell.py b/tests/noauto/testn_spell.py deleted file mode 100644 index 9bdcae616..000000000 --- a/tests/noauto/testn_spell.py +++ /dev/null @@ -1,84 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -# DEPRECATED: This file is kept for backward compatibility only. -# New tests should be added to the appropriate modular test suite: -# - tests.noauto-cython for phunspell tests -# - tests.noauto-torch for wanchanberta tests -# - tests.noauto-network for HuggingFace Hub tests - -# Tests for spell functions that require phunspell (Cython) or torch -# These tests are NOT run in automated CI workflows due to: -# - Compilation issues (phunspell requires Cython) -# - Large dependencies (torch ~800MB) -# - Python 3.13+ compatibility issues - -import unittest - -from pythainlp.spell import ( - correct, - correct_sent, - get_words_spell_suggestion, - spell, - spell_sent, -) - -from ..core.test_spell import SENT_TOKS - - -class SpellPhunspellTestCaseN(unittest.TestCase): - """Tests for phunspell engine (requires Cython compilation) - - DEPRECATED: Moved to tests.noauto-cython.testn_spell_cython - """ - - def test_spell_phunspell(self): - result = spell("เน้ร", engine="phunspell") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - result = spell("เกสมร์", engine="phunspell") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - def test_word_correct_phunspell(self): - result = correct("ทดสอง", engine="phunspell") - self.assertIsInstance(result, str) - self.assertNotEqual(result, "") - - def test_spell_sent_phunspell(self): - self.assertIsNotNone(spell_sent(SENT_TOKS, engine="phunspell")) - - def test_correct_sent_phunspell(self): - self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) - - -class SpellWanchanbertaTestCaseN(unittest.TestCase): - """Tests for wanchanberta_thai_grammarly engine (requires torch) - - DEPRECATED: Moved to tests.noauto-torch.testn_spell_torch - """ - - def test_word_correct_wanchanberta(self): - result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") - self.assertIsInstance(result, str) - self.assertNotEqual(result, "") - - def test_correct_sent_wanchanberta(self): - from ..core.test_spell import SENT_TOKS - - self.assertIsNotNone( - correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") - ) - - -class SpellHuggingFaceTestCaseN(unittest.TestCase): - """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access) - - DEPRECATED: Moved to tests.noauto-network.testn_spell_network - """ - - def test_get_words_spell_suggestion(self): - self.assertIsNotNone(get_words_spell_suggestion("คมดี")) - self.assertIsNotNone(get_words_spell_suggestion(["คมดี", "มะนา"])) diff --git a/tests/noauto/testn_tag.py b/tests/noauto/testn_tag.py deleted file mode 100644 index c6c73b3d0..000000000 --- a/tests/noauto/testn_tag.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -# DEPRECATED: This file is kept for backward compatibility only. -# New tests should be added to tests.noauto-torch for transformers-based tests. - -# Tests for tag functions that require transformers or torch -# These tests are NOT run in automated CI workflows due to: -# - Large dependencies (transformers, torch) -# - Python 3.13+ compatibility issues - -import unittest - -from pythainlp.tag import ( - NER, - NNER, - pos_tag_transformers, -) - - -class TagTransformersTestCaseN(unittest.TestCase): - """Tests for transformers-based engines (requires transformers, torch) - - DEPRECATED: Moved to tests.noauto-torch.testn_tag_torch - """ - - def test_NER_class(self): - with self.assertRaises(ValueError): - NER(engine="thainer", corpus="cat") - - ner = NER(engine="thainer") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="thainer-v2") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="wangchanberta") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="tltk") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - # Test thai-nner engine - ner = NER(engine="thai-nner") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - def test_NNER_class(self): - nner = NNER() - # Test basic tagging - self.assertIsNotNone(nner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - - # Test with top_level_only parameter - tokens, entities = nner.tag("แมวทำอะไรตอนห้าโมงเช้า") - self.assertIsInstance(tokens, list) - self.assertIsInstance(entities, list) - - tokens_top, entities_top = nner.tag("แมวทำอะไรตอนห้าโมงเช้า", top_level_only=True) - self.assertIsInstance(tokens_top, list) - self.assertIsInstance(entities_top, list) - # Top-level entities should be less than or equal to all entities - self.assertLessEqual(len(entities_top), len(entities)) - - def test_pos_tag_transformers(self): - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="bert", - corpus="blackboard", - ) - ) - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="mdeberta", - corpus="pud", - ) - ) - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="wangchanberta", - corpus="pud", - ) - ) - with self.assertRaises(ValueError): - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", engine="non-existing-engine" - ) - with self.assertRaises(ValueError): - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="bert", - corpus="non-existing corpus", - ) diff --git a/tests/noauto/testn_tokenize.py b/tests/noauto/testn_tokenize.py deleted file mode 100644 index e7b8ac2cc..000000000 --- a/tests/noauto/testn_tokenize.py +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 - -# DEPRECATED: This file is kept for backward compatibility only. -# New tests should be added to the appropriate modular test suite: -# - tests.noauto-torch for attacut, wtp, subword tokenizers -# - tests.noauto-tensorflow for deepcut -# - tests.noauto-onnx for oskut, sefr_cut - -# Tests for tokenize functions that require TensorFlow, Keras, or transformers -# These tests are NOT run in automated CI workflows due to: -# - Large dependencies (TensorFlow, Keras, transformers, torch) -# - Python 3.13+ compatibility issues - -import unittest - -from pythainlp.tokenize import ( - attacut, - deepcut, - oskut, - paragraph_tokenize, - sefr_cut, - sent_tokenize, - word_dict_trie, - word_tokenize, -) - -from ..core.test_tokenize import ( - SENT_3, - TEXT_1, -) -from ..test_helpers import ( - assert_segment_handles_none_and_empty, - assert_subword_tokenize_basic, -) - - -class DetokenizeTestCaseN(unittest.TestCase): - """DEPRECATED: Tests split across noauto-torch, noauto-tensorflow, and noauto-onnx""" - - def test_numeric_data_format(self): - engines = ["attacut", "deepcut", "sefr_cut"] - - for engine in engines: - self.assertIn( - "127.0.0.1", - word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine=engine), - ) - - tokens = word_tokenize( - "เวลา 12:12pm มีโปรโมชั่น 11.11", engine=engine - ) - self.assertTrue( - any(value in tokens for value in ["12:12pm", "12:12"]), - msg=f"{engine}: {tokens}", - ) - self.assertIn("11.11", tokens) - - self.assertIn( - "1,234,567.89", - word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine=engine), - ) - - tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine=engine) - self.assertIn("2.5:1", tokens) - self.assertIn("5:2", tokens) - - # try turning off `join_broken_num` - engine = "attacut" - self.assertNotIn( - "127.0.0.1", - word_tokenize( - "ไอพีของคุณคือ 127.0.0.1 ครับ", - engine=engine, - join_broken_num=False, - ), - ) - self.assertNotIn( - "1,234,567.89", - word_tokenize( - "รางวัลมูลค่า 1,234,567.89 บาท", - engine=engine, - join_broken_num=False, - ), - ) - - -class WordTokenizeAttacutTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" - - def test_word_tokenize_attacut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) - - def test_attacut(self): - assert_segment_handles_none_and_empty(self, attacut.segment) - self.assertEqual( - word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertEqual( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertIsNotNone( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") - ) - - -class WordTokenizeDeepcutTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-tensorflow.testn_tokenize_tensorflow""" - - def test_word_tokenize_deepcut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) - - def test_deepcut(self): - assert_segment_handles_none_and_empty(self, deepcut.segment) - self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) - self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) - self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) - self.assertIsNotNone( - word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() - ) - ) - - -class WordTokenizeOSKutTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-onnx.testn_tokenize_onnx""" - - def test_word_tokenize_oskut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) - - def test_oskut(self): - assert_segment_handles_none_and_empty(self, oskut.segment) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), - ) - - -class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-onnx.testn_tokenize_onnx""" - - def test_word_tokenize_sefr_cut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) - - def test_sefr_cut(self): - assert_segment_handles_none_and_empty(self, sefr_cut.segment) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), - ) - - -class ParagraphTokenizeTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" - - def test_paragraph_tokenize(self): - sent = ( - "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" - "จากผลงานวิจัยที่เคยทำมาในอดีต" - " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" - " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" - ) - self.assertIsNotNone(paragraph_tokenize(sent)) - with self.assertRaises(ValueError): - paragraph_tokenize( - sent, engine="ai2+2thai" - ) # engine does not exist - - -class SentTokenizeWTPTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" - - def test_sent_tokenize_wtp(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp", - ), - ) - - def test_sent_tokenize_wtp_tiny(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp-tiny", - ), - ) - - -class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" - - def test_subword_tokenize_phayathai(self): - assert_subword_tokenize_basic(self, "phayathai") - - -class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): - """DEPRECATED: Moved to tests.noauto-torch.testn_tokenize_torch""" - - def test_subword_tokenize_wangchanberta(self): - assert_subword_tokenize_basic(self, "wangchanberta") diff --git a/tests/noauto-cython/__init__.py b/tests/noauto_cython/__init__.py similarity index 96% rename from tests/noauto-cython/__init__.py rename to tests/noauto_cython/__init__.py index 2757d4405..11632b3ab 100644 --- a/tests/noauto-cython/__init__.py +++ b/tests/noauto_cython/__init__.py @@ -19,7 +19,7 @@ # Names of module to be tested test_packages: list[str] = [ - "tests.noauto-cython.testn_spell_cython", + "tests.noauto_cython.testn_spell_cython", ] diff --git a/tests/noauto-cython/testn_spell_cython.py b/tests/noauto_cython/testn_spell_cython.py similarity index 100% rename from tests/noauto-cython/testn_spell_cython.py rename to tests/noauto_cython/testn_spell_cython.py diff --git a/tests/noauto-network/__init__.py b/tests/noauto_network/__init__.py similarity index 95% rename from tests/noauto-network/__init__.py rename to tests/noauto_network/__init__.py index 2795f2311..d2d4bd765 100644 --- a/tests/noauto-network/__init__.py +++ b/tests/noauto_network/__init__.py @@ -22,7 +22,7 @@ # Names of module to be tested test_packages: list[str] = [ - "tests.noauto-network.testn_spell_network", + "tests.noauto_network.testn_spell_network", ] diff --git a/tests/noauto-network/testn_spell_network.py b/tests/noauto_network/testn_spell_network.py similarity index 100% rename from tests/noauto-network/testn_spell_network.py rename to tests/noauto_network/testn_spell_network.py diff --git a/tests/noauto-onnx/__init__.py b/tests/noauto_onnx/__init__.py similarity index 96% rename from tests/noauto-onnx/__init__.py rename to tests/noauto_onnx/__init__.py index 84e670ecc..3c381d8be 100644 --- a/tests/noauto-onnx/__init__.py +++ b/tests/noauto_onnx/__init__.py @@ -21,7 +21,7 @@ # Names of module to be tested test_packages: list[str] = [ - "tests.noauto-onnx.testn_tokenize_onnx", + "tests.noauto_onnx.testn_tokenize_onnx", ] diff --git a/tests/noauto-onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py similarity index 100% rename from tests/noauto-onnx/testn_tokenize_onnx.py rename to tests/noauto_onnx/testn_tokenize_onnx.py diff --git a/tests/noauto-tensorflow/__init__.py b/tests/noauto_tensorflow/__init__.py similarity index 95% rename from tests/noauto-tensorflow/__init__.py rename to tests/noauto_tensorflow/__init__.py index 81bc46445..da2e79252 100644 --- a/tests/noauto-tensorflow/__init__.py +++ b/tests/noauto_tensorflow/__init__.py @@ -21,7 +21,7 @@ # Names of module to be tested test_packages: list[str] = [ - "tests.noauto-tensorflow.testn_tokenize_tensorflow", + "tests.noauto_tensorflow.testn_tokenize_tensorflow", ] diff --git a/tests/noauto-tensorflow/testn_tokenize_tensorflow.py b/tests/noauto_tensorflow/testn_tokenize_tensorflow.py similarity index 100% rename from tests/noauto-tensorflow/testn_tokenize_tensorflow.py rename to tests/noauto_tensorflow/testn_tokenize_tensorflow.py diff --git a/tests/noauto-torch/__init__.py b/tests/noauto_torch/__init__.py similarity index 89% rename from tests/noauto-torch/__init__.py rename to tests/noauto_torch/__init__.py index de452d11b..67538e446 100644 --- a/tests/noauto-torch/__init__.py +++ b/tests/noauto_torch/__init__.py @@ -24,9 +24,9 @@ # Names of module to be tested test_packages: list[str] = [ - "tests.noauto-torch.testn_spell_torch", - "tests.noauto-torch.testn_tag_torch", - "tests.noauto-torch.testn_tokenize_torch", + "tests.noauto_torch.testn_spell_torch", + "tests.noauto_torch.testn_tag_torch", + "tests.noauto_torch.testn_tokenize_torch", ] diff --git a/tests/noauto-torch/testn_spell_torch.py b/tests/noauto_torch/testn_spell_torch.py similarity index 100% rename from tests/noauto-torch/testn_spell_torch.py rename to tests/noauto_torch/testn_spell_torch.py diff --git a/tests/noauto-torch/testn_tag_torch.py b/tests/noauto_torch/testn_tag_torch.py similarity index 100% rename from tests/noauto-torch/testn_tag_torch.py rename to tests/noauto_torch/testn_tag_torch.py diff --git a/tests/noauto-torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py similarity index 100% rename from tests/noauto-torch/testn_tokenize_torch.py rename to tests/noauto_torch/testn_tokenize_torch.py From 6b664a2b2e7d2603369d28bae0871798510c2936 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:34:41 +0000 Subject: [PATCH 06/17] Add comprehensive tests for torch and onnx-based functions Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/noauto_onnx/testn_tokenize_onnx.py | 37 +++++++++ tests/noauto_torch/testn_tokenize_torch.py | 87 ++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/tests/noauto_onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py index d581b0b79..6feecef6c 100644 --- a/tests/noauto_onnx/testn_tokenize_onnx.py +++ b/tests/noauto_onnx/testn_tokenize_onnx.py @@ -78,3 +78,40 @@ def test_sefr_cut(self): self.assertIsNotNone( sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), ) + + +class TransliterateONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based transliteration (requires onnxruntime)""" + + def test_thai2rom_onnx(self): + from pythainlp.transliterate import thai2rom_onnx + + result = thai2rom_onnx("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + +class TagONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based POS tagging (requires onnxruntime)""" + + def test_pos_tag_wangchanberta_onnx(self): + from pythainlp.tag import pos_tag + + result = pos_tag( + ["แมว", "กิน", "ปลา"], + engine="wangchanberta_onnx" + ) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + self.assertEqual(len(result), 3) + + +class SpellONNXTestCaseN(unittest.TestCase): + """Tests for ONNX-based spell correction (requires onnxruntime)""" + + def test_words_spelling_correction(self): + from pythainlp.spell import words_spelling_correction + + result = words_spelling_correction("สวัสดี") + self.assertIsInstance(result, list) + diff --git a/tests/noauto_torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py index 0829b74fa..d7feffa36 100644 --- a/tests/noauto_torch/testn_tokenize_torch.py +++ b/tests/noauto_torch/testn_tokenize_torch.py @@ -143,3 +143,90 @@ class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): def test_subword_tokenize_wangchanberta(self): assert_subword_tokenize_basic(self, "wangchanberta") + + +class TransliterateTestCaseN(unittest.TestCase): + """Tests for transliteration functions (requires torch)""" + + def test_thai2rom(self): + from pythainlp.transliterate import thai2rom + + result = thai2rom("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_thaig2p(self): + from pythainlp.transliterate import thaig2p + + result = thaig2p("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_thaig2p_v2(self): + from pythainlp.transliterate import thaig2p_v2 + + result = thaig2p_v2("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + def test_umt5_thaig2p(self): + from pythainlp.transliterate import umt5_thaig2p + + result = umt5_thaig2p("สวัสดี") + self.assertIsInstance(result, str) + self.assertGreater(len(result), 0) + + +class ParseTestCaseN(unittest.TestCase): + """Tests for parsing functions (requires torch and transformers)""" + + def test_dependency_parsing(self): + from pythainlp.parse import dependency_parsing + + result = dependency_parsing("แมวกินปลา") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_dependency_parsing_v2(self): + from pythainlp.parse import dependency_parsing_v2 + + result = dependency_parsing_v2("แมวกินปลา") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + +class SummarizeTestCaseN(unittest.TestCase): + """Tests for summarization functions (requires transformers)""" + + def test_summarize_keybert(self): + from pythainlp.summarize import keybert + + text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" + result = keybert(text, n=2) + self.assertIsInstance(result, list) + + def test_summarize_mt5(self): + from pythainlp.summarize import mt5 + + text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" + result = mt5(text) + self.assertIsInstance(result, str) + + +class AugmentTestCaseN(unittest.TestCase): + """Tests for augmentation functions (requires transformers)""" + + def test_augment_wangchanberta(self): + from pythainlp.augment.lm import aug_wangchanberta + + result = aug_wangchanberta("แมวกิน") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_augment_phayathaibert(self): + from pythainlp.augment.lm import aug_phayathaibert + + result = aug_phayathaibert("แมวกิน") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + From e80e99eada5f34fefdeef735cf2549fd68adbf9e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:40:45 +0000 Subject: [PATCH 07/17] Remove unused import in testn_tokenize_torch.py Removed unused import 'word_dict_trie' from test file. --- tests/noauto_torch/testn_tokenize_torch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/noauto_torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py index d7feffa36..cf73c7ad7 100644 --- a/tests/noauto_torch/testn_tokenize_torch.py +++ b/tests/noauto_torch/testn_tokenize_torch.py @@ -13,7 +13,6 @@ attacut, paragraph_tokenize, sent_tokenize, - word_dict_trie, word_tokenize, ) From c6d0b4dfaf7c398a7168efbc33e60a35f6d43c6e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:53:30 +0000 Subject: [PATCH 08/17] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 753dbdecb..e877619e1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -106,7 +106,7 @@ By separating tests by dependency group, we can: **PyTorch-based: tests.noauto_torch** - Run `unittest tests.noauto_torch` - - Need dependencies from `pip install "pythainlp[noauto_torch]"` + - Need dependencies from `pip install "pythainlp[noauto-torch]"` - Tests requiring PyTorch and its ecosystem: - torch, transformers (PyTorch backend), sentence-transformers - attacut, thai-nner, wtpsplit, tltk From d753451cb3fea5977dfd51568b0a71e434b68bda Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:54:02 +0000 Subject: [PATCH 09/17] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index e877619e1..da4c7a607 100644 --- a/tests/README.md +++ b/tests/README.md @@ -119,7 +119,7 @@ By separating tests by dependency group, we can: **TensorFlow-based: tests.noauto_tensorflow** - Run `unittest tests.noauto_tensorflow` - - Need dependencies from `pip install "pythainlp[noauto_tensorflow]"` + - Need dependencies from `pip install "pythainlp[noauto-tensorflow]"` - Tests requiring TensorFlow: - deepcut tokenizer - Dependencies: ~1-2 GB From 4d087d6b451464235b71783b70d59085ae472a83 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:54:23 +0000 Subject: [PATCH 10/17] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index da4c7a607..7e47b3f34 100644 --- a/tests/README.md +++ b/tests/README.md @@ -129,7 +129,7 @@ By separating tests by dependency group, we can: **ONNX Runtime-based: tests.noauto_onnx** - Run `unittest tests.noauto_onnx` - - Need dependencies from `pip install "pythainlp[noauto_onnx]"` + - Need dependencies from `pip install "pythainlp[noauto-onnx]"` - Tests requiring ONNX Runtime: - oskut, sefr_cut tokenizers - Dependencies: ~200-500 MB From dbaa40f24c5ffe55b54aa131d79ecaa23ab59de0 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:54:33 +0000 Subject: [PATCH 11/17] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 7e47b3f34..747c14ce2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,7 +148,7 @@ By separating tests by dependency group, we can: **Network-dependent: tests.noauto_network** - Run `unittest tests.noauto_network` - - Need dependencies from `pip install "pythainlp[noauto_network]"` + - Need dependencies from `pip install "pythainlp[noauto-network]"` - Tests requiring network access: - HuggingFace Hub model downloads - External API calls From 3f510618633cf511c9b2b08912486d9ea92288ae Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 14:55:03 +0000 Subject: [PATCH 12/17] Update pyproject.toml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8d58da7a4..66b4f167a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,7 +224,7 @@ noauto-cython = [ "phunspell>=0.1.6", ] -# Network-dependent tests - for tests.noauto-network +# Network-dependent tests - for tests.noauto_network # These tests require network access but minimal dependencies noauto-network = [ "huggingface-hub>=0.16.0", From 42d6db249506cc0db3a7987eec6877e5ed31d991 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:58:39 +0000 Subject: [PATCH 13/17] Fix: Correct pip extras naming and module imports based on code review - Change pip install commands to use hyphens (noauto-cython, not noauto_cython) in README - Update pyproject.toml comments to use underscores for Python module names (tests.noauto_torch not tests.noauto-torch) - Fix imports in test files to import actual functions not modules: * thai2rom.romanize instead of module thai2rom * KeyBERT class and mT5Summarizer class instead of module references * thai2rom_onnx.romanize instead of module * words_spelling_correction.correct instead of module - Remove duplicate unittest imports in __init__.py files (use 'from unittest import main' in __main__ block) Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 10 ++++----- tests/README.md | 2 +- tests/noauto_cython/__init__.py | 4 ++-- tests/noauto_network/__init__.py | 4 ++-- tests/noauto_onnx/__init__.py | 4 ++-- tests/noauto_onnx/testn_tokenize_onnx.py | 10 ++++----- tests/noauto_tensorflow/__init__.py | 4 ++-- tests/noauto_torch/__init__.py | 4 ++-- tests/noauto_torch/testn_tokenize_torch.py | 26 ++++++++++++---------- 9 files changed, 35 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66b4f167a..76459bf78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,10 +190,10 @@ extra = [ "tltk>=1.10", ] -# Noauto test dependencies - for tests.noauto-* modules +# Noauto test dependencies - for tests.noauto_* modules # These are grouped by dependency framework to avoid conflicts -# PyTorch-based dependencies - for tests.noauto-torch +# PyTorch-based dependencies - for tests.noauto_torch noauto-torch = [ "attacut>=1.0.6", "numpy>=1.26.0", @@ -205,13 +205,13 @@ noauto-torch = [ "wtpsplit>=1.0.1", ] -# TensorFlow-based dependencies - for tests.noauto-tensorflow +# TensorFlow-based dependencies - for tests.noauto_tensorflow noauto-tensorflow = [ "deepcut>=0.7.0", "numpy>=1.26.0", ] -# ONNX Runtime-based dependencies - for tests.noauto-onnx +# ONNX Runtime-based dependencies - for tests.noauto_onnx noauto-onnx = [ "numpy>=1.26.0", "onnxruntime>=1.10.0", @@ -219,7 +219,7 @@ noauto-onnx = [ "sefr_cut>=1.1", ] -# Cython-based dependencies - for tests.noauto-cython +# Cython-based dependencies - for tests.noauto_cython noauto-cython = [ "phunspell>=0.1.6", ] diff --git a/tests/README.md b/tests/README.md index 747c14ce2..76eb9d3bd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -138,7 +138,7 @@ By separating tests by dependency group, we can: **Cython-compiled: tests.noauto_cython** - Run `unittest tests.noauto_cython` - - Need dependencies from `pip install "pythainlp[noauto_cython]"` + - Need dependencies from `pip install "pythainlp[noauto-cython]"` - Tests requiring Cython-compiled packages: - phunspell spell checker - Requires: Cython, C compiler, system libraries (hunspell) diff --git a/tests/noauto_cython/__init__.py b/tests/noauto_cython/__init__.py index 11632b3ab..92e348e7d 100644 --- a/tests/noauto_cython/__init__.py +++ b/tests/noauto_cython/__init__.py @@ -37,6 +37,6 @@ def load_tests( if __name__ == "__main__": - import unittest + from unittest import main - unittest.main() + main() diff --git a/tests/noauto_network/__init__.py b/tests/noauto_network/__init__.py index d2d4bd765..57b6322ca 100644 --- a/tests/noauto_network/__init__.py +++ b/tests/noauto_network/__init__.py @@ -40,6 +40,6 @@ def load_tests( if __name__ == "__main__": - import unittest + from unittest import main - unittest.main() + main() diff --git a/tests/noauto_onnx/__init__.py b/tests/noauto_onnx/__init__.py index 3c381d8be..5f27180ca 100644 --- a/tests/noauto_onnx/__init__.py +++ b/tests/noauto_onnx/__init__.py @@ -39,6 +39,6 @@ def load_tests( if __name__ == "__main__": - import unittest + from unittest import main - unittest.main() + main() diff --git a/tests/noauto_onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py index 6feecef6c..68b2bac8d 100644 --- a/tests/noauto_onnx/testn_tokenize_onnx.py +++ b/tests/noauto_onnx/testn_tokenize_onnx.py @@ -84,9 +84,9 @@ class TransliterateONNXTestCaseN(unittest.TestCase): """Tests for ONNX-based transliteration (requires onnxruntime)""" def test_thai2rom_onnx(self): - from pythainlp.transliterate import thai2rom_onnx + from pythainlp.transliterate.thai2rom_onnx import romanize - result = thai2rom_onnx("สวัสดี") + result = romanize("สวัสดี") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) @@ -110,8 +110,8 @@ class SpellONNXTestCaseN(unittest.TestCase): """Tests for ONNX-based spell correction (requires onnxruntime)""" def test_words_spelling_correction(self): - from pythainlp.spell import words_spelling_correction + from pythainlp.spell.words_spelling_correction import correct - result = words_spelling_correction("สวัสดี") - self.assertIsInstance(result, list) + result = correct("สวัสดี") + self.assertIsInstance(result, (str, list)) diff --git a/tests/noauto_tensorflow/__init__.py b/tests/noauto_tensorflow/__init__.py index da2e79252..dd71f2b28 100644 --- a/tests/noauto_tensorflow/__init__.py +++ b/tests/noauto_tensorflow/__init__.py @@ -39,6 +39,6 @@ def load_tests( if __name__ == "__main__": - import unittest + from unittest import main - unittest.main() + main() diff --git a/tests/noauto_torch/__init__.py b/tests/noauto_torch/__init__.py index 67538e446..6ce8f86e6 100644 --- a/tests/noauto_torch/__init__.py +++ b/tests/noauto_torch/__init__.py @@ -44,6 +44,6 @@ def load_tests( if __name__ == "__main__": - import unittest + from unittest import main - unittest.main() + main() diff --git a/tests/noauto_torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py index cf73c7ad7..062a6c3b6 100644 --- a/tests/noauto_torch/testn_tokenize_torch.py +++ b/tests/noauto_torch/testn_tokenize_torch.py @@ -148,30 +148,30 @@ class TransliterateTestCaseN(unittest.TestCase): """Tests for transliteration functions (requires torch)""" def test_thai2rom(self): - from pythainlp.transliterate import thai2rom + from pythainlp.transliterate.thai2rom import romanize - result = thai2rom("สวัสดี") + result = romanize("สวัสดี") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) def test_thaig2p(self): - from pythainlp.transliterate import thaig2p + from pythainlp.transliterate.thaig2p import transliterate - result = thaig2p("สวัสดี") + result = transliterate("สวัสดี") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) def test_thaig2p_v2(self): - from pythainlp.transliterate import thaig2p_v2 + from pythainlp.transliterate.thaig2p_v2 import transliterate - result = thaig2p_v2("สวัสดี") + result = transliterate("สวัสดี") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) def test_umt5_thaig2p(self): - from pythainlp.transliterate import umt5_thaig2p + from pythainlp.transliterate.umt5_thaig2p import transliterate - result = umt5_thaig2p("สวัสดี") + result = transliterate("สวัสดี") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) @@ -198,17 +198,19 @@ class SummarizeTestCaseN(unittest.TestCase): """Tests for summarization functions (requires transformers)""" def test_summarize_keybert(self): - from pythainlp.summarize import keybert + from pythainlp.summarize.keybert import KeyBERT text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" - result = keybert(text, n=2) + keybert = KeyBERT() + result = keybert.extract_keywords(text, n=2) self.assertIsInstance(result, list) def test_summarize_mt5(self): - from pythainlp.summarize import mt5 + from pythainlp.summarize.mt5 import mT5Summarizer text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" - result = mt5(text) + summarizer = mT5Summarizer() + result = summarizer.summarize(text) self.assertIsInstance(result, str) From e95a7d22a5f1bcd0fe198f41c62197bfacd9f149 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 16:06:39 +0000 Subject: [PATCH 14/17] Fix: Correct test imports and method signatures based on code review - Remove test_dependency_parsing_v2 (function doesn't exist) - Fix KeyBERT.extract_keywords to use max_keywords parameter instead of n - Fix mT5Summarizer.summarize to expect list return type instead of str - Fix augmentation tests to use Thai2transformersAug and ThaiTextAugmenter classes - Fix spell correction test to use get_words_spell_suggestion function Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/noauto_onnx/testn_tokenize_onnx.py | 8 +++++--- tests/noauto_torch/testn_tokenize_torch.py | 21 ++++++++------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/noauto_onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py index 68b2bac8d..fd5bc3748 100644 --- a/tests/noauto_onnx/testn_tokenize_onnx.py +++ b/tests/noauto_onnx/testn_tokenize_onnx.py @@ -110,8 +110,10 @@ class SpellONNXTestCaseN(unittest.TestCase): """Tests for ONNX-based spell correction (requires onnxruntime)""" def test_words_spelling_correction(self): - from pythainlp.spell.words_spelling_correction import correct + from pythainlp.spell.words_spelling_correction import ( + get_words_spell_suggestion, + ) - result = correct("สวัสดี") - self.assertIsInstance(result, (str, list)) + result = get_words_spell_suggestion("สวัสดี") + self.assertIsInstance(result, list) diff --git a/tests/noauto_torch/testn_tokenize_torch.py b/tests/noauto_torch/testn_tokenize_torch.py index 062a6c3b6..ae3591253 100644 --- a/tests/noauto_torch/testn_tokenize_torch.py +++ b/tests/noauto_torch/testn_tokenize_torch.py @@ -186,13 +186,6 @@ def test_dependency_parsing(self): self.assertIsInstance(result, list) self.assertGreater(len(result), 0) - def test_dependency_parsing_v2(self): - from pythainlp.parse import dependency_parsing_v2 - - result = dependency_parsing_v2("แมวกินปลา") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - class SummarizeTestCaseN(unittest.TestCase): """Tests for summarization functions (requires transformers)""" @@ -202,7 +195,7 @@ def test_summarize_keybert(self): text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" keybert = KeyBERT() - result = keybert.extract_keywords(text, n=2) + result = keybert.extract_keywords(text, max_keywords=2) self.assertIsInstance(result, list) def test_summarize_mt5(self): @@ -211,23 +204,25 @@ def test_summarize_mt5(self): text = "แมวเป็นสัตว์เลี้ยงที่น่ารัก แมวชอบกินปลา แมวชอบนอนหลับ" summarizer = mT5Summarizer() result = summarizer.summarize(text) - self.assertIsInstance(result, str) + self.assertIsInstance(result, list) class AugmentTestCaseN(unittest.TestCase): """Tests for augmentation functions (requires transformers)""" def test_augment_wangchanberta(self): - from pythainlp.augment.lm import aug_wangchanberta + from pythainlp.augment.lm import Thai2transformersAug - result = aug_wangchanberta("แมวกิน") + augmenter = Thai2transformersAug() + result = augmenter.augment("แมวกิน") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) def test_augment_phayathaibert(self): - from pythainlp.augment.lm import aug_phayathaibert + from pythainlp.augment.lm import ThaiTextAugmenter - result = aug_phayathaibert("แมวกิน") + augmenter = ThaiTextAugmenter() + result = augmenter.augment("แมวกิน") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) From 9ae1d7ad4c0ef0599a70c1a919e5ac831ad73495 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:23:01 +0000 Subject: [PATCH 15/17] Fix: Update documentation to match actual dependencies - Remove sentence-transformers from documentation (not in pyproject.toml dependencies) - Fix thai_nner to thai-nner with hyphen to match pyproject.toml naming Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/README.md | 2 +- tests/noauto_torch/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 76eb9d3bd..af806a1f8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -108,7 +108,7 @@ By separating tests by dependency group, we can: - Run `unittest tests.noauto_torch` - Need dependencies from `pip install "pythainlp[noauto-torch]"` - Tests requiring PyTorch and its ecosystem: - - torch, transformers (PyTorch backend), sentence-transformers + - torch, transformers (PyTorch backend) - attacut, thai-nner, wtpsplit, tltk - Tests: spell correction (wanchanberta), NER/POS tagging (transformers-based), tokenization (attacut), subword tokenization (phayathai, wangchanberta), diff --git a/tests/noauto_torch/__init__.py b/tests/noauto_torch/__init__.py index 6ce8f86e6..675181b6c 100644 --- a/tests/noauto_torch/__init__.py +++ b/tests/noauto_torch/__init__.py @@ -6,9 +6,8 @@ Test functions that require PyTorch and its ecosystem dependencies: - torch - transformers (when using PyTorch backend) -- sentence-transformers - attacut -- thai_nner +- thai-nner - wtpsplit These tests are NOT run in automated CI workflows due to: From 789d54b5fdec395fbe9abb6a5f308071f53df875 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Feb 2026 20:00:58 +0000 Subject: [PATCH 16/17] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index af806a1f8..af26a0fff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -97,7 +97,7 @@ By separating tests by dependency group, we can: #### Umbrella suite: tests.noauto - Run `unittest tests.noauto` -- Includes all noauto test suites (legacy and new modular suites) +- Includes all modular noauto test suites - Use this for comprehensive testing when all dependencies are available - Test case class suffix: `TestCaseN` From 0cf1aa1d4b1d329b22e9ae4e8ae6ae6e8f087249 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 10:49:53 +0000 Subject: [PATCH 17/17] Organize optional dependencies in pyproject.toml Updated optional dependencies section with headings for better organization. --- pyproject.toml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76459bf78..696a444b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,8 @@ dependencies = [ [project.optional-dependencies] -# Development and testing dependencies +## 1) Development ######################################## + dev = [ "black>=25.11.0", "build>=1.0.0", @@ -79,14 +80,13 @@ dev = [ "tox>=4.30.3", ] -# Documentation dependencies docs = [ "Sphinx>=6.2", "sphinx-copybutton>=0.5.2", "sphinx-rtd-theme>=3.1.0", ] -# Below are dependencies for optional features +## 2) Feature sets ####################################### abbreviation = ["khamyo>=0.2.0"] @@ -166,9 +166,13 @@ wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"] wunsen = ["wunsen>=0.0.3"] -# Below are defined groups of CI testing dependencies +## 3) Testing ############################################ + +# Below are dependency groups for each defined test suites -# Compact dependencies - safe small set of optional dependencies +# Compact dependencies - for tests.compact modules +# Safe small set of optional dependencies +# for common tasks like tokenization and tagging. compact = [ "nlpo3>=1.4.0", "numpy>=1.26.0", @@ -178,7 +182,6 @@ compact = [ ] # Extra dependencies - for tests.extra modules -# Note: Some packages excluded due to Python 3.13 compatibility issues extra = [ "bpemb>=0.3.2", "budoux>=0.7.0", @@ -191,7 +194,7 @@ extra = [ ] # Noauto test dependencies - for tests.noauto_* modules -# These are grouped by dependency framework to avoid conflicts +# These are further grouped by dependency framework to avoid conflicts # PyTorch-based dependencies - for tests.noauto_torch noauto-torch = [ @@ -230,7 +233,6 @@ noauto-network = [ "huggingface-hub>=0.16.0", ] - # Full dependencies - pinned where available full = [ "attacut==1.0.6",