Merge pull request #1290 from PyThaiNLP/copilot/refactor-noauto-test-suites

Reorganize noauto test suite by dependency groups to resolve conflicts

Author: bact

CHANGELOG.md

@@ -34,6 +34,7 @@ See PR for prompt and details.
 - Make package zip-safe #1212
 - Ensure thread-safety for tokenizers #1213
 - Add Thai-NNER integration with top-level entity filtering #1221
+- Reorganize noauto test suite by dependency groups (torch, tensorflow, onnx, cython, network) #935
 - Improved documentation; code cleanup; more tests
 
 ## Version 5.1.2 -> 5.2.0

pyproject.toml

@@ -68,7 +68,8 @@ dependencies = [
 
 [project.optional-dependencies]
 
-# Development and testing dependencies
+## 1) Development ########################################
+
 dev = [
     "black>=25.11.0",
     "build>=1.0.0",
@@ -79,14 +80,13 @@ dev = [
     "tox>=4.30.3",
 ]
 
-# Documentation dependencies
 docs = [
     "Sphinx>=6.2",
     "sphinx-copybutton>=0.5.2",
     "sphinx-rtd-theme>=3.1.0",
 ]
 
-# Below are dependencies for optional features
+## 2) Feature sets #######################################
 
 abbreviation = ["khamyo>=0.2.0"]
 
@@ -166,9 +166,13 @@ wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"]
 
 wunsen = ["wunsen>=0.0.3"]
 
-# Below are defined groups of CI testing dependencies
+## 3) Testing ############################################
+
+# Below are dependency groups for each defined test suites
 
-# Compact dependencies - safe small set of optional dependencies
+# Compact dependencies - for tests.compact modules
+# Safe small set of optional dependencies
+# for common tasks like tokenization and tagging.
 compact = [
     "nlpo3>=1.4.0",
     "numpy>=1.26.0",
@@ -178,7 +182,6 @@ compact = [
 ]
 
 # Extra dependencies - for tests.extra modules
-# Note: Some packages excluded due to Python 3.13 compatibility issues
 extra = [
     "bpemb>=0.3.2",
     "budoux>=0.7.0",
@@ -190,6 +193,46 @@ extra = [
     "tltk>=1.10",
 ]
 
+# Noauto test dependencies - for tests.noauto_* modules
+# These are further grouped by dependency framework to avoid conflicts 
+
+# PyTorch-based dependencies - for tests.noauto_torch
+noauto-torch = [
+    "attacut>=1.0.6",
+    "numpy>=1.26.0",
+    "sentencepiece>=0.1.91",
+    "thai-nner>=0.3",
+    "tltk>=1.10",
+    "torch>=1.13.1",
+    "transformers>=4.22.1",
+    "wtpsplit>=1.0.1",
+]
+
+# TensorFlow-based dependencies - for tests.noauto_tensorflow
+noauto-tensorflow = [
+    "deepcut>=0.7.0",
+    "numpy>=1.26.0",
+]
+
+# ONNX Runtime-based dependencies - for tests.noauto_onnx
+noauto-onnx = [
+    "numpy>=1.26.0",
+    "onnxruntime>=1.10.0",
+    "oskut>=1.3",
+    "sefr_cut>=1.1",
+]
+
+# Cython-based dependencies - for tests.noauto_cython
+noauto-cython = [
+    "phunspell>=0.1.6",
+]
+
+# Network-dependent tests - for tests.noauto_network
+# These tests require network access but minimal dependencies
+noauto-network = [
+    "huggingface-hub>=0.16.0",
+]
+
 # Full dependencies - pinned where available
 full = [
     "attacut==1.0.6",

tests/README.md

@@ -74,16 +74,88 @@ The CI/CD test workflow is at
 
 ## Noauto tests (testn_*.py)
 
-- These dependencies might include huge libraries like `tensorflow`.
-- Due to dependency complexities, these functionalities may not be tested
-  in the CI/CD pipeline.
-  - In the future, we might create a separate
-    step or workflow to run this test suite.
-    It will be triggered manually.
-    We may also need to group test cases by
-    a non-conflicting set of dependencies.
+The noauto (no-automated) test suite contains tests for functionalities
+that require heavy dependencies which are not feasible to run in automated
+CI/CD pipelines. These tests are organized into specialized suites based
+on their dependency requirements.
+
+### Why separate noauto test suites?
+
+Different ML/AI frameworks often have conflicting version requirements for
+their dependencies. For example:
+- PyTorch and TensorFlow may require different versions of numpy or protobuf
+- Large frameworks take significant time to install (~1-3 GB each)
+- Some packages require Cython compilation or system libraries
+
+By separating tests by dependency group, we can:
+- Test each framework independently without conflicts
+- Optimize CI/CD resources by running only relevant test groups
+- Make it easier for developers to test specific functionality
+
+### Noauto test suites
+
+#### Umbrella suite: tests.noauto
+
+- Run `unittest tests.noauto`
+- Includes all modular noauto test suites
+- Use this for comprehensive testing when all dependencies are available
 - Test case class suffix: `TestCaseN`
 
+#### Modular suites by dependency:
+
+**PyTorch-based: tests.noauto_torch**
+
+- Run `unittest tests.noauto_torch`
+  - Need dependencies from `pip install "pythainlp[noauto-torch]"`
+- Tests requiring PyTorch and its ecosystem:
+  - torch, transformers (PyTorch backend)
+  - attacut, thai-nner, wtpsplit, tltk
+- Tests: spell correction (wanchanberta), NER/POS tagging (transformers-based),
+  tokenization (attacut), subword tokenization (phayathai, wangchanberta),
+  sentence tokenization (wtp)
+- Dependencies: ~2-3 GB
+- Test case class suffix: `TestCaseN`
+
+**TensorFlow-based: tests.noauto_tensorflow**
+
+- Run `unittest tests.noauto_tensorflow`
+  - Need dependencies from `pip install "pythainlp[noauto-tensorflow]"`
+- Tests requiring TensorFlow:
+  - deepcut tokenizer
+- Dependencies: ~1-2 GB
+- Note: May conflict with PyTorch dependencies
+- Test case class suffix: `TestCaseN`
+
+**ONNX Runtime-based: tests.noauto_onnx**
+
+- Run `unittest tests.noauto_onnx`
+  - Need dependencies from `pip install "pythainlp[noauto-onnx]"`
+- Tests requiring ONNX Runtime:
+  - oskut, sefr_cut tokenizers
+- Dependencies: ~200-500 MB
+- Test case class suffix: `TestCaseN`
+
+**Cython-compiled: tests.noauto_cython**
+
+- Run `unittest tests.noauto_cython`
+  - Need dependencies from `pip install "pythainlp[noauto-cython]"`
+- Tests requiring Cython-compiled packages:
+  - phunspell spell checker
+- Requires: Cython, C compiler, system libraries (hunspell)
+- Platform-specific build requirements
+- Test case class suffix: `TestCaseN`
+
+**Network-dependent: tests.noauto_network**
+
+- Run `unittest tests.noauto_network`
+  - Need dependencies from `pip install "pythainlp[noauto-network]"`
+- Tests requiring network access:
+  - HuggingFace Hub model downloads
+  - External API calls
+- Requires: Internet connection, may involve large downloads
+- Test case class suffix: `TestCaseN`
+
+
 ## Robustness tests (test_robustness.py)
 
 A comprehensive test suite within core tests that tests edge cases important

tests/noauto/__init__.py

@@ -11,16 +11,28 @@
 
 These tests are NOT run in automated CI workflows but are kept for
 manual testing and future re-enabling when dependencies improve.
+
+This test suite serves as an umbrella that includes all specialized
+noauto test suites:
+- noauto_torch: PyTorch and transformers-based tests
+- noauto_tensorflow: TensorFlow-based tests
+- noauto_onnx: ONNX Runtime-based tests
+- noauto_cython: Cython-compiled package tests
+- noauto_network: Network-dependent tests
+
+For targeted testing, use the specific test suites instead of this umbrella.
 """
 
 from unittest import TestLoader, TestSuite
 
 # Names of module to be tested
 # Note: These tests are NOT included in automated CI runs
 test_packages: list[str] = [
-    "tests.noauto.testn_spell",
-    "tests.noauto.testn_tag",
-    "tests.noauto.testn_tokenize",
+    "tests.noauto_torch",
+    "tests.noauto_tensorflow",
+    "tests.noauto_onnx",
+    "tests.noauto_cython",
+    "tests.noauto_network",
 ]
 
 
@@ -29,6 +41,14 @@ def load_tests(
 ) -> TestSuite:
     """Load test protocol
     See: https://docs.python.org/3/library/unittest.html#id1
+
+    This loads all modular test suites.
+    For targeted testing, use specific test suites directly:
+    - unittest tests.noauto_torch
+    - unittest tests.noauto_tensorflow
+    - unittest tests.noauto_onnx
+    - unittest tests.noauto_cython
+    - unittest tests.noauto_network
     """
     suite = TestSuite()
     for test_package in test_packages: