diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 81701d8cf..f3e44913f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -163,6 +163,14 @@ - [ ] API endpoints must use proper HTTP return codes. - [ ] Follow web best practices as recommended by OpenAPI, IETF, W3C, etc. +## Git + +- [ ] Follow these guidelines for writing a good commit message: + - How to Write a Git Commit Message + + - Commit Verbs 101: why I like to use this and why you should also like it. + + ## Python - [ ] Defensive coding: always check for None/empty and handle exceptions diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index c6a296d23..0fa4ee0ac 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -35,11 +35,6 @@ jobs: # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 # setuptools<=73.0.1 because https://github.com/pypa/setuptools/issues/4620 - #- name: Install dependencies - # env: - # SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True - # run: | - # if [ -f docker_requirements.txt ]; then pip install -r docker_requirements.txt; fi - name: Install PyThaiNLP run: pip install ".[docs]" - name: Build sphinx documentation diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index 3835805f9..8f7ad80d1 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -15,28 +15,36 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Ideally, this is the minimum supported version, as in requires-python in pyproject.toml - python-version: ["3.9"] + # Ideally, this should be the minimum supported version, + # as in `requires-python` in pyproject.toml + # But for now, we just test on the second-latest supported version + # because some dependencies may not support the older versions. + python-version: ["3.12"] steps: - uses: actions/checkout@v6 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install dependencies env: SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True run: | python -m pip install --upgrade "pip<24.1" setuptools - python -m pip install pythainlp[testing,full] + python -m pip install ".[compact]" python -m nltk.downloader omw-1.4 + + # Download the sdist from PyPI, extract it, and run the tests inside it + # (the binary distribution does not contain tests). - name: Test run: | set -euo pipefail mkdir -p pythainlp_test cd pythainlp_test - # Download sdist from PyPI (the binary distribution does not contain tests) pip download --no-binary=:all: --no-dependencies pythainlp archive=$(ls -1 *.tar.gz | head -n1) echo "Found archive: $archive" @@ -49,4 +57,4 @@ jobs: if [ -d data ] && [ -d tests ]; then mv data tests/ fi - python -m unittest discover -v + python -m unittest tests.core tests.compact -v diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index fdc0e9666..7e2a85328 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-License-Identifier: CC0-1.0 +# SPDX-License-Identifier: Apache-2.0 name: Unit test @@ -32,11 +32,34 @@ jobs: strategy: fail-fast: false matrix: - # Test latest and earliest versions of Python on every OSes + # To save time and resources, and manage dependency compatibility, + # we divide test cases into 3 groups + # and run them in different configurations: + # - Core (O): All supported Python versions on Ubuntu, + # plus latest and earliest Python versions + # on all supported OSes + # - Compact (C): Second-latest and earliest supported Python versions + # on all supported OSes, plus latest on Ubuntu + # - Extra (X): Second-latest supported Python version on Ubuntu only + # + # | Python | Ubuntu | Windows | macOS | + # |--------|---------|---------|-------| + # | 3.14 | O+C | O | O | + # | 3.13 | O+C+X | O+C | O+C | + # | 3.12 | O | | | + # | 3.11 | O | | | + # | 3.10 | O | | | + # | 3.9 | O+C | O+C | O+C | + # + # We should also consider to reduce the number of dependencies + # to avoid conflicts between them. + # See: https://github.com/PyThaiNLP/pythainlp/issues/935 os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.13", "3.9"] # Latest and earliest + python-version: ["3.14", "3.9"] # Latest and earliest include: # Test the rest of Python versions only on Ubuntu + - os: "ubuntu-latest" + python-version: "3.13" - os: "ubuntu-latest" python-version: "3.12" - os: "ubuntu-latest" @@ -46,87 +69,89 @@ jobs: runs-on: ${{ matrix.os }} env: - PYICU_WIN_VER: 2.14 - INSTALL_TORCH: false + PYICU_WIN_VER: 2.15 # 2.15 is the last version that supports Python 3.9 INSTALL_FULL_DEPS: false - PYTHON_VERSION_LATEST: "3.13" - PYTHON_VERSION_LATEST_2: "3.12" + PYTHON_VERSION_LATEST: "3.14" + PYTHON_VERSION_LATEST_2: "3.13" # Second-latest supported version + PYTHON_VERSION_EARLIEST: "3.9" steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install build tools - run: | - pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" - pip install coverage coveralls - # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 - # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 - # setuptools<=73.0.1 because https://github.com/pypa/setuptools/issues/4620 - - name: Install ICU (macOS) - if: startsWith(matrix.os, 'macos-') - run: | - brew install icu4c - PKG_CONFIG_PATH=$(brew --prefix)/opt/icu4c/lib/pkgconfig - echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> "${GITHUB_ENV}" - ICU_VER=$(pkg-config --modversion icu-i18n) - echo "ICU_VER=${ICU_VER}" - echo "ICU_VER=${ICU_VER}" >> "${GITHUB_ENV}" - - name: Install PyICU (Windows) - if: startsWith(matrix.os, 'windows-') && (matrix.python-version == '3.12' || matrix.python-version == '3.13') - shell: powershell - run: | - $PYTHON_WIN_VER = "${{ matrix.python-version }}" - $CP_VER = "cp" + $PYTHON_WIN_VER.Replace(".", "") - $WHEEL_URL = "https://github.com/cgohlke/pyicu-build/releases/download/v${{ env.PYICU_WIN_VER }}/PyICU-${{ env.PYICU_WIN_VER }}-${CP_VER}-${CP_VER}-win_amd64.whl" - pip install "$WHEEL_URL" - # Get wheel URL from https://github.com/cgohlke/pyicu-build/releases - - name: Install PyTorch - if: env.INSTALL_TORCH == 'true' - run: pip install torch - # If torch for the platform is not available in PyPI, use this command: - # pip install "" - # Get wheel URL from http://download.pytorch.org/whl/torch/ - - name: Install testing dependencies - if: env.INSTALL_FULL_DEPS == 'true' - env: - SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True - run: pip install ".[testing]" - - name: Install PyThaiNLP + dependencies (minimum) - if: matrix.python-version != env.PYTHON_VERSION_LATEST && matrix.python-version != env.PYTHON_VERSION_LATEST_2 - run: pip install . - - name: Install PyThaiNLP + dependencies (compact) - if: matrix.python-version == env.PYTHON_VERSION_LATEST || matrix.python-version == env.PYTHON_VERSION_LATEST_2 - run: pip install ".[compact]" - # If you want to install a safe small set of optional dependencies, use: - # pip install ".[compact]" - # We can gradually run more test cases by installing more optional - # dependencies. But we should also consider to reduce the number - # of dependencies to avoid the conflict between dependencies. - # See: https://github.com/PyThaiNLP/pythainlp/issues/935 - - name: Unit test (core) - if: matrix.python-version != env.PYTHON_VERSION_LATEST && matrix.python-version != env.PYTHON_VERSION_LATEST_2 - env: - PYTHONIOENCODING: utf-8 - run: coverage run -m unittest tests.core - - name: Unit test (core + compact) - if: matrix.python-version == env.PYTHON_VERSION_LATEST || matrix.python-version == env.PYTHON_VERSION_LATEST_2 - env: - PYTHONIOENCODING: utf-8 - run: coverage run -m unittest tests.core tests.compact - # Only test "compact" set with the latest two stable Python versions. - # Use 'unittest ' instead of 'unittest discover' to avoid - # loading tests with dependencies more than expected. - # Test cases loaded is defined in __init__.py in the tests directory. - # See also tests/README.md - - name: Coverage report - if: matrix.os == 'ubuntu-latest' && matrix.python-version == env.PYTHON_VERSION_LATEST - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_SERVICE_NAME: github - run: coveralls - # Only submit a report from the latest Python version on ubuntu-latest. + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install build tools + run: | + pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" + pip install coverage coveralls + # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 + # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 + # setuptools<=73.0.1 because https://github.com/pypa/setuptools/issues/4620 + + - name: Install ICU (macOS) + if: startsWith(matrix.os, 'macos-') + run: | + brew install icu4c + PKG_CONFIG_PATH=$(brew --prefix)/opt/icu4c/lib/pkgconfig + echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> "${GITHUB_ENV}" + ICU_VER=$(pkg-config --modversion icu-i18n) + echo "ICU_VER=${ICU_VER}" + echo "ICU_VER=${ICU_VER}" >> "${GITHUB_ENV}" + + - name: Install PyICU (Windows) + if: startsWith(matrix.os, 'windows-') && (matrix.python-version == env.PYTHON_VERSION_LATEST_2 || matrix.python-version == env.PYTHON_VERSION_EARLIEST) + shell: powershell + # Get the wheel URL from https://github.com/cgohlke/pyicu-build/releases + run: | + $PYTHON_WIN_VER = "${{ matrix.python-version }}" + $CP_VER = "cp" + $PYTHON_WIN_VER.Replace(".", "") + $WHEEL_URL = "https://github.com/cgohlke/pyicu-build/releases/download/v${{ env.PYICU_WIN_VER }}/PyICU-${{ env.PYICU_WIN_VER }}-${CP_VER}-${CP_VER}-win_amd64.whl" + pip install "$WHEEL_URL" + + - name: Install PyThaiNLP + full testing dependencies + if: env.INSTALL_FULL_DEPS == 'true' + env: + SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True + run: pip install ".[full]" + + - name: Test (core) + if: ${{ (matrix.os == 'ubuntu-latest' && matrix.python-version != env.PYTHON_VERSION_LATEST_2 && matrix.python-version != env.PYTHON_VERSION_EARLIEST) || (matrix.os != 'ubuntu-latest' && matrix.python-version == env.PYTHON_VERSION_LATEST) }} + env: + PYTHONIOENCODING: utf-8 + run: | + pip install . + python -m unittest tests.core + # Use 'unittest ' instead of 'unittest discover' to avoid + # loading tests with dependencies more than expected. + # Test cases loaded is defined in __init__.py in the tests directory. + # See also tests/README.md + + - name: Test (compact + core) + if: ${{ ((matrix.python-version == env.PYTHON_VERSION_LATEST_2) || (matrix.python-version == env.PYTHON_VERSION_EARLIEST)) && !(matrix.os == 'ubuntu-latest' && matrix.python-version == env.PYTHON_VERSION_LATEST_2) }} + env: + PYTHONIOENCODING: utf-8 + run: | + pip install ".[compact]" + python -m unittest tests.core tests.compact + + - name: Test (extra + compact + core) + if: matrix.os == 'ubuntu-latest' && matrix.python-version == env.PYTHON_VERSION_LATEST_2 + env: + PYTHONIOENCODING: utf-8 + run: | + pip install ".[compact,extra]" + coverage run -m unittest tests.core tests.compact tests.extra + + # Only submit a report from the "extra" run, to get maximum coverage + - name: Coverage report + if: matrix.os == 'ubuntu-latest' && matrix.python-version == env.PYTHON_VERSION_LATEST_2 + env: + COVERALLS_SERVICE_NAME: github + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTHONIOENCODING: utf-8 + run: coveralls diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fd9ef007..113fe2aba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,9 +6,11 @@ SPDX-License-Identifier: CC0-1.0 # Contributing to PyThaiNLP -Hi! Thanks for your interest in contributing to [PyThaiNLP](https://github.com/PyThaiNLP/pythainlp). +Hi! Thanks for your interest in contributing to +[PyThaiNLP](https://github.com/PyThaiNLP/pythainlp). -Please refer to our [Contributor Covenant Code of Conduct](https://github.com/PyThaiNLP/pythainlp/blob/dev/CODE_OF_CONDUCT.md). +Please refer to our +[Contributor Covenant Code of Conduct](https://github.com/PyThaiNLP/pythainlp/blob/dev/CODE_OF_CONDUCT.md). ## Issue Report and Discussion @@ -52,8 +54,9 @@ Please refer to our [Contributor Covenant Code of Conduct](https://github.com/Py ### Version Control System -- We use [Git](http://git-scm.com/) as our [version control system](http://en.wikipedia.org/wiki/Revision_control), -so it may be a good idea to familiarize yourself with it. +- We use [Git](http://git-scm.com/) as our + [version control system](http://en.wikipedia.org/wiki/Revision_control), + so it may be a good idea to familiarize yourself with it. - You can start with the [Pro Git book](http://git-scm.com/book/) (free!). ### Commit Message @@ -65,19 +68,24 @@ so it may be a good idea to familiarize yourself with it. - We use the famous [gitflow][] to manage our branches. - When you create pull requests on GitHub, GitHub Actions will run tests -and several checks automatically. Click the "Details" link at the end of -each check to see what needs to be fixed. + and several checks automatically. Click the "Details" link + at the end of each check to see what needs to be fixed. [gitflow]: http://nvie.com/posts/a-successful-git-branching-model/ ## Documentation -- We use [Sphinx](https://www.sphinx-doc.org/en/master/) to generate API document -automatically from "docstring" comments in source codes. This means the comment -section in the source codes is important for the quality of documentation. -- A docstring should start with one summary line, end with one line with a full stop (period), -then be followed by a blank line before starting a new paragraph. -- A commit to release branches (e.g. `2.2`, `2.1`) with a title **"(build and deploy docs)"** (without quotes) will trigger the system to rebuild the documentation files and upload them to the website . +- We use [Sphinx](https://www.sphinx-doc.org/en/master/) to generate + API document automatically from "docstring" comments in source codes. + This means the comment section in the source codes is important for the + quality of documentation. +- A docstring should start with one summary line, end with one line with + a full stop (period), then be followed by a blank line before starting + a new paragraph. +- A commit to release branches (e.g. `2.2`, `2.1`) with a title + **"(build and deploy docs)"** (without quotes) will trigger the system + to rebuild the documentation files and upload them to the website + . ## Testing @@ -124,14 +132,12 @@ Install with optional dependency groups: # Install with compact set of dependencies (recommended for development) pip install -e ".[compact]" -# Install with full dependencies -pip install -e ".[full]" - -# Install with testing dependencies (pinned versions for reproducibility) -pip install -e ".[testing]" +# Install with extra set of dependencies (can be huge) +pip install -e ".[compact,extra]" ``` -See all available optional dependency groups in `pyproject.toml` under `[project.optional-dependencies]`. +See all available optional dependency groups in `pyproject.toml` +under `[project.optional-dependencies]`. ### Building Distribution Packages @@ -145,8 +151,11 @@ This will create distribution packages in the `dist/` directory. ## Releasing -- We use [semantic versioning](https://semver.org/): MAJOR.MINOR.PATCH, with development build suffix: MAJOR.MINOR.PATCH-devBUILD -- We use [`bump-my-version`](https://github.com/callowayproject/bump-my-version) to manage versioning. The configuration is in `pyproject.toml` under `[tool.bumpversion]`. +- We use [semantic versioning](https://semver.org/): MAJOR.MINOR.PATCH, + with development build suffix: MAJOR.MINOR.PATCH-devBUILD +- We use [`bump-my-version`](https://github.com/callowayproject/bump-my-version) + to manage versioning. The configuration is in `pyproject.toml` + under `[tool.bumpversion]`. - `bump-my-version bump [major|minor|patch|release|build]` - Example: @@ -190,7 +199,8 @@ This will create distribution packages in the `dist/` directory. [![Contributors](https://contributors-img.firebaseapp.com/image?repo=PyThaiNLP/pythainlp)](https://github.com/PyThaiNLP/pythainlp/graphs/contributors) -Thanks to all [contributors](https://github.com/PyThaiNLP/pythainlp/graphs/contributors). (Image made with [contributors-img](https://contributors-img.firebaseapp.com)) +Thanks to all [contributors](https://github.com/PyThaiNLP/pythainlp/graphs/contributors). +(Image made with [contributors-img](https://contributors-img.firebaseapp.com)) ### Development Leads diff --git a/Dockerfile b/Dockerfile index 5aa9ab61e..7cbf8c3cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# Install PyThaiNLP with testing dependencies (replaces docker_requirements.txt) -RUN pip install -e ".[testing]" && pip cache purge +RUN pip install -e ".[full]" && pip cache purge diff --git a/README.md b/README.md index 066a32062..1456f9f4f 100644 --- a/README.md +++ b/README.md @@ -73,64 +73,45 @@ Install different releases: ### Installation Options -Some functionalities, like Thai WordNet, may require extra packages. To install those requirements, specify a set of `[name]` immediately after `pythainlp`: +Some functionalities, like Thai WordNet, may require extra packages. +To install those requirements, +specify a set of `[name]` immediately after `pythainlp`: ```sh pip install "pythainlp[extra1,extra2,...]" ``` -Possible `extras`: - -- `full` (install everything) -- `compact` (install a stable and small subset of dependencies) -- `abbreviation` (for Thai abbreviation support) -- `attacut` (to support attacut, a fast and accurate tokenizer) -- `benchmarks` (for [word tokenization benchmarking](tokenization-benchmark.md)) -- `budoux` (for BudouX text segmentation) -- `coreference_resolution` (for coreference resolution) -- `dependency_parsing` (for dependency parsing) -- `el` (for entity linking) -- `esupar` (for esupar parser support) -- `generate` (for text generation) -- `icu` (for ICU, International Components for Unicode, support in transliteration and tokenization) -- `ipa` (for IPA, International Phonetic Alphabet, support in transliteration) -- `ml` (to support ULMFiT models for classification) -- `mt5` (for mT5 model support) -- `nlpo3` (for nlpo3 Thai word tokenizer) -- `onnx` (for ONNX model support) -- `oskut` (for OSKut Thai word tokenizer) -- `sefr_cut` (for SEFR CUT Thai word tokenizer) -- `spacy_thai` (for spaCy Thai language support) -- `spell` (for spelling correction) -- `ssg` (for sentence segmentation) -- `testing` (pinned versions for CI/CD reproducibility) -- `textaugment` (for text augmentation) -- `thai_nner` (for Thai named entity recognition) -- `thai2fit` (for Thai word vector) -- `thai2rom` (for machine-learnt romanization) -- `transformers_ud` (for Universal Dependencies with transformers) -- `translate` (for machine translation) -- `wangchanberta` (for WangchanBERTa model) -- `wangchanglm` (for WangchanGLM model) -- `word_approximation` (for word approximation) -- `wordnet` (for Thai WordNet API) -- `wsd` (for word sense disambiguation) -- `wtp` (for Where's the Point text segmentation) -- `wunsen` (for Wunsen spell checker) - -For dependency details, look at the `[project.optional-dependencies]` section in +Possible `extras` included: + +- `compact` — install a stable and small subset of dependencies (recommended) +- `translate` — machine translation support +- `wordnet` — WordNet support +- `full` — install all optional dependencies + (may introduce large dependencies and conflicts) + +The documentation website maintains +[full list of extras](https://pythainlp.org/dev-docs/notes/installation.html). + +For dependency details, +look at the `[project.optional-dependencies]` section in [`pyproject.toml`](https://github.com/PyThaiNLP/pythainlp/blob/dev/pyproject.toml). ## Data Directory -- Some additional data, like word lists and language models, may be automatically downloaded during runtime. -- PyThaiNLP caches these data under the directory `~/pythainlp-data` by default. -- The data directory can be changed by specifying the environment variable `PYTHAINLP_DATA_DIR`. -- See the data catalog (`db.json`) at +- Some additional data, like word lists and language models, + may be automatically downloaded during runtime. +- PyThaiNLP caches these data under the directory `~/pythainlp-data` + by default. +- The data directory can be changed by specifying the environment variable + `PYTHAINLP_DATA_DIR`. +- See the data catalog (`db.json`) at + ### Using PyThaiNLP in Distributed Environments -When using PyThaiNLP in distributed computing environments (e.g., Apache Spark), set the `PYTHAINLP_DATA_DIR` environment variable inside the function that will be distributed to worker nodes: +When using PyThaiNLP in distributed computing environments +(e.g., Apache Spark), set the `PYTHAINLP_DATA_DIR` environment variable +inside the function that will be distributed to worker nodes: ```python def tokenize_thai(text): @@ -142,11 +123,13 @@ def tokenize_thai(text): rdd.map(tokenize_thai) ``` -This ensures each worker uses a local writable directory. See `examples/distributed_pyspark.py` for more examples. +This ensures each worker uses a local writable directory. +See `examples/distributed_pyspark.py` for more examples. ## Command-Line Interface -Some of PyThaiNLP functionalities can be used via command line with the `thainlp` command. +Some of PyThaiNLP functionalities can be used via command line +with the `thainlp` command. For example, to display a catalog of datasets: @@ -176,7 +159,7 @@ For more detailed information on testing, please refer to the tests README: ## Licenses | | License | -|:---|:----| +| :-- | :-- | | PyThaiNLP source codes and notebooks | [Apache Software License 2.0](https://github.com/PyThaiNLP/pythainlp/blob/dev/LICENSE) | | Corpora, datasets, and documentations created by PyThaiNLP | [Creative Commons Zero 1.0 Universal Public Domain Dedication License (CC0)](https://creativecommons.org/publicdomain/zero/1.0/)| | Language models created by PyThaiNLP | [Creative Commons Attribution 4.0 International Public License (CC-by)](https://creativecommons.org/licenses/by/4.0/) | diff --git a/docs/notes/installation.rst b/docs/notes/installation.rst index 44a8d81c5..1d6ad73a7 100644 --- a/docs/notes/installation.rst +++ b/docs/notes/installation.rst @@ -14,33 +14,41 @@ Some features (for example, named entity recognition) require additional optiona pip install pythainlp[extra1,extra2,...] The extras can include: + - ``compact`` — install a stable and small subset of dependencies (recommended) + - ``full`` — install all optional dependencies (may introduce large dependencies and conflicts) + - ``abbreviation`` — abbreviation expansion utilities - ``attacut`` — support for AttaCut (a fast and accurate tokenizer) + - ``budoux`` — support for BudouX text segmentation - ``benchmarks`` — support for running benchmarks + - ``coreference_resolution`` — coreference resolution support + - ``dependency_parsing`` — dependency parsing support + - ``el`` — entity linking support + - ``esupar`` — ESuPAR parser support + - ``generate`` — support for text generation - ``icu`` — support for ICU (International Components for Unicode) used in transliteration and tokenization - ``ipa`` — support for IPA (International Phonetic Alphabet) in transliteration - ``ml`` — support for ULMFiT models used in classification - - ``ssg`` — support for SSG (syllable tokenizer) - - ``thai2fit`` — Thai word vectors (thai2fit) - - ``thai2rom`` — machine-learned romanization - - ``translate`` — translation support - - ``wangchanberta`` — WangchanBERTa models - ``mt5`` — mT5 models for Thai text summarization - - ``wordnet`` — WordNet support - - ``spell`` — support for spell-checkers (phunspell & symspellpy) - - ``generate`` — support for text generation (ULMFiT or thai2fit) - - ``textaugment`` — text augmentation utilities + - ``nlpo3`` — nlpo3 Thai word tokenization support + - ``onnx`` - ONNX model support - ``oskut`` — OSKUT support - - ``nlpo3`` — NLPO3 engine support - - ``spacy_thai`` — spaCy Thai tokenizer integration - - ``esupar`` — ESuPAR support + - ``sefr_cut`` — SEFR CUT Thai word tokenization support + - ``spacy_thai`` — spaCy Thai language support + - ``spell`` — support for more spell-checkers (phunspell & symspellpy) + - ``ssg`` — support for SSG syllable tokenizer + - ``textaugment`` — text augmentation utilities + - ``thai_nner`` — Thai named entity recognition support + - ``thai2fit`` — Thai word vectors (thai2fit) + - ``thai2rom`` — machine-learned romanization - ``transformers_ud`` — transformers_ud engine support - - ``dependency_parsing`` — dependency parsing engines - - ``coreference_resolution`` — coreference resolution engines + - ``translate`` — machine translation support + - ``wangchanberta`` — WangchanBERTa models - ``wangchanglm`` — WangchangLM model support + - ``word_approximation`` — word approximation support + - ``wordnet`` — WordNet support - ``wsd`` — word-sense disambiguation support (pythainlp.wsd) - - ``el`` — EL support (pythainlp.el) - - ``abbreviation`` — abbreviation expansion utilities - - ``full`` — install all optional dependencies + - ``wtp`` — Where's the Point text segmentation support + - ``wunsen`` — Wunsen spell checker support For dependency details, see the `project.optional-dependencies` section in `pyproject.toml `_. diff --git a/pyproject.toml b/pyproject.toml index 0f5b79ba3..d93d8f9ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ description = "Thai Natural Language Processing library" readme = "README.md" requires-python = ">=3.9" license = "Apache-2.0" -license-files = ["LICENSE", "pythainlp/corpus/corpus_license.md"] authors = [ { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@pythainlp.org" }, { name = "Korakot Chaovavanich" }, @@ -68,6 +67,7 @@ dependencies = [ [project.optional-dependencies] +# Development and testing dependencies dev = [ "black>=25.11.0", "bump-my-version>=1.2.6 ", @@ -77,6 +77,15 @@ dev = [ "tox>=4.30.3", ] +# Documentation dependencies +docs = [ + "Sphinx>=6.2", + "sphinx-copybutton>=0.5.2", + "sphinx-rtd-theme>=3.1.0", +] + +# Below are dependencies for optional features + abbreviation = ["khamyo>=0.2.0"] attacut = ["attacut>=1.0.6"] @@ -107,7 +116,7 @@ ml = ["numpy>=1.22", "torch>=1.0.0"] mt5 = ["sentencepiece>=0.1.91", "transformers>=4.22.1"] -nlpo3 = ["nlpo3>=1.3.1"] +nlpo3 = ["nlpo3>=1.4.0"] onnx = ["numpy>=1.22", "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"] @@ -155,56 +164,31 @@ wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"] wunsen = ["wunsen>=0.0.3"] +# Below are defined groups of CI testing dependencies + # Compact dependencies - safe small set of optional dependencies compact = [ - "nlpo3>=1.3.1", - "numpy>=1.22", + "nlpo3>=1.4.0", + "numpy>=1.26.0", "pyicu>=2.3", "python-crfsuite>=0.9.7", "PyYAML>=5.4.1", ] -# Full dependencies - comprehensive set of all optional features -full = [ - "attacut>=1.0.6", +# Extra dependencies - for tests.extra modules +# Note: Some packages excluded due to Python 3.13 compatibility issues +extra = [ "bpemb>=0.3.2", "budoux>=0.7.0", - "emoji>=0.5.1", - "epitran>=1.1", - 'fairseq>=0.10.0,<0.13;python_version<"3.11"', - 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', - "fastai<2.0", - "fastcoref>=2.1.5", "gensim>=4.0.0", - "khamyo>=0.2.0", - "nlpo3>=1.3.1", "nltk>=3.3", - "numpy>=1.22", - "onnxruntime>=1.10.0", - "oskut>=1.3", "pandas>=0.24", - "panphon>=0.20.0", - "phunspell>=0.1.6", - "pyicu>=2.3", - "sacremoses>=0.0.41", - "sefr_cut>=1.1", - "sentencepiece>=0.1.91", - "sentence-transformers>=2.2.2", - "spacy>=3.0", - "spacy_thai>=0.7.1", "ssg>=0.0.8", "symspellpy>=6.7.6", - "thai_nner>=0.3", - "torch>=1.0.0", - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - "word2word>=1.0.0", - "wtpsplit>=1.0.1", - "wunsen>=0.0.3", ] -# Testing dependencies - pinned versions for CI/CD reproducibility -testing = [ +# Full dependencies - pinned where available +full = [ "attacut==1.0.6", "bpemb>=0.3.6,<0.4", "budoux==0.7.0", @@ -217,10 +201,13 @@ testing = [ "fastai>=1.0.61,<2", "fastcoref==2.1.6", "gensim>=4.3.3,<5", + "khamyo>=0.3.0", "khanaa>=0.1.1,<1", - "nlpo3>=1.3.1", + "nlpo3>=1.4.0", "nltk>=3.6.6,<4", "numpy>=1.26.0,<3", + "onnxruntime>=1.10.0", + "oskut>=1.3", "pandas>=2.2.0,<3", "panphon==0.22.2", "phunspell==0.1.6", @@ -233,6 +220,7 @@ testing = [ "spacy==3.8.7,<4", "spacy_thai==0.7.8", "ssg==0.0.8", + "sefr_cut>=1.1", "symspellpy==6.9.0", "thai-nner==0.3", "tltk>=1.6.8,<2", @@ -244,12 +232,6 @@ testing = [ "wunsen==0.0.3", ] -docs = [ - "Sphinx>=6.2", - "sphinx-copybutton>=0.5.2", - "sphinx-rtd-theme>=3.1.0", -] - [project.urls] homepage = "https://pythainlp.org/" source = "https://github.com/PyThaiNLP/pythainlp.git" diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index 5bc8c44bb..96876ab6e 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -30,15 +30,15 @@ def __init__( self.load(model_path) else: self.training_data = np.array(training_data) - self.Cx2_list = self.train() + self.cx2_list = self.train() def train(self): - Cx2_list = [] + temp_list = [] for i in range(len(self.training_data)): - Cx2_list.append( + temp_list.append( len(gzip.compress(self.training_data[i][0].encode("utf-8"))) ) - return Cx2_list + return temp_list def predict(self, x1: str, k: int = 1) -> str: """:param str x1: the text that we want to predict label for. @@ -66,15 +66,15 @@ def predict(self, x1: str, k: int = 1) -> str: print(model.predict("ฉันดีใจ", k=1)) # output: Positive """ - Cx1 = len(gzip.compress(x1.encode("utf-8"))) + cx1 = len(gzip.compress(x1.encode("utf-8"))) disance_from_x1 = [] - for i in range(len(self.Cx2_list)): + for i in range(len(self.cx2_list)): x2 = self.training_data[i][0] - Cx2 = self.Cx2_list[i] + cx2 = self.cx2_list[i] x1x2 = "".join([x1, x2]) - Cx1x2 = len(gzip.compress(x1x2.encode("utf-8"))) + cx1x2 = len(gzip.compress(x1x2.encode("utf-8"))) # normalized compression distance - ncd = (Cx1x2 - min(Cx1, Cx2)) / max(Cx1, Cx2) + ncd = (cx1x2 - min(cx1, cx2)) / max(cx1, cx2) disance_from_x1.append(ncd) sorted_idx = np.argsort(np.array(disance_from_x1)) @@ -85,22 +85,20 @@ def predict(self, x1: str, k: int = 1) -> str: return predict_class def save(self, path: str): - """:param str path: path to save model - """ + """:param str path: path to save model""" with open(path, "w", encoding="utf-8") as f: json.dump( { "training_data": self.training_data.tolist(), - "Cx2_list": self.Cx2_list, + "cx2_list": self.cx2_list, }, f, ensure_ascii=False, ) def load(self, path: str): - """:param str path: path to load model - """ + """:param str path: path to load model""" with open(path, "r", encoding="utf-8") as f: data = json.load(f) - self.Cx2_list = data["Cx2_list"] + self.cx2_list = data["cx2_list"] self.training_data = np.array(data["training_data"]) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 48520b512..36596032a 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -358,11 +358,7 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str text = text.replace("!", "!") text = text.replace("", ".") sentences = text.split("") - sentences = list(map(str.strip, sentences)) - if "nan" in sentences: - sentences.remove("nan") - - sentences = list(filter(None, sentences)) + sentences = [s for s in map(str.strip, sentences) if s and s != "nan"] if isMiddleCut: return middle_cut(sentences) diff --git a/tests/README.md b/tests/README.md index 1f4d0a97b..b1bec0058 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,50 +1,50 @@ # Test cases +The default test suite, triggered by the `unittest tests` command, encompasses +all test cases within the `tests.core` and `tests.compact` packages. +This suite is defined within the `__init__.py` file in this directory. + Tests are categorized into three groups: core, compact, and extra. -## Core Tests (test_*.py) +## Core tests (test_*.py) - Run `unittest tests.core` - Focus on core functionalities. -- Do not rely on external dependencies beyond the standard library, - except for `requests` which is used for corpus downloading. +- Do not rely on external dependencies beyond the standard library. - Test with all officially supported Python versions - (currently 3.9, 3.10, 3.11, 3.12, and 3.13). - -### Robustness Tests (test_robustness.py) - -A comprehensive test suite within core tests that tests edge cases important -for real-world usage: - -- Empty strings and various whitespace handling (spaces, tabs, unicode spaces) -- Special characters from encoding issues, BOM, terminal copy/paste -- Truncated/malformed Unicode and surrogate pairs -- Emoji and modern Unicode sequences (ZWJ, modifiers, flags) -- Control and hidden/invisible characters (zero-width, control chars) -- Thai-specific edge cases with combining characters and mixed scripts -- Multi-engine robustness testing across all core tokenization engines -- Very long strings that can cause performance issues (issue #893) + (currently 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14). -## Compact Tests (testc_*.py) +## Compact tests (testc_*.py) - Run `unittest tests.compact` - Test a limited set of functionalities that rely on a stable and small subset - of optional dependencies specified in `requirements.txt`. + of optional dependencies specified in `pyproject.toml`. - These dependencies are `PyYAML`, `nlpo3`, `numpy`, `pyicu`, `python-crfsuite`, and `requests`. - Test with the latest two stable Python versions. -## Extra Tests (testx_*.py) +## Extra tests (testx_*.py) - Run `unittest tests.extra` - Explore functionalities that rely on optional dependencies specified in the `project.optional-dependencies` section of `pyproject.toml`. -- These dependencies might include libraries like `gensim`, `tltk`, or `torch`. -- Due to dependency complexities, these functionalities are not part of the - automated test suite and will not be tested in the CI/CD pipeline. -## Default Test Suite +## Noauto tests (testn_*.py) -The default test suite, triggered by the `unittest tests` command, encompasses -all test cases within the `tests.core` and `tests.compact` packages. -This suite is defined within the `__init__.py` file in this directory. +- These dependencies might include huge libraries like `tensorflow`. +- Due to dependency complexities, these functionalities may not be tested + in the CI/CD pipeline. + +## Robustness tests (test_robustness.py) + +A comprehensive test suite within core tests that tests edge cases important +for real-world usage: + +- Empty strings and various whitespace handling (spaces, tabs, unicode spaces) +- Special characters from encoding issues, BOM, terminal copy/paste +- Truncated/malformed Unicode and surrogate pairs +- Emoji and modern Unicode sequences (ZWJ, modifiers, flags) +- Control and hidden/invisible characters (zero-width, control chars) +- Thai-specific edge cases with combining characters and mixed scripts +- Multi-engine robustness testing across all core tokenization engines +- Very long strings that can cause performance issues (issue #893) diff --git a/tests/compact/__init__.py b/tests/compact/__init__.py index 9e884d2c4..07f180e28 100644 --- a/tests/compact/__init__.py +++ b/tests/compact/__init__.py @@ -10,6 +10,8 @@ # Names of module to be tested test_packages: list[str] = [ + "tests.compact.testc_classify", + "tests.compact.testc_el", "tests.compact.testc_parse", "tests.compact.testc_tokenize", "tests.compact.testc_tools", diff --git a/tests/compact/test_cli.py b/tests/compact/test_cli.py deleted file mode 100644 index a0ad0a89b..000000000 --- a/tests/compact/test_cli.py +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -"""Unit tests for pythainlp.cli module. (compact) -""" - -import unittest - -from pythainlp import cli -from pythainlp.cli.misspell import App as MisspellApp - - -class CliTestCase(unittest.TestCase): - def test_cli_misspell(self): - self.assertTrue(hasattr(cli, "misspell")) - - with self.assertRaises(SystemExit) as ex: - MisspellApp(["thainlp", "misspell"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - MisspellApp( - [ - "thainlp", - "misspell", - "--file", - "./tests/data/text.txt", - "--seed", - "1", - "--misspell-ratio", - "0.05", - ] - ) - ) diff --git a/tests/compact/testc_classify.py b/tests/compact/testc_classify.py new file mode 100644 index 000000000..4e6002ec5 --- /dev/null +++ b/tests/compact/testc_classify.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from pythainlp.classify import GzipModel + + +class ClsTestCaseC(unittest.TestCase): + def test_GzipModel(self): + training_data = [ + ("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"), + ("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"), + ("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"), + ("ขับรถแย่มาก", "Negative"), + ("ดีนะครับ", "Positive"), + ("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"), + ("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"), + ("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"), + ("นี่เป็นบทความหนึ่ง", "Neutral"), + ] + model = GzipModel(training_data) + self.assertEqual(model.predict("ฉันดีใจ", k=1), "Positive") + # Edge cases: empty string + self.assertIsNotNone(model.predict("", k=1)) + # Edge cases: different k values + self.assertIsNotNone(model.predict("ฉันดีใจ", k=3)) + # Edge cases: k larger than number of classes + self.assertIsNotNone(model.predict("ฉันดีใจ", k=10)) diff --git a/tests/compact/testc_el.py b/tests/compact/testc_el.py new file mode 100644 index 000000000..84476dd36 --- /dev/null +++ b/tests/compact/testc_el.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from pythainlp.el import EntityLinker + + +class ElTestCaseC(unittest.TestCase): + def test_EntityLinker(self): + with self.assertRaises(NotImplementedError): + EntityLinker(model_name="cat") + with self.assertRaises(NotImplementedError): + EntityLinker(tag="cat") diff --git a/tests/compact/testc_parse.py b/tests/compact/testc_parse.py index 4860c4ddf..52d01147d 100644 --- a/tests/compact/testc_parse.py +++ b/tests/compact/testc_parse.py @@ -7,7 +7,7 @@ from pythainlp.tag import chunk_parse, pos_tag -class ChunkParseTestCase(unittest.TestCase): +class ChunkParseTestCaseC(unittest.TestCase): def test_chunk_parse(self): tokens = ["ผม", "รัก", "คุณ"] diff --git a/tests/compact/testc_tokenize.py b/tests/compact/testc_tokenize.py index 7818692dd..5ccc90fd7 100644 --- a/tests/compact/testc_tokenize.py +++ b/tests/compact/testc_tokenize.py @@ -25,7 +25,7 @@ ) -class SentTokenizeCRFCutTestCase(unittest.TestCase): +class SentTokenizeCRFCutTestCaseC(unittest.TestCase): def test_sent_tokenize(self): # Use default engine (crfcut) self.assertEqual(sent_tokenize(None), []) @@ -61,7 +61,7 @@ def test_sent_tokenize(self): ) -class SubwordTokenizeHanSoloTestCase(unittest.TestCase): +class SubwordTokenizeHanSoloTestCaseC(unittest.TestCase): def test_subword_tokenize(self): self.assertEqual(subword_tokenize(None, engine="han_solo"), []) self.assertEqual( @@ -77,7 +77,7 @@ def test_subword_tokenize(self): ) -class WordTokenizeICUTestCase(unittest.TestCase): +class WordTokenizeICUTestCaseC(unittest.TestCase): def test_icu(self): self.assertEqual(pyicu.segment(None), []) self.assertEqual(pyicu.segment(""), []) @@ -90,7 +90,7 @@ def test_word_tokenize_icu(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="icu")) -class WordTokenizeNlpO3TestCase(unittest.TestCase): +class WordTokenizeNlpO3TestCaseC(unittest.TestCase): def test_word_tokenize_nlpo3(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="nlpo3")) self.assertEqual(word_tokenize("การ์", engine="nlpo3"), ["การ์"]) diff --git a/tests/compact/testc_tools.py b/tests/compact/testc_tools.py index 1a68c05e0..6b04c0a1f 100644 --- a/tests/compact/testc_tools.py +++ b/tests/compact/testc_tools.py @@ -20,7 +20,7 @@ def _count_difference(st1: str, st2: str) -> int: return count -class MisspellTestCase(unittest.TestCase): +class MisspellTestCaseC(unittest.TestCase): def setUp(self): self.texts = ["เรารักคุณมากที่สุดในโลก", "เราอยู่ที่มหาวิทยาลัยขอนแก่น"] diff --git a/tests/compact/testc_transliterate.py b/tests/compact/testc_transliterate.py index 3c44128d0..a14472ddf 100644 --- a/tests/compact/testc_transliterate.py +++ b/tests/compact/testc_transliterate.py @@ -7,6 +7,6 @@ from pythainlp.transliterate import transliterate -class TransliterateICUTestCase(unittest.TestCase): +class TransliterateICUTestCaseC(unittest.TestCase): def test_transliterate(self): self.assertEqual(transliterate("แมว", "pyicu"), "mæw") diff --git a/tests/compact/testc_util.py b/tests/compact/testc_util.py index e70272606..33a684549 100644 --- a/tests/compact/testc_util.py +++ b/tests/compact/testc_util.py @@ -7,10 +7,11 @@ import unittest +from pythainlp.util import rhyme, thai_word_tone_detector from pythainlp.util.spell_words import spell_word -class SpellWordTestCase(unittest.TestCase): +class SpellWordTestCaseC(unittest.TestCase): def test_spell_word(self): self.assertEqual(spell_word("เสือ"), ["สอ", "เอือ", "เสือ"]) self.assertEqual(spell_word("เสื้อ"), ["สอ", "เอือ", "ไม้โท", "เสื้อ"]) @@ -21,3 +22,18 @@ def test_spell_word(self): # Edge cases: None and empty string self.assertEqual(spell_word(None), []) self.assertEqual(spell_word(""), []) + + +class UtilTestCaseC(unittest.TestCase): + def test_rhyme(self): + self.assertIsInstance(rhyme("แมว"), list) + self.assertGreater(len(rhyme("แมว")), 2) + + def test_thai_word_tone_detector(self): + self.assertIsNotNone(thai_word_tone_detector("คนดี")) + self.assertEqual( + thai_word_tone_detector("ราคา"), [("รา", "m"), ("คา", "m")] + ) + # Edge cases: None and empty string + self.assertEqual(thai_word_tone_detector(None), [("", "")]) + self.assertEqual(thai_word_tone_detector(""), [("", "")]) diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index bdd7572c2..01883908f 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -7,6 +7,7 @@ from pythainlp import __main__, cli from pythainlp.cli.data import App as DataApp +from pythainlp.cli.misspell import App as MisspellApp from pythainlp.cli.soundex import App as SoundexApp from pythainlp.cli.tag import App as TagApp from pythainlp.cli.tokenize import App as TokenizeApp @@ -47,6 +48,28 @@ def test_cli_data(self): self.assertIsNotNone(DataApp(["thainlp", "data", "info", "NOT_EXIST"])) self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "NOT_EXIST"])) + def test_cli_misspell(self): + self.assertTrue(hasattr(cli, "misspell")) + + with self.assertRaises(SystemExit) as ex: + MisspellApp(["thainlp", "misspell"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + MisspellApp( + [ + "thainlp", + "misspell", + "--file", + "./tests/data/text.txt", + "--seed", + "1", + "--misspell-ratio", + "0.05", + ] + ) + ) + def test_cli_soundex(self): self.assertTrue(hasattr(cli, "soundex")) diff --git a/tests/extra/__init__.py b/tests/extra/__init__.py index f9f519fce..a1ce14666 100644 --- a/tests/extra/__init__.py +++ b/tests/extra/__init__.py @@ -9,7 +9,15 @@ from unittest import TestLoader, TestSuite # Names of module to be tested -test_packages: list[str] = [] +test_packages: list[str] = [ + "tests.extra.testx_augment", + "tests.extra.testx_benchmarks", + "tests.extra.testx_cli", + "tests.extra.testx_spell", + "tests.extra.testx_tag", + "tests.extra.testx_tokenize", + "tests.extra.testx_word_vector", +] def load_tests( diff --git a/tests/extra/testx_augment.py b/tests/extra/testx_augment.py index 5276e58e8..69594c42d 100644 --- a/tests/extra/testx_augment.py +++ b/tests/extra/testx_augment.py @@ -11,9 +11,10 @@ # from pythainlp.augment.lm import Thai2transformersAug # from pythainlp.augment.lm.phayathaibert import ThaiTextAugmenter from pythainlp.augment.word2vec import LTW2VAug -from pythainlp.augment.word2vec.bpemb_wv import BPEmbAug from pythainlp.augment.wordnet import postype2wordnet +# from pythainlp.augment.word2vec.bpemb_wv import BPEmbAug + class AugmentTestCaseX(unittest.TestCase): def setUp(self): @@ -34,10 +35,11 @@ def test_WordNetAug(self): # self.assertIsNotNone(_aug.tokenizer(self.text)) # self.assertIsNotNone(_aug.augment(self.text, n_sent=3, p=0.5)) - def test_BPEmbAug(self): - _aug = BPEmbAug() - self.assertIsNotNone(_aug.tokenizer(self.text)) - self.assertIsNotNone(_aug.augment(self.text, n_sent=3, p=0.5)) + # def test_BPEmbAug(self): + # # Skipped: Requires network access to download model from nlp.h-its.org + # _aug = BPEmbAug() + # self.assertIsNotNone(_aug.tokenizer(self.text)) + # self.assertIsNotNone(_aug.augment(self.text, n_sent=3, p=0.5)) def test_LTW2VAug(self): _aug = LTW2VAug() diff --git a/tests/extra/testx_spell.py b/tests/extra/testx_spell.py index edddac781..f932880df 100644 --- a/tests/extra/testx_spell.py +++ b/tests/extra/testx_spell.py @@ -2,12 +2,14 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +# Tests for spell functions that need extra dependencies +# Note: Tests requiring phunspell/tltk/torch/HuggingFace Hub have been moved to tests.noauto + import unittest from pythainlp.spell import ( correct, correct_sent, - get_words_spell_suggestion, spell, spell_sent, symspellpy, @@ -18,14 +20,7 @@ class SpellTestCaseX(unittest.TestCase): def test_spell(self): - result = spell("เน้ร", engine="phunspell") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - result = spell("เกสมร์", engine="phunspell") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - + # Tests for symspellpy only (phunspell and tltk moved to noauto) result = spell("เน้ร", engine="symspellpy") self.assertIsInstance(result, list) self.assertGreater(len(result), 0) @@ -34,39 +29,17 @@ def test_spell(self): self.assertIsInstance(result, list) self.assertGreater(len(result), 0) - result = spell("เน้ร", engine="tltk") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - - result = spell("เดก", engine="tltk") - self.assertIsInstance(result, list) - self.assertGreater(len(result), 0) - def test_word_correct(self): - result = correct("ทดสอง", engine="phunspell") - self.assertIsInstance(result, str) - self.assertNotEqual(result, "") - + # Tests for symspellpy only (phunspell and wanchanberta moved to noauto) result = correct("ทดสอง", engine="symspellpy") self.assertIsInstance(result, str) self.assertNotEqual(result, "") - result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") - self.assertIsInstance(result, str) - self.assertNotEqual(result, "") - def test_spell_sent(self): - self.assertIsNotNone(spell_sent(SENT_TOKS, engine="phunspell")) + # Tests for symspellpy only (phunspell moved to noauto) self.assertIsNotNone(spell_sent(SENT_TOKS, engine="symspellpy")) def test_correct_sent(self): - self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) + # Tests for symspellpy only (phunspell and wanchanberta moved to noauto) self.assertIsNotNone(correct_sent(SENT_TOKS, engine="symspellpy")) - self.assertIsNotNone( - correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") - ) self.assertIsNotNone(symspellpy.correct_sent(SENT_TOKS)) - - def test_get_words_spell_suggestion(self): - self.assertIsNotNone(get_words_spell_suggestion("คมดี")) - self.assertIsNotNone(get_words_spell_suggestion(["คมดี","มะนา"])) diff --git a/tests/extra/testx_tag.py b/tests/extra/testx_tag.py index f5821e826..024661ddb 100644 --- a/tests/extra/testx_tag.py +++ b/tests/extra/testx_tag.py @@ -2,55 +2,17 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +# Tests for tag functions that need extra dependencies +# Note: Tests requiring transformers/tltk have been moved to tests.noautotest + import unittest -from pythainlp.tag import ( - NER, - NNER, - pos_tag, - pos_tag_transformers, - tltk, -) from pythainlp.tag.thainer import ThaiNameTagger class TagTestCaseX(unittest.TestCase): - # ### pythainlp.tag.pos_tag - - def test_pos_tag(self): - tokens = ["ผม", "รัก", "คุณ"] - self.assertIsNotNone(pos_tag(tokens, engine="tltk")) - with self.assertRaises(ValueError): - tltk.pos_tag(tokens, corpus="blackboard") - - # ### pythainlp.tag.named_entity - - def test_tltk_ner(self): - self.assertEqual(tltk.get_ner(""), []) - self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone( - tltk.get_ner("พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า") - ) - self.assertIsNotNone( - tltk.get_ner( - "พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า", - tag=True, - ) - ) - self.assertIsNotNone( - tltk.get_ner( - """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น - จังหวัดหนองคาย 43000""" - ) - ) - self.assertIsNotNone( - tltk.get_ner( - """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น - จังหวัดหนองคาย 43000""", - tag=True, - ) - ) + # Tests for ThaiNameTagger (doesn't require transformers or tltk) + # All tltk and transformers-based tests have been moved to tests.noautotest def test_thai_name_tagger_1_5(self): ner = ThaiNameTagger(version="1.5") @@ -155,63 +117,3 @@ def test_thai_name_tagger_1_4(self): ) ) - def test_NER_class(self): - with self.assertRaises(ValueError): - NER(engine="thainer", corpus="cat") - - ner = NER(engine="thainer") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="thainer-v2") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="wangchanberta") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - ner = NER(engine="tltk") - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) - self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) - - def test_NNER_class(self): - nner = NNER() - self.assertIsNotNone(nner.tag("แมวทำอะไรตอนห้าโมงเช้า")) - - def test_pos_tag_transformers(self): - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="bert", - corpus="blackboard", - ) - ) - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="mdeberta", - corpus="pud", - ) - ) - self.assertIsNotNone( - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="wangchanberta", - corpus="pud", - ) - ) - with self.assertRaises(ValueError): - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", engine="non-existing-engine" - ) - with self.assertRaises(ValueError): - pos_tag_transformers( - sentence="แมวทำอะไรตอนห้าโมงเช้า", - engine="bert", - corpus="non-existing corpus", - ) diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index b07f7569c..d76f3e0bb 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -3,21 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 # Tests for tokenize functions that need extra dependencies +# Note: Tests requiring TensorFlow/Keras/tltk/torch/transformers have been moved to tests.noauto import unittest from pythainlp.tokenize import ( - attacut, - deepcut, nercut, - oskut, - paragraph_tokenize, - sefr_cut, sent_tokenize, ssg, subword_tokenize, - tltk, - word_dict_trie, word_tokenize, ) @@ -30,91 +24,6 @@ ) -class DetokenizeTestCase(unittest.TestCase): - def test_numeric_data_format(self): - engines = ["attacut", "deepcut", "sefr_cut"] - - for engine in engines: - self.assertIn( - "127.0.0.1", - word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine=engine), - ) - - tokens = word_tokenize( - "เวลา 12:12pm มีโปรโมชั่น 11.11", engine=engine - ) - self.assertTrue( - any(value in tokens for value in ["12:12pm", "12:12"]), - msg=f"{engine}: {tokens}", - ) - self.assertIn("11.11", tokens) - - self.assertIn( - "1,234,567.89", - word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine=engine), - ) - - tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine=engine) - self.assertIn("2.5:1", tokens) - self.assertIn("5:2", tokens) - - # try turning off `join_broken_num` - engine = "attacut" - self.assertNotIn( - "127.0.0.1", - word_tokenize( - "ไอพีของคุณคือ 127.0.0.1 ครับ", - engine=engine, - join_broken_num=False, - ), - ) - self.assertNotIn( - "1,234,567.89", - word_tokenize( - "รางวัลมูลค่า 1,234,567.89 บาท", - engine=engine, - join_broken_num=False, - ), - ) - - -class ParagraphTokenizeTestCase(unittest.TestCase): - def test_paragraph_tokenize(self): - sent = ( - "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" - "จากผลงานวิจัยที่เคยทำมาในอดีต" - " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" - " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" - ) - self.assertIsNotNone(paragraph_tokenize(sent)) - with self.assertRaises(ValueError): - paragraph_tokenize( - sent, engine="ai2+2thai" - ) # engine does not exist - - -class SentTokenizeTLTKTestCase(unittest.TestCase): - def test_sent_tokenize_tltk(self): - self.assertIsNotNone( - sent_tokenize( - SENT_1, - engine="tltk", - ), - ) - self.assertIsNotNone( - sent_tokenize( - SENT_2, - engine="tltk", - ), - ) - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="tltk", - ), - ) - - class SentTokenizeThaiSumTestCase(unittest.TestCase): def test_sent_tokenize_thaisum(self): self.assertIsNotNone( @@ -137,52 +46,7 @@ def test_sent_tokenize_thaisum(self): ) self.assertEqual( sent_tokenize(SENT_4, engine="thaisum"), - [["ผม", "กิน", "ข้าว", " ", "เธอ", "เล่น", "เกม"]], - ) - - -class SentTokenizeWTPTestCase(unittest.TestCase): - def test_sent_tokenize_wtp(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp", - ), - ) - - def test_sent_tokenize_wtp_tiny(self): - self.assertIsNotNone( - sent_tokenize( - SENT_3, - engine="wtp-tiny", - ), - ) - # self.assertIsNotNone( - # sent_tokenize( - # SENT_3, - # engine="wtp-base", - # ), - # ) - # self.assertIsNotNone( - # sent_tokenize( - # SENT_3, - # engine="wtp-large", - # ), - # ) - - -class SubwordTokenizePhayathaiTestCase(unittest.TestCase): - def test_subword_tokenize_phayathai(self): - self.assertEqual(subword_tokenize(None, engine="phayathai"), []) - self.assertEqual(subword_tokenize("", engine="phayathai"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="phayathai"), list - ) - self.assertNotIn( - "า", subword_tokenize("สวัสดีดาวอังคาร", engine="phayathai") - ) - self.assertIsInstance( - subword_tokenize("โควิด19", engine="phayathai"), list + [["ผม", "กิน", "ข้าว", " ", " ", "เธอ", "เล่น", "เกม"]], ) @@ -198,92 +62,6 @@ def test_subword_tokenize_ssg(self): self.assertNotIn("า", subword_tokenize("สวัสดีดาวอังคาร", engine="ssg")) -class SubwordTokenizeTLTKTestCase(unittest.TestCase): - def test_subword_tokenize_tltk(self): - self.assertEqual(subword_tokenize(None, engine="tltk"), []) - self.assertEqual(subword_tokenize("", engine="tltk"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="tltk"), list - ) - self.assertNotIn("า", subword_tokenize("สวัสดีดาวอังคาร", engine="tltk")) - self.assertIsInstance(subword_tokenize("โควิด19", engine="tltk"), list) - - -class SubwordTokenizeWangchanbertaTestCase(unittest.TestCase): - def test_subword_tokenize_wangchanberta(self): - self.assertEqual(subword_tokenize(None, engine="wangchanberta"), []) - self.assertEqual(subword_tokenize("", engine="wangchanberta"), []) - self.assertIsInstance( - subword_tokenize("สวัสดิีดาวอังคาร", engine="wangchanberta"), list - ) - self.assertNotIn( - "า", subword_tokenize("สวัสดีดาวอังคาร", engine="wangchanberta") - ) - self.assertIsInstance( - subword_tokenize("โควิด19", engine="wangchanberta"), list - ) - - -class SyllableTokenizeTLTKTestCase(unittest.TestCase): - def test_tltk(self): - self.assertEqual(tltk.segment(None), []) - self.assertEqual(tltk.segment(""), []) - self.assertEqual( - tltk.syllable_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - [ - "ฉัน", - "รัก", - "ภา", - "ษา", - "ไทย", - "เพราะ", - "ฉัน", - "เป็น", - "คน", - "ไทย", - ], - ) - self.assertEqual(tltk.syllable_tokenize(None), []) - self.assertEqual(tltk.syllable_tokenize(""), []) - - -class WordTokenizeAttacutTestCase(unittest.TestCase): - def test_word_tokenize_attacut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) - - def test_attacut(self): - self.assertEqual(attacut.segment(None), []) - self.assertEqual(attacut.segment(""), []) - self.assertEqual( - word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertEqual( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), - ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], - ) - self.assertIsNotNone( - attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") - ) - - -class WordTokenizeDeepcutTestCase(unittest.TestCase): - def test_word_tokenize_deepcut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) - - def test_deepcut(self): - self.assertEqual(deepcut.segment(None), []) - self.assertEqual(deepcut.segment(""), []) - self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) - self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) - self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) - self.assertIsNotNone( - word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() - ) - ) - - class WordTokenizeNERCutTestCase(unittest.TestCase): def test_word_tokenize_nercut(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="nercut")) @@ -299,41 +77,7 @@ def test_nercut(self): self.assertIsNotNone(word_tokenize("ทดสอบ", engine="nercut")) -class WordTokenizeOSKutTestCase(unittest.TestCase): - def test_word_tokenize_oskut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) - - def test_oskut(self): - self.assertEqual(oskut.segment(None), []) - self.assertEqual(oskut.segment(""), []) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), - ) - - -class WordTokenizeSEFRCutTestCase(unittest.TestCase): - def test_word_tokenize_sefr_cut(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) - - def test_sefr_cut(self): - self.assertEqual(sefr_cut.segment(None), []) - self.assertEqual(sefr_cut.segment(""), []) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), - ) - self.assertIsNotNone( - sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), - ) - - -class WordTokenizeTLTKTestCase(unittest.TestCase): - def test_word_tokenize_tltk(self): - self.assertIsNotNone(word_tokenize(TEXT_1, engine="tltk")) - - class WordTokenizeBudouxTestCase(unittest.TestCase): def test_word_tokenize_budoux(self): self.assertIsNotNone(word_tokenize(TEXT_1, engine="budoux")) + diff --git a/tests/noauto/__init__.py b/tests/noauto/__init__.py new file mode 100644 index 000000000..d7cf7bb3d --- /dev/null +++ b/tests/noauto/__init__.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Unit test. No auto test version. + +Test functions that require dependencies that are: +- Very large (TensorFlow, transformers, torch) +- Have compilation issues (Cython-based packages) +- Take a long time to install or run +- Have compatibility issues with latest Python versions + +These tests are NOT run in automated CI workflows but are kept for +manual testing and future re-enabling when dependencies improve. +""" + +from unittest import TestLoader, TestSuite + +# Names of module to be tested +# Note: These tests are NOT included in automated CI runs +test_packages: list[str] = [ + "tests.noauto.testn_spell", + "tests.noauto.testn_tag", + "tests.noauto.testn_tokenize", +] + + +def load_tests( + loader: TestLoader, standard_tests: TestSuite, pattern: str +) -> TestSuite: + """Load test protocol + See: https://docs.python.org/3/library/unittest.html#id1 + """ + suite = TestSuite() + for test_package in test_packages: + tests = loader.loadTestsFromName(test_package) + suite.addTests(tests) + return suite + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tests/noauto/testn_spell.py b/tests/noauto/testn_spell.py new file mode 100644 index 000000000..b9fbc4668 --- /dev/null +++ b/tests/noauto/testn_spell.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for spell functions that require phunspell (Cython) or tltk +# These tests are NOT run in automated CI workflows due to: +# - Compilation issues (phunspell requires Cython) +# - Compilation issues (tltk) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.spell import ( + correct, + correct_sent, + get_words_spell_suggestion, + spell, + spell_sent, +) + +from ..core.test_spell import SENT_TOKS + + +class SpellPhunspellTestCaseN(unittest.TestCase): + """Tests for phunspell engine (requires Cython compilation)""" + + def test_spell_phunspell(self): + result = spell("เน้ร", engine="phunspell") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = spell("เกสมร์", engine="phunspell") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_word_correct_phunspell(self): + result = correct("ทดสอง", engine="phunspell") + self.assertIsInstance(result, str) + self.assertNotEqual(result, "") + + def test_spell_sent_phunspell(self): + self.assertIsNotNone(spell_sent(SENT_TOKS, engine="phunspell")) + + def test_correct_sent_phunspell(self): + self.assertIsNotNone(correct_sent(SENT_TOKS, engine="phunspell")) + + +class SpellTLTKTestCaseN(unittest.TestCase): + """Tests for tltk engine (requires tltk with compilation issues)""" + + def test_spell_tltk(self): + result = spell("เน้ร", engine="tltk") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = spell("เดก", engine="tltk") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + +class SpellWanchanbertaTestCaseN(unittest.TestCase): + """Tests for wanchanberta_thai_grammarly engine (requires torch)""" + + def test_word_correct_wanchanberta(self): + result = correct("ทดสอง", engine="wanchanberta_thai_grammarly") + self.assertIsInstance(result, str) + self.assertNotEqual(result, "") + + def test_correct_sent_wanchanberta(self): + from ..core.test_spell import SENT_TOKS + self.assertIsNotNone( + correct_sent(SENT_TOKS, engine="wanchanberta_thai_grammarly") + ) + + +class SpellHuggingFaceTestCaseN(unittest.TestCase): + """Tests for get_words_spell_suggestion (requires HuggingFace Hub network access)""" + + def test_get_words_spell_suggestion(self): + self.assertIsNotNone(get_words_spell_suggestion("คมดี")) + self.assertIsNotNone(get_words_spell_suggestion(["คมดี", "มะนา"])) diff --git a/tests/noauto/testn_tag.py b/tests/noauto/testn_tag.py new file mode 100644 index 000000000..1397b09e3 --- /dev/null +++ b/tests/noauto/testn_tag.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tag functions that require transformers or tltk +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (transformers, torch) +# - Compilation issues (tltk) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tag import ( + NER, + NNER, + pos_tag, + pos_tag_transformers, + tltk, +) + + +class TagTLTKTestCaseN(unittest.TestCase): + """Tests for tltk engine (requires tltk with compilation issues)""" + + def test_pos_tag_tltk(self): + tokens = ["ผม", "รัก", "คุณ"] + self.assertIsNotNone(pos_tag(tokens, engine="tltk")) + with self.assertRaises(ValueError): + tltk.pos_tag(tokens, corpus="blackboard") + + def test_tltk_ner(self): + self.assertEqual(tltk.get_ner(""), []) + self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone( + tltk.get_ner("พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า") + ) + self.assertIsNotNone( + tltk.get_ner( + "พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า", + tag=True, + ) + ) + self.assertIsNotNone( + tltk.get_ner( + """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น + จังหวัดหนองคาย 43000""" + ) + ) + self.assertIsNotNone( + tltk.get_ner( + """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น + จังหวัดหนองคาย 43000""", + tag=True, + ) + ) + + +class TagTransformersTestCaseN(unittest.TestCase): + """Tests for transformers-based engines (requires transformers, torch)""" + + def test_NER_class(self): + with self.assertRaises(ValueError): + NER(engine="thainer", corpus="cat") + + ner = NER(engine="thainer") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="thainer-v2") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="wangchanberta") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + ner = NER(engine="tltk") + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False)) + self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True)) + + def test_NNER_class(self): + nner = NNER() + self.assertIsNotNone(nner.tag("แมวทำอะไรตอนห้าโมงเช้า")) + + def test_pos_tag_transformers(self): + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="bert", + corpus="blackboard", + ) + ) + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="mdeberta", + corpus="pud", + ) + ) + self.assertIsNotNone( + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="wangchanberta", + corpus="pud", + ) + ) + with self.assertRaises(ValueError): + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", engine="non-existing-engine" + ) + with self.assertRaises(ValueError): + pos_tag_transformers( + sentence="แมวทำอะไรตอนห้าโมงเช้า", + engine="bert", + corpus="non-existing corpus", + ) diff --git a/tests/noauto/testn_tokenize.py b/tests/noauto/testn_tokenize.py new file mode 100644 index 000000000..597d940e6 --- /dev/null +++ b/tests/noauto/testn_tokenize.py @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +# Tests for tokenize functions that require TensorFlow, Keras, or tltk +# These tests are NOT run in automated CI workflows due to: +# - Large dependencies (TensorFlow, Keras) +# - Compilation issues (tltk) +# - Python 3.13+ compatibility issues + +import unittest + +from pythainlp.tokenize import ( + attacut, + deepcut, + oskut, + paragraph_tokenize, + sefr_cut, + sent_tokenize, + subword_tokenize, + tltk, + word_dict_trie, + word_tokenize, +) + +from ..core.test_tokenize import ( + SENT_1, + SENT_2, + SENT_3, + TEXT_1, +) + + +class DetokenizeTestCaseN(unittest.TestCase): + def test_numeric_data_format(self): + engines = ["attacut", "deepcut", "sefr_cut"] + + for engine in engines: + self.assertIn( + "127.0.0.1", + word_tokenize("ไอพีของคุณคือ 127.0.0.1 ครับ", engine=engine), + ) + + tokens = word_tokenize( + "เวลา 12:12pm มีโปรโมชั่น 11.11", engine=engine + ) + self.assertTrue( + any(value in tokens for value in ["12:12pm", "12:12"]), + msg=f"{engine}: {tokens}", + ) + self.assertIn("11.11", tokens) + + self.assertIn( + "1,234,567.89", + word_tokenize("รางวัลมูลค่า 1,234,567.89 บาท", engine=engine), + ) + + tokens = word_tokenize("อัตราส่วน 2.5:1 คือ 5:2", engine=engine) + self.assertIn("2.5:1", tokens) + self.assertIn("5:2", tokens) + + # try turning off `join_broken_num` + engine = "attacut" + self.assertNotIn( + "127.0.0.1", + word_tokenize( + "ไอพีของคุณคือ 127.0.0.1 ครับ", + engine=engine, + join_broken_num=False, + ), + ) + self.assertNotIn( + "1,234,567.89", + word_tokenize( + "รางวัลมูลค่า 1,234,567.89 บาท", + engine=engine, + join_broken_num=False, + ), + ) + + +class SentTokenizeTLTKTestCaseN(unittest.TestCase): + def test_sent_tokenize_tltk(self): + self.assertIsNotNone( + sent_tokenize( + SENT_1, + engine="tltk", + ), + ) + self.assertIsNotNone( + sent_tokenize( + SENT_2, + engine="tltk", + ), + ) + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="tltk", + ), + ) + + +class SubwordTokenizeTLTKTestCaseN(unittest.TestCase): + def test_subword_tokenize_tltk(self): + self.assertEqual(subword_tokenize(None, engine="tltk"), []) + self.assertEqual(subword_tokenize("", engine="tltk"), []) + self.assertIsInstance( + subword_tokenize("สวัสดิีดาวอังคาร", engine="tltk"), list + ) + self.assertNotIn("า", subword_tokenize("สวัสดีดาวอังคาร", engine="tltk")) + self.assertIsInstance(subword_tokenize("โควิด19", engine="tltk"), list) + + +class SyllableTokenizeTLTKTestCaseN(unittest.TestCase): + def test_tltk(self): + self.assertEqual(tltk.segment(None), []) + self.assertEqual(tltk.segment(""), []) + self.assertEqual( + tltk.syllable_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + [ + "ฉัน", + "รัก", + "ภา", + "ษา", + "ไทย", + "เพราะ", + "ฉัน", + "เป็น", + "คน", + "ไทย", + ], + ) + self.assertEqual(tltk.syllable_tokenize(None), []) + self.assertEqual(tltk.syllable_tokenize(""), []) + + +class WordTokenizeAttacutTestCaseN(unittest.TestCase): + def test_word_tokenize_attacut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="attacut")) + + def test_attacut(self): + self.assertEqual(attacut.segment(None), []) + self.assertEqual(attacut.segment(""), []) + self.assertEqual( + word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="attacut"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertEqual( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-sc"), + ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"], + ) + self.assertIsNotNone( + attacut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", model="attacut-c") + ) + + +class WordTokenizeDeepcutTestCaseN(unittest.TestCase): + def test_word_tokenize_deepcut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="deepcut")) + + def test_deepcut(self): + self.assertEqual(deepcut.segment(None), []) + self.assertEqual(deepcut.segment(""), []) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) + self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) + self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) + self.assertIsNotNone( + word_tokenize( + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() + ) + ) + + +class WordTokenizeOSKutTestCaseN(unittest.TestCase): + def test_word_tokenize_oskut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut")) + + def test_oskut(self): + self.assertEqual(oskut.segment(None), []) + self.assertEqual(oskut.segment(""), []) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + oskut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="scads"), + ) + + +class WordTokenizeSEFRCutTestCaseN(unittest.TestCase): + def test_word_tokenize_sefr_cut(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="sefr_cut")) + + def test_sefr_cut(self): + self.assertEqual(sefr_cut.segment(None), []) + self.assertEqual(sefr_cut.segment(""), []) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), + ) + self.assertIsNotNone( + sefr_cut.segment("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="tnhc"), + ) + + +class WordTokenizeTLTKTestCaseN(unittest.TestCase): + def test_word_tokenize_tltk(self): + self.assertIsNotNone(word_tokenize(TEXT_1, engine="tltk")) + + +class ParagraphTokenizeTestCaseN(unittest.TestCase): + def test_paragraph_tokenize(self): + sent = ( + "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา" + "จากผลงานวิจัยที่เคยทำมาในอดีต" + " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" + " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" + ) + self.assertIsNotNone(paragraph_tokenize(sent)) + with self.assertRaises(ValueError): + paragraph_tokenize( + sent, engine="ai2+2thai" + ) # engine does not exist + + +class SentTokenizeWTPTestCaseN(unittest.TestCase): + def test_sent_tokenize_wtp(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp", + ), + ) + + def test_sent_tokenize_wtp_tiny(self): + self.assertIsNotNone( + sent_tokenize( + SENT_3, + engine="wtp-tiny", + ), + ) + + +class SubwordTokenizePhayathaiTestCaseN(unittest.TestCase): + def test_subword_tokenize_phayathai(self): + self.assertEqual(subword_tokenize(None, engine="phayathai"), []) + self.assertEqual(subword_tokenize("", engine="phayathai"), []) + self.assertIsInstance( + subword_tokenize("สวัสดิีดาวอังคาร", engine="phayathai"), list + ) + self.assertNotIn( + "า", subword_tokenize("สวัสดีดาวอังคาร", engine="phayathai") + ) + self.assertIsInstance( + subword_tokenize("โควิด19", engine="phayathai"), list + ) + + +class SubwordTokenizeWangchanbertaTestCaseN(unittest.TestCase): + def test_subword_tokenize_wangchanberta(self): + self.assertEqual(subword_tokenize(None, engine="wangchanberta"), []) + self.assertEqual(subword_tokenize("", engine="wangchanberta"), []) + self.assertIsInstance( + subword_tokenize("สวัสดิีดาวอังคาร", engine="wangchanberta"), list + ) + self.assertNotIn( + "า", subword_tokenize("สวัสดีดาวอังคาร", engine="wangchanberta") + ) + self.assertIsInstance( + subword_tokenize("โควิด19", engine="wangchanberta"), list + )