diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 6c390f24d..897d58552 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -21,7 +21,7 @@ jobs: python-version: "3.10" - name: Install build tools and doc build tools run: | - pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1" + pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" pip install boto smart_open sphinx sphinx-rtd-theme # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index c5c3cdebe..5c73a28a4 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -26,8 +26,7 @@ jobs: SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True run: | python -m pip install --upgrade "pip<24.1" setuptools - python -m pip install -r https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/docker_requirements.txt - python -m pip install pythainlp[full] + python -m pip install pythainlp[testing,full] python -m nltk.downloader omw-1.4 - name: Test run: | diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 9c79a0a08..59683d0d6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -41,7 +41,7 @@ jobs: cache: "pip" - name: Install build tools run: | - pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1" + pip install --upgrade "pip<24.1" "setuptools>=69.0.0,<=73.0.1" pip install coverage coveralls # pip<24.1 because https://github.com/omry/omegaconf/pull/1195 # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1 @@ -70,11 +70,11 @@ jobs: # If torch for the platform is not available in PyPI, use this command: # pip install "" # Get wheel URL from http://download.pytorch.org/whl/torch/ - - name: Install dependencies from docker_requirements.txt + - name: Install testing dependencies if: env.INSTALL_FULL_DEPS == 'true' env: SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True - run: pip install -r docker_requirements.txt + run: pip install ".[testing]" - name: Install PyThaiNLP + dependencies (minimum) if: matrix.python-version != env.PYTHON_VERSION_LATEST && matrix.python-version != env.PYTHON_VERSION_LATEST_2 run: pip install . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87d06b780..667401363 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ --- -SPDX-FileCopyrightText: 2025 PyThaiNLP Project +SPDX-FileCopyrightText: 2025-2026 PyThaiNLP Project SPDX-FileType: DOCUMENTATION SPDX-License-Identifier: CC0-1.0 --- @@ -108,44 +108,79 @@ Make sure the tests pass on GitHub Actions. See more in [tests/README.md](./tests/README.md) +## Installing and Building + +### Installing for Development + +Install PyThaiNLP in editable mode with core dependencies: + +```sh +pip install -e . +``` + +Install with optional dependency groups: + +```sh +# Install with compact set of dependencies (recommended for development) +pip install -e ".[compact]" + +# Install with full dependencies +pip install -e ".[full]" + +# Install with testing dependencies (pinned versions for reproducibility) +pip install -e ".[testing]" +``` + +See all available optional dependency groups in `pyproject.toml` under `[project.optional-dependencies]`. + +### Building Distribution Packages + +To build source distribution and wheel: + +```sh +python -m build +``` + +This will create distribution packages in the `dist/` directory. + ## Releasing - We use [semantic versioning](https://semver.org/): MAJOR.MINOR.PATCH, with development build suffix: MAJOR.MINOR.PATCH-devBUILD -- We use [`bumpversion`](https://github.com/c4urself/bump2version/#installation) to manage versioning. - - `bumpversion [major|minor|patch|release|build]` +- We use [`bump-my-version`](https://github.com/callowayproject/bump-my-version) to manage versioning. The configuration is in `pyproject.toml` under `[tool.bumpversion]`. + - `bump-my-version bump [major|minor|patch|release|build]` - Example: ```sh #current_version = 2.3.3-dev0 - bumpversion build + bump-my-version bump build #current_version = 2.3.3-dev1 - bumpversion build + bump-my-version bump build #current_version = 2.3.3-dev2 - bumpversion release + bump-my-version bump release #current_version = 2.3.3-beta0 - bumpversion release + bump-my-version bump release #current_version = 2.3.3 - bumpversion patch + bump-my-version bump patch #current_version = 2.3.6-dev0 - bumpversion minor + bump-my-version bump minor #current_version = 2.3.1-dev0 - bumpversion build + bump-my-version bump build #current_version = 2.3.1-dev1 - bumpversion major + bump-my-version bump major #current_version = 3.0.0-dev0 - bumpversion release + bump-my-version bump release #current_version = 3.0.0-beta0 - bumpversion release + bump-my-version bump release #current_version = 3.0.0 ``` diff --git a/Dockerfile b/Dockerfile index b26019182..bb8e74ff7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN if [ -f docker_requirements.txt ]; then pip install -r docker_requirements.txt; fi -RUN pip install -e .[full] && pip cache purge +# Install PyThaiNLP with testing dependencies (replaces docker_requirements.txt) +RUN pip install -e ".[testing]" && pip cache purge diff --git a/Makefile b/Makefile index 0f103632c..d0874e03f 100644 --- a/Makefile +++ b/Makefile @@ -50,25 +50,24 @@ lint: ## check style with flake8 flake8 pythainlp tests test: ## run tests quickly with the default Python - python setup.py test + python -m unittest discover test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source pythainlp setup.py test + coverage run --source pythainlp -m unittest discover coverage report -m coverage html $(BROWSER) htmlcov/index.html -release: clean ## package and upload a release - python setup.py sdist upload - python setup.py bdist_wheel upload +release: clean ## package and upload a release (deprecated - use GitHub Actions) + python -m build + python -m twine upload dist/* dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel + python -m build ls -l dist install: clean ## install the package to the active Python's site-packages - python setup.py install + python -m pip install . diff --git a/README.md b/README.md index 14d0ff4e7..f23a05c2f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install pythainlp ## Getting Started -- PyThaiNLP requires Python 3.7+. +- PyThaiNLP requires Python 3.9+. - Python 2.7 users can use PyThaiNLP 1.6. See [2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118) | [Upgrading from 1.7](https://pythainlp.org/docs/2.0/notes/pythainlp-1_7-2_0.html) | [Upgrading ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0) - [PyThaiNLP Get Started notebook](https://pythainlp.org/tutorials/notebooks/pythainlp_get_started.html) | [API document](https://pythainlp.org/docs) | [Tutorials](https://pythainlp.org/tutorials) - [Official website](https://pythainlp.org/) | [PyPI](https://pypi.org/project/pythainlp/) | [Facebook page](https://www.facebook.com/pythainlp/) @@ -81,17 +81,43 @@ Possible `extras`: - `full` (install everything) - `compact` (install a stable and small subset of dependencies) +- `abbreviation` (for Thai abbreviation support) - `attacut` (to support attacut, a fast and accurate tokenizer) - `benchmarks` (for [word tokenization benchmarking](tokenization-benchmark.md)) +- `budoux` (for BudouX text segmentation) +- `coreference_resolution` (for coreference resolution) +- `dependency_parsing` (for dependency parsing) +- `el` (for entity linking) +- `esupar` (for esupar parser support) +- `generate` (for text generation) - `icu` (for ICU, International Components for Unicode, support in transliteration and tokenization) - `ipa` (for IPA, International Phonetic Alphabet, support in transliteration) - `ml` (to support ULMFiT models for classification) +- `mt5` (for mT5 model support) +- `nlpo3` (for nlpo3 Thai word tokenizer) +- `onnx` (for ONNX model support) +- `oskut` (for OSKut Thai word tokenizer) +- `sefr_cut` (for SEFR CUT Thai word tokenizer) +- `spacy_thai` (for spaCy Thai language support) +- `spell` (for spelling correction) +- `ssg` (for sentence segmentation) +- `testing` (pinned versions for CI/CD reproducibility) +- `textaugment` (for text augmentation) +- `thai_nner` (for Thai named entity recognition) - `thai2fit` (for Thai word vector) - `thai2rom` (for machine-learnt romanization) +- `transformers_ud` (for Universal Dependencies with transformers) +- `translate` (for machine translation) +- `wangchanberta` (for WangchanBERTa model) +- `wangchanglm` (for WangchanGLM model) +- `word_approximation` (for word approximation) - `wordnet` (for Thai WordNet API) +- `wsd` (for word sense disambiguation) +- `wtp` (for Where's the Point text segmentation) +- `wunsen` (for Wunsen spell checker) -For dependency details, look at the `extras` variable in -[`setup.py`](https://github.com/PyThaiNLP/pythainlp/blob/dev/setup.py). +For dependency details, look at the `[project.optional-dependencies]` section in +[`pyproject.toml`](https://github.com/PyThaiNLP/pythainlp/blob/dev/pyproject.toml). ## Data Directory diff --git a/README_TH.md b/README_TH.md index 12a060221..452edf008 100644 --- a/README_TH.md +++ b/README_TH.md @@ -86,18 +86,44 @@ pip install pythainlp[extra1,extra2,...] - `full` (ติดตั้งทุกอย่าง) - `compact` (ติดตั้งไลบารีชุดเล็กที่ทดสอบแล้วว่าไม่ตีกันเองและติดตั้งได้ในทุกระบบปฏิบัติการ) +- `abbreviation` (สำหรับการย่อคำภาษาไทย) - `attacut` (เพื่อสนับสนุน attacut ซึ่งเป็นตัวตัดคำที่ทำงานได้รวดเร็วและมีประสิทธิภาพ) - `benchmarks` (สำหรับ [word tokenization benchmarking](tokenization-benchmark.md)) +- `budoux` (สำหรับการแบ่งข้อความด้วย BudouX) +- `coreference_resolution` (สำหรับการหาคำที่อ้างอิงถึงกัน) +- `dependency_parsing` (สำหรับการวิเคราะห์โครงสร้างประโยค) +- `el` (สำหรับการเชื่อมโยงเอนทิตี) +- `esupar` (สำหรับการรองรับ esupar parser) +- `generate` (สำหรับการสร้างข้อความ) - `icu` (สำหรับการรองรับ ICU หรือ International Components for Unicode ในการถอดเสียงเป็นอักษรและการตัดแบ่งคำ) - `ipa` (สำหรับการรองรับ IPA หรือ International Phonetic Alphabet ในการถอดเสียงเป็นอักษร) - `ml` (เพื่อให้สนับสนุนตัวแบบภาษา ULMFiT สำหรับการจำแนกข้อความ) +- `mt5` (สำหรับรองรับโมเดล mT5) +- `nlpo3` (สำหรับตัวตัดคำภาษาไทย nlpo3) +- `onnx` (สำหรับรองรับโมเดล ONNX) +- `oskut` (สำหรับตัวตัดคำภาษาไทย OSKut) +- `sefr_cut` (สำหรับตัวตัดคำภาษาไทย SEFR CUT) +- `spacy_thai` (สำหรับรองรับภาษาไทยใน spaCy) +- `spell` (สำหรับการแก้ไขคำสะกดผิด) +- `ssg` (สำหรับการแบ่งประโยค) +- `testing` (เวอร์ชันที่ปักหมุดสำหรับ CI/CD) +- `textaugment` (สำหรับการเพิ่มข้อมูลข้อความ) +- `thai_nner` (สำหรับการจดจำชื่อเฉพาะภาษาไทย) - `thai2fit` (สำหรับ Thai word vector) - `thai2rom` (สำหรับการถอดอักษรไทยเป็นอักษรโรมัน) +- `transformers_ud` (สำหรับ Universal Dependencies ด้วย transformers) +- `translate` (สำหรับการแปลภาษา) +- `wangchanberta` (สำหรับโมเดล WangchanBERTa) +- `wangchanglm` (สำหรับโมเดล WangchanGLM) +- `word_approximation` (สำหรับการประมาณคำ) - `wordnet` (สำหรับ Thai WordNet API) +- `wsd` (สำหรับการแก้ความกำกวมของความหมายคำ) +- `wtp` (สำหรับการแบ่งข้อความด้วย Where's the Point) +- `wunsen` (สำหรับตัวตรวจการสะกดคำ Wunsen) -สำหรับโมดูลที่ต้องการ สามารถดูรายละเอียดได้ที่ตัวแปร `extras` ใน [`setup.py`](https://github.com/PyThaiNLP/pythainlp/blob/dev/setup.py). +สำหรับโมดูลที่ต้องการ สามารถดูรายละเอียดได้ที่ส่วน `[project.optional-dependencies]` ใน [`pyproject.toml`](https://github.com/PyThaiNLP/pythainlp/blob/dev/pyproject.toml). ## Command-line @@ -117,7 +143,7 @@ thainlp help ## ผู้ใช้งาน Python 2 -- PyThaiNLP สนับสนุน Python 3.7 ขึ้นไป บางความสามารถ สามารถใช้งานกับ Python 3 รุ่นก่อนหน้าได้ แต่ไม่ได้มีการทดสอบว่าใช้งานได้หรือไม่ อ่านเพิ่มเติม [1.7 -> 2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118). +- PyThaiNLP สนับสนุน Python 3.9 ขึ้นไป บางความสามารถ สามารถใช้งานกับ Python 3 รุ่นก่อนหน้าได้ แต่ไม่ได้มีการทดสอบว่าใช้งานได้หรือไม่ อ่านเพิ่มเติม [1.7 -> 2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118). - [Upgrading from 1.7](https://pythainlp.org/docs/2.0/notes/pythainlp-1_7-2_0.html) - [Upgrade ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0) - ผู้ใช้งาน Python 2.7 สามารถใช้งาน PyThaiNLP 1.6 diff --git a/codemeta.json b/codemeta.json index 7df85191a..d7f0babc7 100644 --- a/codemeta.json +++ b/codemeta.json @@ -3,7 +3,7 @@ "@type": "SoftwareSourceCode", "name": "PyThaiNLP", "description": "Thai Natural Language Processing in Python", - "version": "5.1.0", + "version": "5.2.0", "author": [ { "@type": "Person", @@ -61,15 +61,15 @@ "issueTracker": "https://github.com/PyThaiNLP/pythainlp/issues", "url": "https://pythainlp.org/", "keywords": [ + "NLP", "natural language processing", - "Thai", - "Python", + "tokenization", "text processing", + "linguistics", + "localization", "computational linguistics", - "tokenization", - "word segmentation", - "NLP", - "Thai language", - "Thai NLP" + "ThaiNLP", + "Thai NLP", + "Thai language" ] } diff --git a/docker_requirements.txt b/docker_requirements.txt deleted file mode 100644 index 1b80ccfc0..000000000 --- a/docker_requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -PyYAML>=5.4.1,<6.0.2 -attacut==1.0.6 -bpemb>=0.3.6,<0.4 -deepcut==0.7.0.0 -emoji>=0.6.0,<1 -epitran==1.26.0 -esupar>=1.3.9,<2 -fairseq>=0.10.0,<0.13;python_version<"3.11" -fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11" -fastai>=1.0.61,<2 -fastcoref==2.1.6 -gensim>=4.3.3,<5 -khanaa>=0.1.1,<1 -nlpo3>=1.3.1 -nltk>=3.6.6,<4 -numpy>=1.26.0,<3 -pandas>=2.2.0,<3 -panphon==0.22.2 -phunspell==0.1.6 -pyicu>=2.15.2,<3 -python-crfsuite==0.9.12 -requests>=2.32.0,<2.33 -sacremoses==0.1.1 -sentence-transformers>=2.7.0,<3 -sentencepiece==0.2.1 -spacy_thai==0.7.8 -spacy==3.8.7,<4 -ssg==0.0.8 -symspellpy==6.9.0 -thai-nner==0.3 -tltk>=1.6.8,<2 -torch>=1.13.1,<3 -transformers==4.57.3 -ufal.chu-liu-edmonds==1.0.3 -wtpsplit==1.3.0 -wunsen==0.0.3 -word2word>=1.0.0,<2 -budoux==0.7.0 diff --git a/pyproject.toml b/pyproject.toml index b01c8fd28..00f70a467 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,290 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +[build-system] +requires = ["setuptools>=69.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pythainlp" +version = "5.2.0" +description = "Thai Natural Language Processing library" +readme = "README.md" +requires-python = ">=3.9" +license = "Apache-2.0" +license-files = ["LICENSE", "pythainlp/corpus/corpus_license.md"] +authors = [ + { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@pythainlp.org" }, + { name = "Korakot Chaovavanich" }, + { name = "Charin Polpanumas" }, + { name = "Arthit Suriyawongkul", email = "suriyawa@tcd.ie" }, + { name = "Lalita Lowphansirikul" }, + { name = "Pattarawat Chormai" }, + { name = "Peerat Limkonchotiwat" }, + { name = "Thanathip Suntorntip" }, + { name = "Can Udomcharoenchaikit" }, +] +maintainers = [ + { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@pythainlp.org" }, + { name = "Arthit Suriyawongkul", email = "suriyawa@tcd.ie" }, +] +keywords = [ + "pythainlp", + "NLP", + "natural language processing", + "tokenization", + "text processing", + "linguistics", + "localization", + "computational linguistics", + "ThaiNLP", + "Thai NLP", + "Thai language", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Intended Audience :: Developers", + "Natural Language :: Thai", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Localization", + "Topic :: Text Processing", + "Topic :: Text Processing :: General", + "Topic :: Text Processing :: Linguistic", +] + +# Core dependencies +dependencies = ["requests>=2.31", "tzdata; sys_platform == 'win32'"] + +[project.optional-dependencies] + +abbreviation = ["khamyo>=0.2.0"] + +attacut = ["attacut>=1.0.6"] + +benchmarks = ["numpy>=1.22", "pandas>=0.24", "PyYAML>=5.4.1"] + +budoux = ["budoux>=0.7.0"] + +coreference_resolution = ["fastcoref>=2.1.5", "spacy>=3.0"] + +dependency_parsing = [ + "spacy_thai>=0.7.1", + "transformers>=4.22.1", + "ufal.chu-liu-edmonds>=1.0.2", +] + +el = ["multiel>=0.5"] + +esupar = ["esupar>=1.3.8", "numpy>=1.22", "transformers>=4.22.1"] + +generate = ["fastai<2.0"] + +icu = ["pyicu>=2.3"] + +ipa = ["epitran>=1.1"] + +ml = ["numpy>=1.22", "torch>=1.0.0"] + +mt5 = ["sentencepiece>=0.1.91", "transformers>=4.22.1"] + +nlpo3 = ["nlpo3>=1.3.1"] + +onnx = ["numpy>=1.22", "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"] + +oskut = ["oskut>=1.3"] + +sefr_cut = ["sefr_cut>=1.1"] + +spacy_thai = ["spacy_thai>=0.7.1"] + +spell = ["phunspell>=0.1.6", "symspellpy>=6.7.6"] + +ssg = ["ssg>=0.0.8"] + +textaugment = ["bpemb>=0.3.2", "gensim>=4.0.0"] + +thai_nner = ["thai_nner"] + +thai2fit = ["emoji>=0.5.1", "gensim>=4.0.0", "numpy>=1.22"] + +thai2rom = ["numpy>=1.22", "torch>=1.0.0"] + +translate = [ + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "sacremoses>=0.0.41", + "sentencepiece>=0.1.91", + "torch>=1.0.0", + "transformers>=4.22.1", + "word2word>=1.0.0", +] + +transformers_ud = ["transformers>=4.22.1", "ufal.chu-liu-edmonds>=1.0.2"] + +wangchanberta = ["sentencepiece>=0.1.91", "transformers>=4.22.1"] + +wangchanglm = ["pandas>=0.24", "sentencepiece>=0.1.91", "transformers>=4.22.1"] + +word_approximation = ["panphon>=0.20.0"] + +wordnet = ["nltk>=3.3"] + +wsd = ["sentence-transformers>=2.2.2"] + +wtp = ["transformers>=4.22.1", "wtpsplit>=1.0.1"] + +wunsen = ["wunsen>=0.0.3"] + +# Compact dependencies - safe small set of optional dependencies +compact = [ + "nlpo3>=1.3.1", + "numpy>=1.22", + "pyicu>=2.3", + "python-crfsuite>=0.9.7", + "PyYAML>=5.4.1", +] + +# Full dependencies - comprehensive set of all optional features +full = [ + "attacut>=1.0.6", + "bpemb>=0.3.2", + "budoux>=0.7.0", + "emoji>=0.5.1", + "epitran>=1.1", + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "fastai<2.0", + "fastcoref>=2.1.5", + "gensim>=4.0.0", + "khamyo>=0.2.0", + "nlpo3>=1.3.1", + "nltk>=3.3", + "numpy>=1.22", + "onnxruntime>=1.10.0", + "oskut>=1.3", + "pandas>=0.24", + "panphon>=0.20.0", + "phunspell>=0.1.6", + "pyicu>=2.3", + "sacremoses>=0.0.41", + "sefr_cut>=1.1", + "sentencepiece>=0.1.91", + "sentence-transformers>=2.2.2", + "spacy>=3.0", + "spacy_thai>=0.7.1", + "ssg>=0.0.8", + "symspellpy>=6.7.6", + "thai_nner", + "torch>=1.0.0", + "transformers>=4.22.1", + "ufal.chu-liu-edmonds>=1.0.2", + "word2word>=1.0.0", + "wtpsplit>=1.0.1", + "wunsen>=0.0.3", +] + +# Testing dependencies - pinned versions for CI/CD reproducibility +testing = [ + "attacut==1.0.6", + "bpemb>=0.3.6,<0.4", + "budoux==0.7.0", + "deepcut==0.7.0.0", + "emoji>=0.6.0,<1", + "epitran==1.26.0", + "esupar>=1.3.9,<2", + 'fairseq>=0.10.0,<0.13;python_version<"3.11"', + 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', + "fastai>=1.0.61,<2", + "fastcoref==2.1.6", + "gensim>=4.3.3,<5", + "khanaa>=0.1.1,<1", + "nlpo3>=1.3.1", + "nltk>=3.6.6,<4", + "numpy>=1.26.0,<3", + "pandas>=2.2.0,<3", + "panphon==0.22.2", + "phunspell==0.1.6", + "pyicu>=2.15.2,<3", + "python-crfsuite==0.9.12", + "PyYAML>=5.4.1,<6.0.2", + "sacremoses==0.1.1", + "sentence-transformers>=2.7.0,<3", + "sentencepiece==0.2.1", + "spacy==3.8.7,<4", + "spacy_thai==0.7.8", + "ssg==0.0.8", + "symspellpy==6.9.0", + "thai-nner==0.3", + "tltk>=1.6.8,<2", + "torch>=1.13.1,<3", + "transformers==4.57.3", + "ufal.chu-liu-edmonds==1.0.3", + "word2word>=1.0.0,<2", + "wtpsplit==1.3.0", + "wunsen==0.0.3", +] + +[project.urls] +homepage = "https://pythainlp.org/" +source = "https://github.com/PyThaiNLP/pythainlp.git" +download = "https://pypi.org/project/pythainlp/#files" +changelog = "https://github.com/PyThaiNLP/pythainlp/blob/dev/CHANGELOG.md" +releasenotes = "https://github.com/PyThaiNLP/pythainlp/releases" +documentation = "https://pythainlp.org/docs/" +issues = "https://github.com/PyThaiNLP/pythainlp/issues" +"Tutorials" = "https://pythainlp.org/tutorials/" + +[project.scripts] +thainlp = "pythainlp.__main__:main" + +[tool.setuptools] +zip-safe = false +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["tests", "tests.*"] + +[tool.setuptools.package-data] +pythainlp = ["corpus/*"] + +# Bumpversion configuration +[tool.bumpversion] +current_version = "5.2.0" +commit = true +tag = true +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\-(?P[a-z]+)(?P\\d+))?" +serialize = [ + "{major}.{minor}.{patch}-{release}{build}", + "{major}.{minor}.{patch}", +] + +[[tool.bumpversion.files]] +filename = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "pythainlp/__init__.py" +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' + +[tool.bumpversion.parts.release] +optional_value = "prod" +first_value = "dev" +values = ["dev", "beta", "prod"] + +# Coverage configuration +[tool.coverage.run] +source = ["pythainlp"] + +# Ruff configuration [tool.ruff] line-length = 79 indent-width = 4 diff --git a/pythainlp/corpus/corpus_license.md b/pythainlp/corpus/corpus_license.md index 88c8f15f6..70b72c335 100644 --- a/pythainlp/corpus/corpus_license.md +++ b/pythainlp/corpus/corpus_license.md @@ -4,12 +4,11 @@ - Language models created by PyThaiNLP project are released under [Creative Commons Attribution 4.0 International Public License](https://creativecommons.org/licenses/by/4.0/) (CC-by). - For more information about corpora that PyThaiNLP use, see [https://github.com/PyThaiNLP/pythainlp-corpus/](https://github.com/PyThaiNLP/pythainlp-corpus/). - ## Dictionaries and Word Lists The following word lists are created by the PyThaiNLP project and released under **Creative Commons Zero 1.0 Universal Public Domain Dedication License** -https://creativecommons.org/publicdomain/zero/1.0/ + | Filename | Description | | ---------------------------- | ------------------------------------------------------ | @@ -25,10 +24,10 @@ https://creativecommons.org/publicdomain/zero/1.0/ | words_th_thai2fit_201810.txt | List of Thai words (frozen for thai2fit) | The following word lists are from **Thai Male and Female Names Corpus** -https://github.com/korkeatw/thai-names-corpus/ by Korkeat Wannapat + by Korkeat Wannapat and released under their original licenses which are **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + | Filename | Description | | -------------------------- | -------------------------------- | @@ -36,37 +35,34 @@ https://creativecommons.org/licenses/by-sa/4.0/ | person_names_female_th.txt | List of female names in Thailand | | person_names_male_th.txt | List of male names in Thailand | - ## Models The following language models are created by the PyThaiNLP project and released under **Creative Commons Attribution 4.0 International Public License** -https://creativecommons.org/licenses/by/4.0/ + -| Filename | Description | -| ------------------------- | ----------------------------------------------------------------------------------------------------- | -| pos_orchid_perceptron.json | Part-of-speech tagging model, trained from ORCHID data, using perceptron | -| pos_orchid_unigram.json | Part-of-speech tagging model, trained from ORCHID data, using unigram | +| Filename | Description | +| ------------------------------- | ----------------------------------------------------------------------------------------------------- | +| pos_orchid_perceptron.json | Part-of-speech tagging model, trained from ORCHID data, using perceptron | +| pos_orchid_unigram.json | Part-of-speech tagging model, trained from ORCHID data, using unigram | | pos_ud_perceptron-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using perceptron | -| pos_ud_unigram-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram | -| sentenceseg_crfcut.model | Sentence segmentation model, trained from TED subtitles, using CRF | -| tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron | -| tdtb-unigram_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram | -| pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron | -| pos_tud_unigram.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram | - +| pos_ud_unigram-v0.2.json | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram | +| sentenceseg_crfcut.model | Sentence segmentation model, trained from TED subtitles, using CRF | +| tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron | +| tdtb-unigram_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram | +| pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron | +| pos_tud_unigram.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram | ## Thai Dictionary for ICU BreakIterator A Thai word list from ICU (International Components for Unicode) project (icubrk_th.txt) is copyrighted by Unicode, Inc. and others., released under **Unicode License Agreement - Data Files and Software (2016)** -http://www.unicode.org/copyright.html + Original data: -https://github.com/unicode-org/icu/blob/main/icu4c/source/data/brkitr/dictionaries/thaidict.txt - + ## Thai WordNet @@ -74,7 +70,7 @@ Thai WordNet (wordnet_th.db) is created by Thai Computational Linguistic Laboratory at National Institute of Information and Communications Technology (NICT), Japan, and released under the following license: -``` +```text Copyright: 2011 NICT Thai WordNet @@ -116,8 +112,7 @@ For more information about Thai WordNet, see S. Thoongsup et al., ‘Thai WordNet construction’, in Proceedings of the 7th Workshop on Asian Language Resources, Suntec, Singapore, Aug. 2009, pp. 139–144. -https://www.aclweb.org/anthology/W09-3420.pdf - + ## Thai Wikipedia Titles @@ -125,14 +120,13 @@ Thai Wikipedia titles corpus (wikipedia_titles.txt), prepared by konbraphat51, using a Thai Wikipedia dump from 21 November 2023, and released under their original license which is **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + Original data: -https://dumps.wikimedia.org/thwiki/latest/thwiki-latest-all-titles.gz + Preparation code: -https://github.com/konbraphat51/Thai_Dictionary_Cleaner/ - + ## Volubilis @@ -141,10 +135,10 @@ A corpus of Thai words registered in Volubilis dictionary using data from Volubilis 23.1 (Mar. 2023) by Francis Bastien, and released under their original license which is **Creative Commons Attribution-ShareAlike 4.0 International Public License** -https://creativecommons.org/licenses/by-sa/4.0/ + Original data: -https://belisan-volubilis.blogspot.com/ + Preparation code: -https://github.com/konbraphat51/Thai_Dictionary_Cleaner/ + diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 44904e295..000000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -PyYAML>=5.4.1 -numpy>=1.22 -pyicu>=2.3 -python-crfsuite>=0.9.7 -requests>=2.31 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c0245293d..000000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[bumpversion] -current_version = 5.2.0 -commit = True -tag = True -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}-{release}{build} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - beta - prod - -[bumpversion:part:build] - -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - -[bumpversion:file:pythainlp/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[metadata] -description_file = README.md - -[coverage:run] -source = pythainlp diff --git a/setup.py b/setup.py deleted file mode 100644 index 0b0a3c2d0..000000000 --- a/setup.py +++ /dev/null @@ -1,217 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -""" -Setup script for PyThaiNLP. - -https://github.com/PyThaiNLP/pythainlp -""" - -from setuptools import find_packages, setup - -PYYAML = "PyYAML>=5.4.1" -PANDAS = "pandas>=0.24" -NUMPY = "numpy>=1.22" - -LONG_DESC = """ -![PyThaiNLP Logo](https://avatars0.githubusercontent.com/u/32934255?s=200&v=4) - -PyThaiNLP is a Python library for Thai natural language processing. -The library provides functions like word tokenization, part-of-speech tagging, -transliteration, soundex generation, spell checking, and -date and time parsing/formatting. - -Website: [pythainlp.github.io](https://pythainlp.org/) - -# Install - -For stable version: - -```sh -pip install pythainlp -``` - -For development version: - -```sh -pip install --upgrade --pre pythainlp -``` - -Some functionalities, like named-entity recognition, require extra packages. -See https://github.com/PyThaiNLP/pythainlp for installation options. -""" - -requirements = [ - "backports.zoneinfo; python_version<'3.9'", - "requests>=2.31", - "tzdata; sys_platform == 'win32'", -] - -extras = { - "abbreviation": ["khamyo>=0.2.0"], - "attacut": ["attacut>=1.0.6"], - "benchmarks": [PYYAML, NUMPY, PANDAS], - "coreference_resolution": [ - "fastcoref>=2.1.5", - "spacy>=3.0", - ], - "dependency_parsing": [ - "spacy_thai>=0.7.1", - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - ], - "el": ["multiel>=0.5"], - "esupar": [ - "esupar>=1.3.8", - "numpy", - "transformers>=4.22.1", - ], - "generate": ["fastai<2.0"], - "icu": ["pyicu>=2.3"], - "ipa": ["epitran>=1.1"], - "ml": [NUMPY, "torch>=1.0.0"], - "mt5": ["sentencepiece>=0.1.91", "transformers>=4.6.0"], - "nlpo3": ["nlpo3>=1.3.1"], - "onnx": [NUMPY, "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"], - "oskut": ["oskut>=1.3"], - "sefr_cut": ["sefr_cut>=1.1"], - "spacy_thai": ["spacy_thai>=0.7.1"], - "spell": ["phunspell>=0.1.6", "symspellpy>=6.7.6"], - "ssg": ["ssg>=0.0.8"], - "textaugment": ["bpemb", "gensim>=4.0.0"], - "thai_nner": ["thai_nner"], - "thai2fit": ["emoji>=0.5.1", "gensim>=4.0.0", NUMPY], - "thai2rom": [NUMPY, "torch>=1.0.0"], - "budoux": ["budoux>=0.7.0"], - "translate": [ - 'fairseq>=0.10.0,<0.13;python_version<"3.11"', - 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', - "sacremoses>=0.0.41", - "sentencepiece>=0.1.91", - "torch>=1.0.0", - "transformers>=4.6.0", - "word2word>=1.0.0" - ], - "transformers_ud": [ - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - ], - "wangchanberta": ["sentencepiece>=0.1.91", "transformers>=4.6.0"], - "wangchanglm": [ - PANDAS, - "sentencepiece>=0.1.91", - "transformers>=4.6.0", - ], - "word_approximation": ["panphon>=0.20.0"], - "wordnet": ["nltk>=3.3"], - "wsd": ["sentence-transformers>=2.2.2"], - "wtp": ["transformers>=4.6.0", "wtpsplit>=1.0.1"], - "wunsen": ["wunsen>=0.0.1"], - # Compact dependencies, this one matches requirements.txt - "compact": [ - PYYAML, - "nlpo3>=1.3.1", - NUMPY, - "pyicu>=2.3", - "python-crfsuite>=0.9.7", - ], - # Full dependencies - "full": [ - PYYAML, - "attacut>=1.0.4", - "bpemb>=0.3.2", - "emoji>=0.5.1", - "epitran>=1.1", - 'fairseq>=0.10.0,<0.13;python_version<"3.11"', - 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', - "fastai<2.0", - "fastcoref>=2.1.5", - "gensim>=4.0.0", - "khamyo>=0.2.0", - "nlpo3>=1.3.1", - "nltk>=3.3", - NUMPY, - "onnxruntime>=1.10.0", - "oskut>=1.3", - PANDAS, - "panphon>=0.20.0", - "phunspell>=0.1.6", - "pyicu>=2.3", - "sacremoses>=0.0.41", - "sefr_cut>=1.1", - "sentencepiece>=0.1.91", - "sentence-transformers>=2.2.2", - "spacy>=3.0", - "spacy_thai>=0.7.1", - "ssg>=0.0.8", - "symspellpy>=6.7.6", - "thai_nner", - "torch>=1.0.0", - "transformers>=4.22.1", - "ufal.chu-liu-edmonds>=1.0.2", - "wtpsplit>=1.0.1", - "wunsen>=0.0.3", - "word2word>=1.0.0", - "budoux>=0.7.0", - ], -} - -setup( - name="pythainlp", - version="5.2.0", - description="Thai Natural Language Processing library", - long_description=LONG_DESC, - long_description_content_type="text/markdown", - author="PyThaiNLP", - author_email="wannaphong@pythainlp.org", - url="https://github.com/PyThaiNLP/pythainlp", - packages=find_packages(exclude=["tests", "tests.*"]), - test_suite="tests", - python_requires=">=3.7", - package_data={ - "pythainlp": [ - "corpus/*", - ], - }, - include_package_data=True, - install_requires=requirements, - extras_require=extras, - license="Apache-2.0", - zip_safe=False, - keywords=[ - "pythainlp", - "NLP", - "natural language processing", - "text analytics", - "text processing", - "localization", - "computational linguistics", - "ThaiNLP", - "Thai NLP", - "Thai language", - ], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: Thai", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Text Processing", - "Topic :: Text Processing :: General", - "Topic :: Text Processing :: Linguistic", - ], - entry_points={ - "console_scripts": [ - "thainlp = pythainlp.__main__:main", - ], - }, - project_urls={ - "Documentation": "https://pythainlp.org/docs/5.2/", - "Tutorials": "https://pythainlp.org/tutorials/", - "Source Code": "https://github.com/PyThaiNLP/pythainlp", - "Bug Tracker": "https://github.com/PyThaiNLP/pythainlp/issues", - }, -) - -# TODO: Check extras and decide whether or not additional data, like model files, should be downloaded diff --git a/tox.ini b/tox.ini index ae24e75d4..59ee3c9dd 100644 --- a/tox.ini +++ b/tox.ini @@ -8,10 +8,10 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/pythainlp changedir = tests commands = discover deps = discover -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following lines: +; If you want to install the package with dependencies for testing: ; deps = -; -r{toxinidir}/requirements.txt +; .[compact] +; discover [testenv:flake8] basepython = python