From 66c2317f661a07ff6548499b2b0c344b8f5ad87f Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 12:32:45 +0000 Subject: [PATCH 1/5] Add whitespace check and fix few imports --- .github/workflows/lint.yml | 2 +- pythainlp/__init__.py | 15 +++++++++++++++ pythainlp/spell/words_spelling_correction.py | 3 +-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 22c99450a..2daea5b79 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,4 +26,4 @@ jobs: uses: astral-sh/ruff-action@v3 with: src: "./pythainlp" - args: check --verbose --line-length 79 --select C901 + args: check --verbose --line-length 79 --select C901,W291 diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index bd43ddd25..a41f1c374 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -50,6 +50,21 @@ โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ ๑๒ กรกฎาคม ๒๕๕๘""" +__all__ = [ + "collate", + "correct", + "pos_tag", + "romanize", + "spell", + "sent_tokenize", + "subword_tokenize", + "soundex", + "thai_strftime", + "transliterate", + "Tokenizer", + "word_tokenize", +] + from pythainlp.soundex import soundex from pythainlp.spell import correct, spell from pythainlp.tag import pos_tag diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index fa6a27ed6..7daa8c26e 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -41,8 +41,7 @@ def __init__( maxn (int): Maximum character length for subwords. """ try: - import numpy as np # reduce load - import onnxruntime + import numpy as np self.np = np except ModuleNotFoundError: From 1242c1a31f560d4e1fe74520fe7b83f1bfdd1466 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 12:37:45 +0000 Subject: [PATCH 2/5] Remove trailing whitespace --- examples/khavee.py | 2 +- pythainlp/tokenize/core.py | 4 ++-- pythainlp/util/abbreviation.py | 2 +- pythainlp/util/remove_trailing_repeat_consonants.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/khavee.py b/examples/khavee.py index 79e1a72be..82b19283a 100644 --- a/examples/khavee.py +++ b/examples/khavee.py @@ -64,7 +64,7 @@ เรื่องวิศวะเก่งกาจประหลาดใจ เรื่องฟิสิกส์ไร้ผู้ใดมาต่อไป นริศราอีฟเก่งกว่าใครเพื่อน คอยช่วยเตือนเรื่องงานคอยสั่งสอน อ่านตำราหาความรู้ไม่ละทอน เป็นคนดีศรีนครของจิตรลดา -ภัสนันท์นาคลออหรือมีมี่ เรื่องเกมเอ่อเก่งกาจไม่กังขา +ภัสนันท์นาคลออหรือมีมี่ เรื่องเกมเอ่อเก่งกาจไม่กังขา เกมอะไรก็เล่นได้ไม่ลดวา สุดฉลาดมากปัญญามาครบครัน""", k_type=8, ) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 234be457e..cb7d7c211 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -129,7 +129,7 @@ def word_tokenize( * *icu* - wrapper for a word tokenizer in `PyICU `_., from ICU (International Components for Unicode), - dictionary-based + dictionary-based * *longest* - dictionary-based, longest matching * *mm* - "multi-cut", dictionary-based, maximum matching * *nercut* - dictionary-based, maximal matching, @@ -565,7 +565,7 @@ def paragraph_tokenize( paragraph_tokenize(sent) # output: [ - # ['(1) '], + # ['(1) '], # [ # 'บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต ', # 'มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด ', diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index fbb844314..380344472 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -41,7 +41,7 @@ def abbreviation_to_full_text( raise ImportError( """ This function needs to use khamyo. - You can install by pip install khamyo or + You can install by pip install khamyo or pip install pythainlp[abbreviation]. """ ) diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index dfff7e87c..3fb4f004c 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -42,7 +42,7 @@ def remove_trailing_repeat_consonants( :param str text: input text :param Trie dictionary: Trie dictionary to check the last word. If None, pythainlp.corpus.thai_words() will be used - :param bool has_dictionary_updated: If the dictionary is updated + :param bool has_dictionary_updated: If the dictionary is updated or the first time using in the kernel, set this true. If not, set this false to save time. :return: text without repeating Thai consonants diff --git a/setup.py b/setup.py index 0b0a3c2d0..db5486b88 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ "oskut": ["oskut>=1.3"], "sefr_cut": ["sefr_cut>=1.1"], "spacy_thai": ["spacy_thai>=0.7.1"], - "spell": ["phunspell>=0.1.6", "symspellpy>=6.7.6"], + "spell": ["phunspell>=0.1.6", "symspellpy>=6.7.6"], "ssg": ["ssg>=0.0.8"], "textaugment": ["bpemb", "gensim>=4.0.0"], "thai_nner": ["thai_nner"], From aebd5f7653818377d6fb7db7d1cd94f9f5bbb788 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 13:02:28 +0000 Subject: [PATCH 3/5] Fix general formatting issues --- .github/workflows/lint.yml | 2 +- docs/conf.py | 8 +- examples/khavee.py | 4 +- notebooks/convert_thai2rom_to_onnx.ipynb | 2 +- notebooks/create_words.ipynb | 4 +- notebooks/test_chat.ipynb | 5 +- notebooks/test_el.ipynb | 1 + notebooks/test_wangchanglm.ipynb | 5 +- notebooks/test_wsd.ipynb | 2 +- pyproject.toml | 2 +- pythainlp/ancient/__init__.py | 3 +- pythainlp/ancient/aksonhan.py | 3 +- pythainlp/ancient/currency.py | 3 +- pythainlp/augment/__init__.py | 3 +- pythainlp/augment/lm/__init__.py | 3 +- pythainlp/augment/lm/fasttext.py | 15 ++-- pythainlp/augment/lm/phayathaibert.py | 3 +- pythainlp/augment/lm/wangchanberta.py | 3 +- pythainlp/augment/word2vec/__init__.py | 3 +- pythainlp/augment/word2vec/bpemb_wv.py | 12 +-- pythainlp/augment/word2vec/core.py | 9 +- pythainlp/augment/word2vec/ltw2v.py | 12 +-- pythainlp/augment/word2vec/thai2fit.py | 12 +-- pythainlp/augment/wordnet.py | 15 ++-- pythainlp/benchmarks/__init__.py | 3 +- pythainlp/benchmarks/word_tokenization.py | 24 ++--- pythainlp/chat/__init__.py | 3 +- pythainlp/chat/core.py | 12 +-- pythainlp/classify/__init__.py | 3 +- pythainlp/classify/param_free.py | 12 +-- pythainlp/cli/data.py | 3 +- pythainlp/cli/soundex.py | 3 +- pythainlp/cli/tag.py | 3 +- pythainlp/cli/tokenize.py | 3 +- pythainlp/coref/__init__.py | 3 +- pythainlp/coref/core.py | 3 +- pythainlp/corpus/__init__.py | 40 ++++----- pythainlp/corpus/common.py | 45 ++++------ pythainlp/corpus/core.py | 46 ++++------ pythainlp/corpus/icu.py | 7 +- pythainlp/corpus/oscar.py | 9 +- pythainlp/corpus/th_en_translit.py | 6 +- pythainlp/corpus/tnc.py | 15 ++-- pythainlp/corpus/ttc.py | 9 +- pythainlp/corpus/util.py | 17 ++-- pythainlp/corpus/volubilis.py | 8 +- pythainlp/corpus/wikipedia.py | 6 +- pythainlp/corpus/wordnet.py | 42 +++------ pythainlp/el/__init__.py | 3 +- pythainlp/el/core.py | 6 +- pythainlp/generate/__init__.py | 3 +- pythainlp/generate/core.py | 27 ++---- pythainlp/generate/thai2fit.py | 6 +- pythainlp/generate/wangchanglm.py | 9 +- pythainlp/khavee/core.py | 33 +++---- pythainlp/lm/text_util.py | 7 +- pythainlp/morpheme/__init__.py | 3 +- pythainlp/morpheme/thaiwordcheck.py | 8 +- pythainlp/morpheme/word_formation.py | 3 +- pythainlp/parse/__init__.py | 3 +- pythainlp/parse/core.py | 3 +- pythainlp/parse/esupar_engine.py | 3 +- pythainlp/parse/spacy_thai_engine.py | 3 +- pythainlp/parse/transformers_ud.py | 3 +- pythainlp/parse/ud_goeswith.py | 3 +- pythainlp/phayathaibert/__init__.py | 3 +- pythainlp/phayathaibert/core.py | 37 +++----- pythainlp/soundex/__init__.py | 3 +- pythainlp/soundex/core.py | 6 +- pythainlp/soundex/lk82.py | 6 +- pythainlp/soundex/metasound.py | 7 +- pythainlp/soundex/prayut_and_somchaip.py | 12 +-- pythainlp/soundex/sound.py | 12 +-- pythainlp/soundex/udom83.py | 7 +- pythainlp/spell/__init__.py | 3 +- pythainlp/spell/core.py | 15 ++-- pythainlp/spell/phunspell.py | 3 +- pythainlp/spell/pn.py | 36 +++----- pythainlp/spell/symspellpy.py | 3 +- pythainlp/spell/tltk.py | 3 +- .../spell/wanchanberta_thai_grammarly.py | 3 +- pythainlp/spell/words_spelling_correction.py | 14 ++- pythainlp/summarize/__init__.py | 3 +- pythainlp/summarize/core.py | 9 +- pythainlp/summarize/freq.py | 3 +- pythainlp/summarize/keybert.py | 9 +- pythainlp/summarize/mt5.py | 6 +- pythainlp/tag/__init__.py | 3 +- pythainlp/tag/_tag_perceptron.py | 27 ++---- pythainlp/tag/blackboard.py | 6 +- pythainlp/tag/chunk.py | 3 +- pythainlp/tag/crfchunk.py | 3 +- pythainlp/tag/locations.py | 6 +- pythainlp/tag/named_entity.py | 15 ++-- pythainlp/tag/orchid.py | 9 +- pythainlp/tag/perceptron.py | 6 +- pythainlp/tag/pos_tag.py | 26 +++--- pythainlp/tag/thainer.py | 12 +-- pythainlp/tag/tltk.py | 3 +- pythainlp/tag/unigram.py | 6 +- pythainlp/tokenize/__init__.py | 3 +- pythainlp/tokenize/_utils.py | 12 +-- pythainlp/tokenize/attacut.py | 6 +- pythainlp/tokenize/budoux.py | 3 +- pythainlp/tokenize/core.py | 37 +++----- pythainlp/tokenize/crfcut.py | 9 +- pythainlp/tokenize/deepcut.py | 3 +- pythainlp/tokenize/etcc.py | 7 +- pythainlp/tokenize/han_solo.py | 3 +- pythainlp/tokenize/longest.py | 6 +- pythainlp/tokenize/multi_cut.py | 3 +- pythainlp/tokenize/nercut.py | 6 +- pythainlp/tokenize/newmm.py | 3 +- pythainlp/tokenize/oskut.py | 3 +- pythainlp/tokenize/pyicu.py | 6 +- pythainlp/tokenize/sefr_cut.py | 3 +- pythainlp/tokenize/ssg.py | 3 +- pythainlp/tokenize/tcc.py | 13 +-- pythainlp/tokenize/tcc_p.py | 13 +-- pythainlp/tokenize/thaisumcut.py | 3 +- pythainlp/tokenize/wtsplit.py | 3 +- pythainlp/tools/core.py | 3 +- pythainlp/tools/misspell.py | 3 +- pythainlp/tools/path.py | 12 +-- pythainlp/translate/__init__.py | 3 +- pythainlp/translate/core.py | 12 +-- pythainlp/translate/en_th.py | 18 ++-- pythainlp/translate/small100.py | 6 +- pythainlp/translate/th_fr.py | 9 +- pythainlp/translate/tokenization_small100.py | 87 +++++++++---------- pythainlp/translate/word2word_translate.py | 3 +- pythainlp/translate/zh_th.py | 15 ++-- pythainlp/transliterate/__init__.py | 3 +- pythainlp/transliterate/core.py | 18 ++-- pythainlp/transliterate/ipa.py | 3 +- pythainlp/transliterate/iso_11940.py | 6 +- pythainlp/transliterate/lookup.py | 12 +-- pythainlp/transliterate/pyicu.py | 6 +- pythainlp/transliterate/royin.py | 9 +- pythainlp/transliterate/spoonerism.py | 3 +- pythainlp/transliterate/thai2rom.py | 13 +-- pythainlp/transliterate/thai2rom_onnx.py | 12 +-- pythainlp/transliterate/thaig2p.py | 13 +-- pythainlp/transliterate/thaig2p_v2.py | 6 +- pythainlp/transliterate/tltk.py | 3 +- pythainlp/transliterate/umt5_thaig2p.py | 6 +- pythainlp/transliterate/w2p.py | 6 +- pythainlp/transliterate/wunsen.py | 9 +- pythainlp/ulmfit/__init__.py | 3 +- pythainlp/ulmfit/core.py | 13 +-- pythainlp/ulmfit/preprocess.py | 35 +++----- pythainlp/ulmfit/tokenizer.py | 9 +- pythainlp/util/__init__.py | 35 ++++---- pythainlp/util/abbreviation.py | 6 +- pythainlp/util/collate.py | 6 +- pythainlp/util/date.py | 18 ++-- pythainlp/util/digitconv.py | 18 ++-- pythainlp/util/emojiconv.py | 7 +- pythainlp/util/encoding.py | 6 +- pythainlp/util/keyboard.py | 12 +-- pythainlp/util/keywords.py | 6 +- pythainlp/util/lcs.py | 3 +- pythainlp/util/morse.py | 6 +- pythainlp/util/normalize.py | 30 +++---- pythainlp/util/numtoword.py | 10 +-- pythainlp/util/phoneme.py | 14 ++- pythainlp/util/pronounce.py | 12 +-- .../util/remove_trailing_repeat_consonants.py | 21 ++--- pythainlp/util/spell_words.py | 6 +- pythainlp/util/strftime.py | 12 +-- pythainlp/util/syllable.py | 15 ++-- pythainlp/util/thai.py | 14 ++- pythainlp/util/thai_lunar_date.py | 12 +-- pythainlp/util/time.py | 9 +- pythainlp/util/trie.py | 15 ++-- pythainlp/util/wordtonum.py | 12 +-- pythainlp/wangchanberta/core.py | 15 ++-- pythainlp/word_vector/__init__.py | 3 +- pythainlp/word_vector/core.py | 24 ++--- pythainlp/wsd/__init__.py | 3 +- pythainlp/wsd/core.py | 11 ++- tests/__init__.py | 3 +- tests/compact/__init__.py | 3 +- tests/compact/test_cli.py | 5 +- tests/compact/testc_util.py | 3 +- tests/core/__init__.py | 3 +- tests/core/test_corpus.py | 2 - tests/core/test_tokenize.py | 5 +- tests/core/test_util.py | 7 +- tests/extra/__init__.py | 3 +- tests/extra/testx_cli.py | 2 +- tests/extra/testx_coref.py | 2 - tests/extra/testx_spell.py | 2 +- tests/extra/testx_tokenize.py | 2 +- tests/extra/testx_util.py | 3 +- 195 files changed, 654 insertions(+), 1172 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2daea5b79..92944d046 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,4 +26,4 @@ jobs: uses: astral-sh/ruff-action@v3 with: src: "./pythainlp" - args: check --verbose --line-length 79 --select C901,W291 + args: check --fix --verbose --line-length 79 --select I,W,C901,W291,W293 diff --git a/docs/conf.py b/docs/conf.py index cd5bf5be1..250bf5b86 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,8 +9,8 @@ import os import sys import traceback -from datetime import datetime -from datetime import date +from datetime import date, datetime + import pythainlp # -- Path setup -------------------------------------------------------------- @@ -34,7 +34,7 @@ # -- Get version information and date from Git ---------------------------- try: - from subprocess import check_output, STDOUT + from subprocess import STDOUT, check_output current_branch = ( os.environ["CURRENT_BRANCH"] @@ -69,7 +69,7 @@ # .decode() # .strip() # ) -except Exception as e: +except Exception: traceback.print_exc() release = pythainlp.__version__ # today = "" diff --git a/examples/khavee.py b/examples/khavee.py index 82b19283a..1a1c0d766 100644 --- a/examples/khavee.py +++ b/examples/khavee.py @@ -2,13 +2,11 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Example of using KhaveeVerifier from pythainlp.khavee +"""Example of using KhaveeVerifier from pythainlp.khavee """ from pythainlp.khavee import KhaveeVerifier - kv = KhaveeVerifier() # การเช็คสระ diff --git a/notebooks/convert_thai2rom_to_onnx.ipynb b/notebooks/convert_thai2rom_to_onnx.ipynb index a6383c81a..7233b7799 100644 --- a/notebooks/convert_thai2rom_to_onnx.ipynb +++ b/notebooks/convert_thai2rom_to_onnx.ipynb @@ -75,6 +75,7 @@ "outputs": [], "source": [ "import torch\n", + "\n", "from pythainlp.corpus import get_corpus_path\n", "from pythainlp.transliterate.thai2rom import _MODEL_NAME\n", "\n", @@ -115,7 +116,6 @@ "outputs": [], "source": [ "import torch\n", - "import numpy as np\n", "\n", "input_tensor = torch.Tensor([[30, 19, 8, 30, 38, 37, 10, 3]]).long()\n", "\n", diff --git a/notebooks/create_words.ipynb b/notebooks/create_words.ipynb index d8d3ced83..14e848c65 100644 --- a/notebooks/create_words.ipynb +++ b/notebooks/create_words.ipynb @@ -6,8 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from pythainlp.transliterate import pronunciate\n", - "from pythainlp import thai_consonants" + "from pythainlp import thai_consonants\n", + "from pythainlp.transliterate import pronunciate" ] }, { diff --git a/notebooks/test_chat.ipynb b/notebooks/test_chat.ipynb index d3c64f3c1..d0d7a7d6b 100644 --- a/notebooks/test_chat.ipynb +++ b/notebooks/test_chat.ipynb @@ -9,8 +9,9 @@ }, "outputs": [], "source": [ - "from pythainlp.chat.core import ChatBotModel\n", - "import torch" + "import torch\n", + "\n", + "from pythainlp.chat.core import ChatBotModel" ] }, { diff --git a/notebooks/test_el.ipynb b/notebooks/test_el.ipynb index da6060583..3ca1cf8af 100644 --- a/notebooks/test_el.ipynb +++ b/notebooks/test_el.ipynb @@ -8,6 +8,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"" ] }, diff --git a/notebooks/test_wangchanglm.ipynb b/notebooks/test_wangchanglm.ipynb index 2235ad0a0..bb7f886e1 100644 --- a/notebooks/test_wangchanglm.ipynb +++ b/notebooks/test_wangchanglm.ipynb @@ -9,8 +9,9 @@ }, "outputs": [], "source": [ - "from pythainlp.generate.wangchanglm import WangChanGLM\n", - "import torch" + "import torch\n", + "\n", + "from pythainlp.generate.wangchanglm import WangChanGLM" ] }, { diff --git a/notebooks/test_wsd.ipynb b/notebooks/test_wsd.ipynb index 07ffbf589..5b283c929 100644 --- a/notebooks/test_wsd.ipynb +++ b/notebooks/test_wsd.ipynb @@ -80,7 +80,7 @@ }, "outputs": [], "source": [ - "from pythainlp.corpus import get_corpus_path, thai_wsd_dict" + "from pythainlp.corpus import thai_wsd_dict" ] }, { diff --git a/pyproject.toml b/pyproject.toml index 00f70a467..6a6f6f6da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -301,4 +301,4 @@ docstring-code-format = true [tool.ruff.lint.mccabe] # Flag errors (`C901`) whenever the complexity level exceeds 5. Default is 10. # We should aim to gradually reduce this to 10. -max-complexity = 40 +max-complexity = 38 diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py index f8d7b41ae..23a0f50ab 100644 --- a/pythainlp/ancient/__init__.py +++ b/pythainlp/ancient/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Ancient versions of the Thai language +"""Ancient versions of the Thai language """ __all__ = ["aksonhan_to_current", "convert_currency"] diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 15e41e5f3..e9fef0cec 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -23,8 +23,7 @@ def aksonhan_to_current(word: str) -> str: - """ - Convert AksonHan words to current Thai words + """Convert AksonHan words to current Thai words AksonHan (อักษรหัน) writes down two consonants for the \ spelling of the /a/ vowels. (สระ อะ). diff --git a/pythainlp/ancient/currency.py b/pythainlp/ancient/currency.py index 0725d780d..24aa41ec9 100644 --- a/pythainlp/ancient/currency.py +++ b/pythainlp/ancient/currency.py @@ -5,8 +5,7 @@ def convert_currency(value: float, from_unit: str) -> dict: - """ - Convert ancient Thai currency to other units + """Convert ancient Thai currency to other units * เบี้ย (Bia) * อัฐ (At) diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py index 8935333b6..79c7de2b3 100644 --- a/pythainlp/augment/__init__.py +++ b/pythainlp/augment/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai text augment +"""Thai text augment """ __all__ = ["WordNetAug"] diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py index f4a69dee5..8c4e0cea2 100644 --- a/pythainlp/augment/lm/__init__.py +++ b/pythainlp/augment/lm/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Language Models +"""Language Models """ __all__ = [ diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index 6bf31b237..3621a7896 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -12,15 +12,13 @@ class FastTextAug: - """ - Text Augment from fastText + """Text Augment from fastText :param str model_path: path of model file """ def __init__(self, model_path: str): - """ - :param str model_path: path of model file + """:param str model_path: path of model file """ if model_path.endswith(".bin"): self.model = FastText_gensim.load_facebook_vectors(model_path) @@ -31,8 +29,7 @@ def __init__(self, model_path: str): self.dict_wv = list(self.model.key_to_index.keys()) def tokenize(self, text: str) -> list[str]: - """ - Thai text tokenization for fastText + """Thai text tokenization for fastText :param str text: Thai text @@ -42,8 +39,7 @@ def tokenize(self, text: str) -> list[str]: return word_tokenize(text, engine="icu") def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: - """ - :param str sent: text of sentence + """:param str sent: text of sentence :param float p: probability :rtype: List[List[str]] """ @@ -62,8 +58,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 ) -> list[tuple[str]]: - """ - Text Augment from fastText + """Text Augment from fastText You may want to download the Thai model from https://fasttext.cc/docs/en/crawl-vectors.html. diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index 73bdddf5a..a8cf8b4a0 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -57,8 +57,7 @@ def generate( def augment( self, text: str, num_augs: int = 3, sample: bool = False ) -> list[str]: - """ - Text augmentation from PhayaThaiBERT + """Text augmentation from PhayaThaiBERT :param str text: Thai text :param int num_augs: an amount of augmentation text needed as an output diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index 64702d4fe..b6173bbf4 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -51,8 +51,7 @@ def generate(self, sentence: str, num_replace_tokens: int = 3): return self.sent2 def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]: - """ - Text augmentation from WangchanBERTa + """Text augmentation from WangchanBERTa :param str sentence: Thai sentence :param int num_replace_tokens: number replace tokens diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py index 8a51b2593..ff04e2cfc 100644 --- a/pythainlp/augment/word2vec/__init__.py +++ b/pythainlp/augment/word2vec/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Word2Vec +"""Word2Vec """ __all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"] diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index 8d0a1a54c..0f39f3597 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -7,8 +7,7 @@ class BPEmbAug: - """ - Thai Text Augment using word2vec from BPEmb + """Thai Text Augment using word2vec from BPEmb BPEmb: `github.com/bheinzerling/bpemb `_ @@ -22,15 +21,13 @@ def __init__(self, lang: str = "th", vs: int = 100000, dim: int = 300): self.load_w2v() def tokenizer(self, text: str) -> list[str]: - """ - :param str text: Thai text + """:param str text: Thai text :rtype: List[str] """ return self.bpemb_temp.encode(text) def load_w2v(self): - """ - Load BPEmb model + """Load BPEmb model """ self.aug = Word2VecAug( self.model, tokenize=self.tokenizer, type="model" @@ -39,8 +36,7 @@ def load_w2v(self): def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 ) -> list[tuple[str]]: - """ - Text Augment using word2vec from BPEmb + """Text Augment using word2vec from BPEmb :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py index 72559ae43..8a6033f45 100644 --- a/pythainlp/augment/word2vec/core.py +++ b/pythainlp/augment/word2vec/core.py @@ -10,8 +10,7 @@ class Word2VecAug: def __init__( self, model: str, tokenize: object, type: str = "file" ) -> None: - """ - :param str model: path of model + """:param str model: path of model :param object tokenize: tokenize function :param str type: model type (file, binary) """ @@ -29,8 +28,7 @@ def __init__( self.dict_wv = list(self.model.key_to_index.keys()) def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: - """ - :param str sent: text of sentence + """:param str sent: text of sentence :param float p: probability :rtype: List[List[str]] """ @@ -49,8 +47,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 ) -> list[tuple[str]]: - """ - :param str sentence: text of sentence + """:param str sentence: text of sentence :param int n_sent: maximum number of synonymous sentences :param int p: probability diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index fb73b76e6..679a44600 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -9,8 +9,7 @@ class LTW2VAug: - """ - Text Augment using word2vec from LTW2V + """Text Augment using word2vec from LTW2V LTW2V: `github.com/PyThaiNLP/large-thaiword2vec `_ @@ -21,23 +20,20 @@ def __init__(self): self.load_w2v() def tokenizer(self, text: str) -> list[str]: - """ - :param str text: Thai text + """:param str text: Thai text :rtype: List[str] """ return word_tokenize(text, engine="newmm") def load_w2v(self): # insert substitute - """ - Load LTW2V's word2vec model + """Load LTW2V's word2vec model """ self.aug = Word2VecAug(self.ltw2v_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 ) -> list[tuple[str]]: - """ - Text Augment using word2vec from Thai2Fit + """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 5d0326852..6f0bd6af2 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -9,8 +9,7 @@ class Thai2fitAug: - """ - Text Augment using word2vec from Thai2Fit + """Text Augment using word2vec from Thai2Fit Thai2Fit: `github.com/cstorm125/thai2fit `_ @@ -21,24 +20,21 @@ def __init__(self): self.load_w2v() def tokenizer(self, text: str) -> list[str]: - """ - :param str text: Thai text + """:param str text: Thai text :rtype: List[str] """ tok = thai2fit_tokenizer() return tok.word_tokenize(text) def load_w2v(self): - """ - Load Thai2Fit's word2vec model + """Load Thai2Fit's word2vec model """ self.aug = Word2VecAug(self.thai2fit_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 ) -> list[tuple[str]]: - """ - Text Augment using word2vec from Thai2Fit + """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence :param int n_sent: number of sentence diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index 6c8966ea7..1a45b80a3 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l +"""Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l """ from __future__ import annotations @@ -103,8 +102,7 @@ def postype2wordnet(pos: str, corpus: str): - """ - Convert part-of-speech type to wordnet type + """Convert part-of-speech type to wordnet type :param str pos: POS type :param str corpus: part-of-speech corpus @@ -118,8 +116,7 @@ def postype2wordnet(pos: str, corpus: str): class WordNetAug: - """ - Text Augment using wordnet + """Text Augment using wordnet """ def __init__(self): @@ -128,8 +125,7 @@ def __init__(self): def find_synonyms( self, word: str, pos: str | None = None, postag_corpus: str = "orchid" ) -> list[str]: - """ - Find synonyms using wordnet + """Find synonyms using wordnet :param str word: word :param str | None pos: part-of-speech type. Default is None. @@ -164,8 +160,7 @@ def augment( postag: bool = True, postag_corpus: str = "orchid", ) -> list[list[str]]: - """ - Text Augment using wordnet + """Text Augment using wordnet :param str sentence: Thai sentence :param object tokenize: function for tokenizing words diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index cc25d2705..c63536681 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Performance benchmarking. +"""Performance benchmarking. """ __all__ = ["benchmark"] diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index ce9991cfc..94e7598f3 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -27,8 +27,7 @@ def _f1(precision: float, recall: float) -> float: - """ - Compute f1. + """Compute f1. :param float precision :param float recall @@ -42,8 +41,7 @@ def _f1(precision: float, recall: float) -> float: def _flatten_result(my_dict: dict, sep: str = ":") -> dict: - """ - Flatten two-dimension dictionary. + """Flatten two-dimension dictionary. Use keys in the first dimension as a prefix for keys in the second dimension. For example, @@ -68,8 +66,7 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: - """ - Performance benchmarking for samples. + """Performance benchmarking for samples. Please see :meth:`pythainlp.benchmarks.word_tokenization.compute_stats` for the computed metrics. @@ -112,8 +109,7 @@ def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: def preprocessing(txt: str, remove_space: bool = True) -> str: - """ - Clean up text before performing evaluation. + """Clean up text before performing evaluation. :param str text: text to be preprocessed :param bool remove_space: whether to remove white space @@ -136,8 +132,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: def compute_stats(ref_sample: str, raw_sample: str) -> dict: - """ - Compute statistics for tokenization quality + """Compute statistics for tokenization quality These statistics include: @@ -204,8 +199,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: def _binary_representation(txt: str, verbose: bool = False): - """ - Transform text into {0, 1} sequence. + """Transform text into {0, 1} sequence. where (1) indicates that the corresponding character is the beginning of a word. For example, ผม|ไม่|ชอบ|กิน|ผัก -> 10100... @@ -237,8 +231,7 @@ def _binary_representation(txt: str, verbose: bool = False): def _find_word_boundaries(bin_reps) -> list: - """ - Find the starting and ending location of each word. + """Find the starting and ending location of each word. :param str bin_reps: binary representation of a text @@ -256,8 +249,7 @@ def _find_words_correctly_tokenised( ref_boundaries: list[tuple[int, int]], predicted_boundaries: list[tuple[int, int]], ) -> tuple[int]: - """ - Find whether each word is correctly tokenized. + """Find whether each word is correctly tokenized. :param list[tuple(int, int)] ref_boundaries: word boundaries of reference tokenization :param list[tuple(int, int)] predicted_boundaries: word boundareies of predicted tokenization diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py index 2459ff1cd..8becbc93d 100644 --- a/pythainlp/chat/__init__.py +++ b/pythainlp/chat/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.chat +"""pythainlp.chat """ __all__ = ["ChatBotModel"] diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 3659b9b53..18e2735a4 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -8,14 +8,12 @@ class ChatBotModel: def __init__(self): - """ - Chat using AI generation + """Chat using AI generation """ self.history = [] def reset_chat(self): - """ - Reset chat by cleaning history + """Reset chat by cleaning history """ self.history = [] @@ -29,8 +27,7 @@ def load_model( offload_folder: str = "./", low_cpu_mem_usage: bool = True, ): - """ - Load model + """Load model :param str model_name: Model name (Now, we support wangchanglm only) :param bool return_dict: return_dict @@ -57,8 +54,7 @@ def load_model( raise NotImplementedError(f"We doesn't support {model_name}.") def chat(self, text: str) -> str: - """ - Chatbot + """Chatbot :param str text: text for asking chatbot with. :return: answer from chatbot. diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py index ae305872b..16c3f3151 100644 --- a/pythainlp/classify/__init__.py +++ b/pythainlp/classify/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.classify +"""pythainlp.classify """ __all__ = ["GzipModel"] diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index 097607314..7b21bea3b 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -10,8 +10,7 @@ class GzipModel: - """ - This class is a re-implementation of + """This class is a re-implementation of “Low-Resource” Text Classification: A Parameter-Free Classification Method with Compressors (Jiang et al., Findings 2023) @@ -41,8 +40,7 @@ def train(self): return Cx2_list def predict(self, x1: str, k: int = 1) -> str: - """ - :param str x1: the text that we want to predict label for. + """:param str x1: the text that we want to predict label for. :param str k: k :return: label :rtype: str @@ -86,8 +84,7 @@ def predict(self, x1: str, k: int = 1) -> str: return predict_class def save(self, path: str): - """ - :param str path: path to save model + """:param str path: path to save model """ with open(path, "w", encoding="utf-8") as f: json.dump( @@ -100,8 +97,7 @@ def save(self, path: str): ) def load(self, path: str): - """ - :param str path: path to load model + """:param str path: path to load model """ with open(path, "r", encoding="utf-8") as f: data = json.load(f) diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index 37cba0b93..3004f4852 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's dataset/corpus management. +"""Command line for PyThaiNLP's dataset/corpus management. """ import argparse diff --git a/pythainlp/cli/soundex.py b/pythainlp/cli/soundex.py index d6f97f0db..fc1ee9fd6 100644 --- a/pythainlp/cli/soundex.py +++ b/pythainlp/cli/soundex.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's soundex. +"""Command line for PyThaiNLP's soundex. It takes input text from the command line. """ diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index bd8cabd47..f6e9c8df3 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's taggers. +"""Command line for PyThaiNLP's taggers. """ import argparse diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 156879b10..066414f53 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Command line for PyThaiNLP's tokenizers. +"""Command line for PyThaiNLP's tokenizers. """ import argparse diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py index 366fff8e9..9aa47e950 100644 --- a/pythainlp/coref/__init__.py +++ b/pythainlp/coref/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PyThaiNLP Coreference Resolution +"""PyThaiNLP Coreference Resolution """ __all__ = ["coreference_resolution"] diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 4ffe50f35..be6764966 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -9,8 +9,7 @@ def coreference_resolution( texts: list[str], model_name: str = "han-coref-v1.0", device: str = "cpu" ): - """ - Coreference Resolution + """Coreference Resolution :param List[str] texts: list of texts to apply coreference resolution to :param str model_name: coreference resolution model diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 94b158c0e..48376ed6d 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Corpus related functions. +"""Corpus related functions. Access to dictionaries, word lists, and language models. Including download manager. @@ -72,40 +71,23 @@ def corpus_path() -> str: - """ - Get path where corpus files are kept locally. + """Get path where corpus files are kept locally. """ return _CORPUS_PATH def corpus_db_url() -> str: - """ - Get remote URL of corpus catalog. + """Get remote URL of corpus catalog. """ return _CORPUS_DB_URL def corpus_db_path() -> str: - """ - Get local path of corpus catalog. + """Get local path of corpus catalog. """ return _CORPUS_DB_PATH -from pythainlp.corpus.core import ( - download, - get_corpus, - get_corpus_as_is, - get_corpus_db, - get_corpus_db_detail, - get_corpus_default_db, - get_corpus_path, - get_hf_hub, - get_path_folder_corpus, - make_safe_directory_name, - path_pythainlp_corpus, - remove, -) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.common import ( countries, find_synonyms, @@ -123,6 +105,20 @@ def corpus_db_path() -> str: thai_words, thai_wsd_dict, ) +from pythainlp.corpus.core import ( + download, + get_corpus, + get_corpus_as_is, + get_corpus_db, + get_corpus_db_detail, + get_corpus_default_db, + get_corpus_path, + get_hf_hub, + get_path_folder_corpus, + make_safe_directory_name, + path_pythainlp_corpus, + remove, +) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.icu import thai_icu_words from pythainlp.corpus.volubilis import thai_volubilis_words from pythainlp.corpus.wikipedia import thai_wikipedia_titles diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 285d02838..cf467a754 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Common lists of words. +"""Common lists of words. """ from __future__ import annotations @@ -65,8 +64,7 @@ def countries() -> frozenset[str]: - """ - Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", + """Return a frozenset of country names in Thai such as "แคนาดา", "โรมาเนีย", "แอลจีเรีย", and "ลาว". \n(See: `dev/pythainlp/corpus/countries_th.txt\ `_) @@ -82,8 +80,7 @@ def countries() -> frozenset[str]: def provinces(details: bool = False) -> frozenset[str] | list[dict]: - """ - Return a frozenset of Thailand province names in Thai such as "กระบี่", + """Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". \n(See: `dev/pythainlp/corpus/thailand_provinces_th.txt\ `_) @@ -125,8 +122,7 @@ def provinces(details: bool = False) -> frozenset[str] | list[dict]: def thai_syllables() -> frozenset[str]: - """ - Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", + """Return a frozenset of Thai syllables such as "กรอบ", "ก็", "๑", "โมบ", "โมน", "โม่ง", "กา", "ก่า", and, "ก้า". \n(See: `dev/pythainlp/corpus/syllables_th.txt\ `_) @@ -143,8 +139,7 @@ def thai_syllables() -> frozenset[str]: def thai_words() -> frozenset[str]: - """ - Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", + """Return a frozenset of Thai words such as "กติกา", "กดดัน", "พิษ", and "พิษภัย". \n(See: `dev/pythainlp/corpus/words_th.txt\ `_) @@ -159,8 +154,7 @@ def thai_words() -> frozenset[str]: def thai_orst_words() -> frozenset[str]: - """ - Return a frozenset of Thai words from Royal Society of Thailand + """Return a frozenset of Thai words from Royal Society of Thailand \n(See: `dev/pythainlp/corpus/thai_orst_words.txt\ `_) @@ -175,8 +169,7 @@ def thai_orst_words() -> frozenset[str]: def thai_stopwords() -> frozenset[str]: - """ - Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", + """Return a frozenset of Thai stopwords such as "มี", "ไป", "ไง", "ขณะ", "การ", and "ประการหนึ่ง". \n(See: `dev/pythainlp/corpus/stopwords_th.txt\ `_) We use stopword lists by thesis's เพ็ญศิริ ลี้ตระกูล. @@ -198,8 +191,7 @@ def thai_stopwords() -> frozenset[str]: def thai_negations() -> frozenset[str]: - """ - Return a frozenset of Thai negation words including "ไม่" and "แต่". + """Return a frozenset of Thai negation words including "ไม่" and "แต่". \n(See: `dev/pythainlp/corpus/negations_th.txt\ `_) @@ -214,8 +206,7 @@ def thai_negations() -> frozenset[str]: def thai_family_names() -> frozenset[str]: - """ - Return a frozenset of Thai family names + """Return a frozenset of Thai family names \n(See: `dev/pythainlp/corpus/family_names_th.txt\ `_) @@ -230,8 +221,7 @@ def thai_family_names() -> frozenset[str]: def thai_female_names() -> frozenset[str]: - """ - Return a frozenset of Thai female names + """Return a frozenset of Thai female names \n(See: `dev/pythainlp/corpus/person_names_female_th.txt\ `_) @@ -246,8 +236,7 @@ def thai_female_names() -> frozenset[str]: def thai_male_names() -> frozenset[str]: - """ - Return a frozenset of Thai male names + """Return a frozenset of Thai male names \n(See: `dev/pythainlp/corpus/person_names_male_th.txt\ `_) @@ -262,8 +251,7 @@ def thai_male_names() -> frozenset[str]: def thai_dict() -> dict: - """ - Return Thai dictionary with definition from wiktionary. + """Return Thai dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -292,8 +280,7 @@ def thai_dict() -> dict: def thai_wsd_dict() -> dict: - """ - Return Thai Word Sense Disambiguation dictionary with definition from wiktionary. + """Return Thai Word Sense Disambiguation dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -320,8 +307,7 @@ def thai_wsd_dict() -> dict: def thai_synonyms() -> dict: - """ - Return Thai synonyms. + """Return Thai synonyms. \n(See: `thai_synonym\ `_) @@ -361,8 +347,7 @@ def thai_synonym() -> dict: def find_synonyms(word: str) -> list[str]: - """ - Find synonyms + """Find synonyms :param str word: Thai word :return: List of synonyms of the input word or an empty list if it isn't exist. diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 084202403..f509a0f02 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Corpus related functions. +"""Corpus related functions. """ from __future__ import annotations @@ -19,8 +18,7 @@ def get_corpus_db(url: str): - """ - Get corpus catalog from server. + """Get corpus catalog from server. :param str url: URL corpus catalog """ @@ -38,8 +36,7 @@ def get_corpus_db(url: str): def get_corpus_db_detail(name: str, version: str = "") -> dict: - """ - Get details about a corpus, using information from local catalog. + """Get details about a corpus, using information from local catalog. :param str name: name of corpus :return: details about corpus @@ -61,8 +58,7 @@ def get_corpus_db_detail(name: str, version: str = "") -> dict: def path_pythainlp_corpus(filename: str) -> str: - """ - Get path pythainlp.corpus data + """Get path pythainlp.corpus data :param str filename: filename of the corpus to be read @@ -73,8 +69,7 @@ def path_pythainlp_corpus(filename: str) -> str: def get_corpus(filename: str, comments: bool = True) -> frozenset: - """ - Read corpus data from file and return a frozenset. + """Read corpus data from file and return a frozenset. Each line in the file will be a member of the set. @@ -144,8 +139,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: def get_corpus_as_is(filename: str) -> list: - """ - Read corpus data from file, as it is, and return a list. + """Read corpus data from file, as it is, and return a list. Each line in the file will be a member of the list. @@ -180,8 +174,7 @@ def get_corpus_as_is(filename: str) -> list: def get_corpus_default_db(name: str, version: str = "") -> str | None: - """ - Get model path from default_db.json + """Get model path from default_db.json :param str name: corpus name :return: path to the corpus or **None** if the corpus doesn't \ @@ -212,8 +205,7 @@ def get_corpus_default_db(name: str, version: str = "") -> str | None: def get_corpus_path( name: str, version: str = "", force: bool = False ) -> str | None: - """ - Get corpus path. + """Get corpus path. :param str name: corpus name :param str version: version @@ -252,7 +244,6 @@ def get_corpus_path( print(get_corpus_path('wiki_lm_lstm')) # output: /root/pythainlp-data/thwiki_model_lstm.pth """ - CUSTOMIZE: dict[str, str] = { # "the corpus name":"path" } @@ -286,8 +277,7 @@ def get_corpus_path( def _download(url: str, dst: str) -> int: - """ - Download helper. + """Download helper. @param: URL for downloading file @param: dst place to put the file into @@ -322,8 +312,7 @@ def _download(url: str, dst: str) -> int: def _check_hash(dst: str, md5: str) -> None: - """ - Check hash helper. + """Check hash helper. @param: dst place to put the file into @param: md5 place to file hash (MD5) @@ -340,8 +329,7 @@ def _check_hash(dst: str, md5: str) -> None: def _version2int(v: str) -> int: - """ - X.X.X => X0X0X + """X.X.X => X0X0X """ if "-" in v: v = v.split("-")[0] @@ -405,8 +393,7 @@ def _check_version(cause: str) -> bool: def download( name: str, force: bool = False, url: str = "", version: str = "" ) -> bool: - """ - Download corpus. + """Download corpus. The available corpus names can be seen in this file: https://pythainlp.org/pythainlp-corpus/db.json @@ -558,8 +545,7 @@ def download( def remove(name: str) -> bool: - """ - Remove corpus + """Remove corpus :param str name: corpus name :return: **True** if the corpus is found and successfully removed. @@ -615,8 +601,7 @@ def get_path_folder_corpus(name, version, *path): def make_safe_directory_name(name: str) -> str: - """ - Make safe directory name + """Make safe directory name :param str name: directory name :return: safe directory name @@ -657,8 +642,7 @@ def make_safe_directory_name(name: str) -> str: def get_hf_hub(repo_id: str, filename: str = "") -> str: - """ - HuggingFace Hub in :mod:`pythainlp` data directory. + """HuggingFace Hub in :mod:`pythainlp` data directory. :param str repo_id: repo_id :param str filename: filename (optional, default is empty string). diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py index ada98f81a..400be2863 100644 --- a/pythainlp/corpus/icu.py +++ b/pythainlp/corpus/icu.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Provides an optional word list from International Components for Unicode (ICU) dictionary. +"""Provides an optional word list from International Components for Unicode (ICU) dictionary. """ from __future__ import annotations @@ -13,14 +12,12 @@ def thai_icu_words() -> frozenset[str]: - """ - Return a frozenset of words from the Thai dictionary for BreakIterator of the + """Return a frozenset of words from the Thai dictionary for BreakIterator of the International Components for Unicode (ICU). :return: :class:`frozenset` containing Thai words. :rtype: :class:`frozenset` """ - _WORDS = get_corpus(_THAI_ICU_FILENAME, comments=False) return _WORDS diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index 0a0b540fc..193af0e2c 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai unigram word frequency from OSCAR Corpus (words tokenized using ICU) +"""Thai unigram word frequency from OSCAR Corpus (words tokenized using ICU) Credit: Korakot Chaovavanich https://web.facebook.com/groups/colab.thailand/permalink/1524070061101680/ @@ -20,8 +19,7 @@ def word_freqs() -> list[tuple[str, int]]: - """ - Get word frequency from OSCAR Corpus (words tokenized using ICU) + """Get word frequency from OSCAR Corpus (words tokenized using ICU) """ freqs: list[tuple[str, int]] = [] path = get_corpus_path(_OSCAR_FILENAME) @@ -44,8 +42,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from OSCAR Corpus (words tokenized using ICU) + """Get unigram word frequency from OSCAR Corpus (words tokenized using ICU) """ freqs: dict[str, int] = defaultdict(int) path = get_corpus_path(_OSCAR_FILENAME) diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 4a163a1f2..94a73f922 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-English Transliteration Dictionary v1.4 +"""Thai-English Transliteration Dictionary v1.4 Wannaphong Phatthiyaphaibun. (2022). wannaphong/thai-english-transliteration-dictionary: v1.4 (v1.4). @@ -27,8 +26,7 @@ def get_transliteration_dict() -> defaultdict: - """ - Get Thai to English transliteration dictionary. + """Get Thai to English transliteration dictionary. The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. """ diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 7aef7b5c2..af9513c6d 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project. # SPDX-License-Identifier: Apache-2.0 -""" -Thai National Corpus word frequency +"""Thai National Corpus word frequency """ from __future__ import annotations @@ -23,8 +22,7 @@ def word_freqs() -> list[tuple[str, int]]: - """ - Get word frequency from Thai National Corpus (TNC) + """Get word frequency from Thai National Corpus (TNC) \n(See: `dev/pythainlp/corpus/tnc_freq.txt\ `_) @@ -41,8 +39,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from Thai National Corpus (TNC) + """Get unigram word frequency from Thai National Corpus (TNC) """ freqs: dict[str, int] = defaultdict(int) lines = list(get_corpus(_UNIGRAM_FILENAME)) @@ -55,8 +52,7 @@ def unigram_word_freqs() -> dict[str, int]: def bigram_word_freqs() -> dict[tuple[str, str], int]: - """ - Get bigram word frequency from Thai National Corpus (TNC) + """Get bigram word frequency from Thai National Corpus (TNC) """ freqs: dict[tuple[str, str], int] = defaultdict(int) path = get_corpus_path(_BIGRAM_CORPUS_NAME) @@ -73,8 +69,7 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]: def trigram_word_freqs() -> dict[tuple[str, str, str], int]: - """ - Get trigram word frequency from Thai National Corpus (TNC) + """Get trigram word frequency from Thai National Corpus (TNC) """ freqs: dict[tuple[str, str, str], int] = defaultdict(int) path = get_corpus_path(_TRIGRAM_CORPUS_NAME) diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index a32051259..99134ca4b 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Textbook Corpus (TTC) word frequency +"""Thai Textbook Corpus (TTC) word frequency Credit: Korakot Chaovavanich https://www.facebook.com/photo.php?fbid=363640477387469&set=gm.434330506948445&type=3&permPage=1 @@ -20,8 +19,7 @@ def word_freqs() -> list[tuple[str, int]]: - """ - Get word frequency from Thai Textbook Corpus (TTC) + """Get word frequency from Thai Textbook Corpus (TTC) \n(See: `dev/pythainlp/corpus/ttc_freq.txt\ `_) """ @@ -36,8 +34,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """ - Get unigram word frequency from Thai Textbook Corpus (TTC) + """Get unigram word frequency from Thai Textbook Corpus (TTC) """ freqs: dict[str, int] = defaultdict(int) diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index d4e86c156..80701fd26 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Tool for creating word lists +"""Tool for creating word lists codes are from Korakot Chaovavanich. :See also: @@ -23,8 +22,7 @@ def index_pairs(words: list[str]) -> Iterator[tuple[int, int]]: - """ - Return beginning and ending indexes of word pairs + """Return beginning and ending indexes of word pairs """ i = 0 for w in words: @@ -36,8 +34,7 @@ def find_badwords( tokenize: Callable[[str], list[str]], training_data: Iterable[Iterable[str]], ) -> set[str]: - """ - Find words that do not work well with the `tokenize` function + """Find words that do not work well with the `tokenize` function for the provided `training_data`. :param Callable[[str], List[str]] tokenize: a tokenize function @@ -73,8 +70,7 @@ def revise_wordset( orig_words: Iterable[str], training_data: Iterable[Iterable[str]], ) -> set[str]: - """ - Revise a set of words that could improve tokenization performance of + """Revise a set of words that could improve tokenization performance of a dictionary-based `tokenize` function. `orig_words` will be used as a base set for the dictionary. @@ -92,7 +88,7 @@ def revise_wordset( :Example:: :: - + from pythainlp.corpus import thai_words from pythainlp.corpus.util import revise_wordset from pythainlp.tokenize.longest import segment @@ -121,8 +117,7 @@ def revise_wordset( def revise_newmm_default_wordset( training_data: Iterable[Iterable[str]], ) -> set[str]: - """ - Revise a set of word that could improve tokenization performance of + """Revise a set of word that could improve tokenization performance of `pythainlp.tokenize.newmm`, a dictionary-based tokenizer and a default tokenizer for PyThaiNLP. diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py index 8f21a5dba..14bbc0ed3 100644 --- a/pythainlp/corpus/volubilis.py +++ b/pythainlp/corpus/volubilis.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Provides an optional word list from the Volubilis dictionary. +"""Provides an optional word list from the Volubilis dictionary. """ from __future__ import annotations @@ -14,9 +13,8 @@ def thai_volubilis_words() -> frozenset[str]: - """ - Return a frozenset of Thai words from the Volubilis dictionary - + """Return a frozenset of Thai words from the Volubilis dictionary + See: `dev/pythainlp/corpus/volubilis_words_th.txt\ `_ diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py index a036f4568..78283e716 100644 --- a/pythainlp/corpus/wikipedia.py +++ b/pythainlp/corpus/wikipedia.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Provides an optional word list from Thai Wikipedia titles. +"""Provides an optional word list from Thai Wikipedia titles. """ from __future__ import annotations @@ -14,8 +13,7 @@ def thai_wikipedia_titles() -> frozenset[str]: - """ - Return a frozenset of words from Thai Wikipedia titles corpus. + """Return a frozenset of words from Thai Wikipedia titles corpus. They are mostly nouns and noun phrases, including event, organization, people, place, and product names. Commonly misspelled words are included intentionally. diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py index afdd27548..7373b96e2 100644 --- a/pythainlp/corpus/wordnet.py +++ b/pythainlp/corpus/wordnet.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -NLTK WordNet wrapper +"""NLTK WordNet wrapper API here is exactly the same as NLTK WordNet API, except that the lang (language) argument is "tha" (Thai) by default. @@ -29,8 +28,7 @@ def synsets(word: str, pos: str | None = None, lang: str = "tha"): - """ - This function returns the synonym set for all lemmas of the given word + """This function returns the synonym set for all lemmas of the given word with an optional argument to constrain the part of speech of the word. :param str word: word to find synsets of @@ -77,8 +75,7 @@ def synsets(word: str, pos: str | None = None, lang: str = "tha"): def synset(name_synsets): - """ - This function returns the synonym set (synset) given the name of the synset + """This function returns the synonym set (synset) given the name of the synset (i.e. 'dog.n.01', 'chase.v.01'). :param str name_synsets: name of the synset @@ -102,8 +99,7 @@ def synset(name_synsets): def all_lemma_names(pos: str | None = None, lang: str = "tha"): - """ - This function returns all lemma names for all synsets of the given + """This function returns all lemma names for all synsets of the given part of speech tag and language. If part of speech tag is not specified, all synsets of all parts of speech will be used. @@ -145,8 +141,7 @@ def all_lemma_names(pos: str | None = None, lang: str = "tha"): def all_synsets(pos: str | None = None): - """ - This function iterates over all synsets constrained by the given + """This function iterates over all synsets constrained by the given part of speech tag. :param str | None pos: part of speech tag. Default is None. @@ -176,8 +171,7 @@ def all_synsets(pos: str | None = None): def langs(): - """ - This function returns a set of ISO-639 language codes. + """This function returns a set of ISO-639 language codes. :return: ISO-639 language codes :rtype: list[str] @@ -195,8 +189,7 @@ def langs(): def lemmas(word: str, pos: str | None = None, lang: str = "tha"): - """ - This function returns all lemmas given the word with an optional + """This function returns all lemmas given the word with an optional argument to constrain the part of speech of the word. :param str word: word to find lemmas of @@ -239,8 +232,7 @@ def lemmas(word: str, pos: str | None = None, lang: str = "tha"): def lemma(name_synsets): - """ - This function returns lemma object given the name. + """This function returns lemma object given the name. .. note:: Support only English language (*eng*). @@ -267,8 +259,7 @@ def lemma(name_synsets): def lemma_from_key(key): - """ - This function returns lemma object given the lemma key. + """This function returns lemma object given the lemma key. This is similar to :func:`lemma` but it needs to be given the key of lemma instead of the name of lemma. @@ -294,8 +285,7 @@ def lemma_from_key(key): def path_similarity(synsets1, synsets2): - """ - This function returns similarity between two synsets based on the + """This function returns similarity between two synsets based on the shortest path distance calculated using the equation below. .. math:: @@ -334,8 +324,7 @@ def path_similarity(synsets1, synsets2): def lch_similarity(synsets1, synsets2): - """ - This function returns Leacock Chodorow similarity (LCH) + """This function returns Leacock Chodorow similarity (LCH) between two synsets, based on the shortest path distance and the maximum depth of the taxonomy. The equation to calculate LCH similarity is shown below: @@ -372,8 +361,7 @@ def lch_similarity(synsets1, synsets2): def wup_similarity(synsets1, synsets2): - """ - This function returns Wu-Palmer similarity (WUP) between two synsets, + """This function returns Wu-Palmer similarity (WUP) between two synsets, based on the depth of the two senses in the taxonomy and their Least Common Subsumer (most specific ancestor node). @@ -404,8 +392,7 @@ def wup_similarity(synsets1, synsets2): def morphy(form, pos: str | None = None): - """ - This function finds a possible base form for the given form, + """This function finds a possible base form for the given form, with the given part of speech. :param str form: the form to finds the base form of @@ -435,8 +422,7 @@ def morphy(form, pos: str | None = None): def custom_lemmas(tab_file, lang: str): - """ - This function reads a custom tab file + """This function reads a custom tab file (see: http://compling.hss.ntu.edu.sg/omw/) containing mappings of lemmas in the given language. diff --git a/pythainlp/el/__init__.py b/pythainlp/el/__init__.py index f6e8a6a28..9638fa2fa 100644 --- a/pythainlp/el/__init__.py +++ b/pythainlp/el/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -pythainlp.el +"""pythainlp.el """ __all__ = ["EntityLinker"] diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index b991d78e2..17ea4cb76 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -11,8 +11,7 @@ def __init__( device: str = "cuda", tag: str = "wikidata", ): - """ - EntityLinker + """EntityLinker :param str model_name: model name (bela) :param str device: device for running model on @@ -37,8 +36,7 @@ def __init__( self.model = MultiEL(model_name=self.model_name, device=self.device) def get_el(self, list_text: list[str] | str) -> list[dict] | str: - """ - Get Entity Linking from Thai Text + """Get Entity Linking from Thai Text :param str Union[List[str], str]: list of Thai text or text :return: list of entity linking diff --git a/pythainlp/generate/__init__.py b/pythainlp/generate/__init__.py index b3cb92c4a..456d90467 100644 --- a/pythainlp/generate/__init__.py +++ b/pythainlp/generate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Text Generation +"""Thai Text Generation """ __all__ = ["Bigram", "Trigram", "Unigram"] diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 5f3bb60fc..bf317e6a7 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text generator using n-gram language model +"""Text generator using n-gram language model codes are from https://towardsdatascience.com/understanding-word-n-grams-and-n-gram-probability-in-natural-language-processing-9d9eef0fa058 @@ -22,8 +21,7 @@ class Unigram: - """ - Text generator using Unigram + """Text generator using Unigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -53,8 +51,7 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> list[str] | str: - """ - :param str start_seq: word to begin sentence with + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence @@ -110,8 +107,7 @@ def _next_word( class Bigram: - """ - Text generator using Bigram + """Text generator using Bigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -126,8 +122,7 @@ def __init__(self, name: str = "tnc"): self.words = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str) -> float: - """ - probability of word + """Probability of word :param int t1: text 1 :param int t2: text 2 @@ -149,8 +144,7 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> list[str] | str: - """ - :param str start_seq: word to begin sentence with + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence @@ -198,8 +192,7 @@ def gen_sentence( class Trigram: - """ - Text generator using Trigram + """Text generator using Trigram :param str name: corpus name * *tnc* - Thai National Corpus (default) @@ -216,8 +209,7 @@ def __init__(self, name: str = "tnc"): self.words = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str, t3: str) -> float: - """ - probability of word + """Probability of word :param int t1: text 1 :param int t2: text 2 @@ -241,8 +233,7 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> list[str] | str: - """ - :param str start_seq: word to begin sentence with + """:param str start_seq: word to begin sentence with :param int N: number of words :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 2498937f2..e33d20b18 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai2fit: Thai Wikipeida Language Model for Text Generation +"""Thai2fit: Thai Wikipeida Language Model for Text Generation Codes are from https://github.com/PyThaiNLP/tutorials/blob/master/source/notebooks/text_generation.ipynb @@ -89,8 +88,7 @@ def gen_sentence( prob: float = 0.001, output_str: bool = True, ) -> list[str] | str: - """ - Text generator using Thai2fit + """Text generator using Thai2fit :param str start_seq: word to begin sentence with :param int N: number of words diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index 99031bc19..12289e338 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -33,8 +33,7 @@ def load_model( offload_folder: str = "./", low_cpu_mem_usage: bool = True, ): - """ - Load model + """Load model :param str model_path: model path :param bool return_dict: return dict @@ -78,8 +77,7 @@ def gen_instruct( thai_only: bool = True, skip_special_tokens: bool = True, ): - """ - Generate Instruct + """Generate Instruct :param str text: text :param int max_new_tokens: maximum number of new tokens @@ -136,8 +134,7 @@ def instruct_generate( thai_only: bool = True, skip_special_tokens: bool = True, ): - """ - Generate Instruct + """Generate Instruct :param str instruct: Instruct :param str context: context (optional, default is empty string) diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index c22a8212f..5f8b0281f 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -11,13 +11,11 @@ class KhaveeVerifier: def __init__(self): - """ - KhaveeVerifier: Thai Poetry verifier + """KhaveeVerifier: Thai Poetry verifier """ def _has_true_final_yl(self, word: str) -> bool: - """ - Check if ย or ล is a true final consonant + """Check if ย or ล is a true final consonant (not just part of the vowel sound with ไ/ใ) :param str word: Thai word @@ -32,8 +30,7 @@ def _has_true_final_yl(self, word: str) -> bool: return consonant_count >= 2 and word[-1] in ["ย", "ล"] def check_sara(self, word: str) -> str: - """ - Check the vowels in the Thai word. + """Check the vowels in the Thai word. :param str word: Thai word :return: vowel name of the word @@ -221,8 +218,7 @@ def check_sara(self, word: str) -> str: return sara[0] def check_marttra(self, word: str) -> str: - """ - Check the Thai spelling Section in the Thai word. + """Check the Thai spelling Section in the Thai word. :param str word: Thai word :return: name of spelling Section of the word. @@ -315,8 +311,7 @@ def check_marttra(self, word: str) -> str: return "Cant find Marttra in this word" def is_sumpus(self, word1: str, word2: str) -> bool: - """ - Check the rhyme between two words. + """Check the rhyme between two words. :param str word1: Thai word :param str word2: Thai word @@ -382,8 +377,7 @@ def check_karu_lahu(self, text): return "lahu" def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: - """ - Check the suitability of the poem according to Thai principles. + """Check the suitability of the poem according to Thai principles. :param str text: Thai poem :param int k_type: type of Thai poem @@ -519,7 +513,8 @@ def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: else: return error except: - return "Something went wrong. Make sure you enter it in the correct form of klon 8." + return ("Something went wrong. " + "Make sure you enter it in the correct form of klon 8.") elif k_type == 4: try: error = [] @@ -626,12 +621,13 @@ def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: def check_aek_too( self, text: list[str] | str, dead_syllable_as_aek: bool = False ) -> list[bool] | list[str] | bool | str: - """ - Checker of Thai tonal words + """Checker of Thai tonal words :param Union[List[str], str] text: Thai word or list of Thai words - :param bool dead_syllable_as_aek: if True, dead syllable will be considered as aek - :return: the check result if the word is aek or too or False (not both) or list of check results if input is list + :param bool dead_syllable_as_aek: if True, dead syllable will be + considered as aek + :return: the check result if the word is aek or too or False (not both) + or list of check results if input is list :rtype: Union[List[bool], List[str], bool, str] :Example: @@ -670,8 +666,7 @@ def check_aek_too( return False def handle_karun_sound_silence(self, word: str) -> str: - """ - Handle silent sounds in Thai words using '์' character (Karun) + """Handle silent sounds in Thai words using '์' character (Karun) by stripping all characters before the 'Karun' character that should be silenced :param str text: Thai word diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 35e4b9167..26fd4fa94 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -8,8 +8,7 @@ def calculate_ngram_counts( list_words: list[str], n_min: int = 2, n_max: int = 4 ) -> dict[tuple[str], int]: - """ - Calculates the counts of n-grams in the list words for the specified range. + """Calculates the counts of n-grams in the list words for the specified range. :param List[str] list_words: List of string :param int n_min: The minimum n-gram size (default: 2). @@ -18,7 +17,6 @@ def calculate_ngram_counts( :return: A dictionary where keys are n-grams and values are their counts. :rtype: Dict[Tuple[str], int] """ - ngram_counts = {} for n in range(n_min, n_max + 1): @@ -30,8 +28,7 @@ def calculate_ngram_counts( def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]: - """ - Remove repeated n-grams + """Remove repeated n-grams :param List[str] string_list: List of string :param int n: n-gram size diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py index 3f8ed10dc..b46042636 100644 --- a/pythainlp/morpheme/__init__.py +++ b/pythainlp/morpheme/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PyThaiNLP morpheme +"""PyThaiNLP morpheme """ __all__ = ["nighit", "is_native_thai"] diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py index 9fce45aaf..6891b1cf0 100644 --- a/pythainlp/morpheme/thaiwordcheck.py +++ b/pythainlp/morpheme/thaiwordcheck.py @@ -1,16 +1,17 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Check if a word is a "native Thai word" +"""Check if a word is a "native Thai word" Adapted from https://github.com/wannaphong/open-thai-nlp-document/blob/master/check_thai_word.md References +---------- - ทีมงานทรูปลูกปัญญา 2015. ลักษณะของคำไทยแท้ \ http://www.trueplookpanya.com/learning/detail/30589-043067 - วารุณี บำรุงรส 2010. คำไทยแท้ https://www.gotoknow.org/posts/377619 + """ from __future__ import annotations @@ -67,8 +68,7 @@ def is_native_thai(word: str) -> bool: - """ - Check if a word is an "native Thai word" (Thai: "คำไทยแท้") + """Check if a word is an "native Thai word" (Thai: "คำไทยแท้") This function is based on a simple heuristic algorithm and cannot be entirely reliable. diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index 11b432d93..212c3e23a 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -7,8 +7,7 @@ def nighit(w1: str, w2: str) -> str: - """ - Nighit (นิคหิต or ํ ) is the niggahita in Thai language for create new \ + """Nighit (นิคหิต or ํ ) is the niggahita in Thai language for create new \ words from Pali language in Thai. The function use simple method to create new Thai word from two words \ that the root is from Pali language. diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py index e4afb08e1..01fbdada9 100644 --- a/pythainlp/parse/__init__.py +++ b/pythainlp/parse/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PyThaiNLP Parse +"""PyThaiNLP Parse """ __all__ = ["dependency_parsing"] diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index 6421dd87a..619a5a8df 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -13,8 +13,7 @@ def dependency_parsing( tag: str = "str", engine: str = "esupar", ) -> list[list[str]] | str: - """ - Dependency Parsing + """Dependency Parsing :param str text: text to apply dependency parsing to :param str model: model for using with engine \ diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 06ae3a35e..232051a16 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -1,5 +1,4 @@ -""" -esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages +"""esupar: Tokenizer, POS tagger and dependency parser with BERT/RoBERTa/DeBERTa models for Japanese and other languages GitHub: https://github.com/KoichiYasuoka/esupar """ diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py index 3f44a61a7..30aeb0112 100644 --- a/pythainlp/parse/spacy_thai_engine.py +++ b/pythainlp/parse/spacy_thai_engine.py @@ -1,5 +1,4 @@ -""" -spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies. +"""spacy_thai: Tokenizer, POS tagger, and dependency parser for the Thai language using Universal Dependencies. GitHub: https://github.com/KoichiYasuoka/spacy-thai diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 2f4aec7b2..71737ae2c 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -1,5 +1,4 @@ -""" -TransformersUD +"""TransformersUD Author: Prof. Koichi Yasuoka diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index 904c64dd3..05dabb59e 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -1,5 +1,4 @@ -""" -UDgoeswith +"""UDgoeswith Author: Prof. Koichi Yasuoka diff --git a/pythainlp/phayathaibert/__init__.py b/pythainlp/phayathaibert/__init__.py index 088f8d219..c2a2e24b2 100644 --- a/pythainlp/phayathaibert/__init__.py +++ b/pythainlp/phayathaibert/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PhayaThaiBERT +"""PhayaThaiBERT """ __all__ = [ diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index a73bcdfe9..0619b16b0 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -32,8 +32,7 @@ def __init__(self): self.SPACE_SPECIAL_TOKEN = "<_>" def replace_url(self, text: str) -> str: - """ - Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) + """Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) :param str text: text to replace url :return: text where urls are replaced :rtype: str @@ -44,8 +43,7 @@ def replace_url(self, text: str) -> str: return re.sub(_PAT_URL, self._TK_URL, text) def rm_brackets(self, text: str) -> str: - """ - Remove all empty brackets and artifacts within brackets from `text`. + """Remove all empty brackets and artifacts within brackets from `text`. :param str text: text to remove useless brackets :return: text where all useless brackets are removed :rtype: str @@ -84,8 +82,7 @@ def rm_brackets(self, text: str) -> str: return new_line def replace_newlines(self, text: str) -> str: - """ - Replace newlines in `text` with spaces. + """Replace newlines in `text` with spaces. :param str text: text to replace all newlines with spaces :return: text where all newlines are replaced with spaces :rtype: str @@ -93,12 +90,10 @@ def replace_newlines(self, text: str) -> str: >>> rm_useless_spaces("hey whats\n\nup") hey whats up """ - return re.sub(r"[\n]", " ", text.strip()) def rm_useless_spaces(self, text: str) -> str: - """ - Remove multiple spaces in `text`. (code from `fastai`) + """Remove multiple spaces in `text`. (code from `fastai`) :param str text: text to replace useless spaces :return: text where all spaces are reduced to one :rtype: str @@ -109,8 +104,7 @@ def rm_useless_spaces(self, text: str) -> str: return re.sub(" {2,}", " ", text) def replace_spaces(self, text: str, space_token: str = "<_>") -> str: - """ - Replace spaces with _ + """Replace spaces with _ :param str text: text to replace spaces :return: text where all spaces replaced with _ :rtype: str @@ -121,8 +115,7 @@ def replace_spaces(self, text: str, space_token: str = "<_>") -> str: return re.sub(" ", space_token, text) def replace_rep_after(self, text: str) -> str: - """ - Replace repetitions at the character level in `text` + """Replace repetitions at the character level in `text` :param str text: input text to replace character repetition :return: text with repetitive tokens removed. :rtype: str @@ -140,8 +133,7 @@ def _replace_rep(m): return re_rep.sub(_replace_rep, text) def replace_wrep_post(self, toks: list[str]) -> list[str]: - """ - Replace repetitive words post tokenization; + """Replace repetitive words post tokenization; fastai `replace_wrep` does not work well with Thai. :param List[str] toks: list of tokens :return: list of tokens where repetitive words are removed. @@ -167,8 +159,7 @@ def replace_wrep_post(self, toks: list[str]) -> list[str]: return res[1:] def remove_space(self, toks: list[str]) -> list[str]: - """ - Do not include space for bag-of-word models. + """Do not include space for bag-of-word models. :param List[str] toks: list of tokens :return: List of tokens where space tokens (" ") are filtered out :rtype: List[str] @@ -254,8 +245,7 @@ def augment( num_augs: int = 3, sample: bool = False, ) -> list[str]: - """ - Text augmentation from PhayaThaiBERT + """Text augmentation from PhayaThaiBERT :param str text: Thai text :param int num_augs: an amount of augmentation text needed as an output @@ -316,8 +306,7 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: def get_tag( self, sentence: str, strategy: str = "simple" ) -> list[list[tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. + """Marks sentences with part-of-speech (POS) tags. :param str sentence: a list of lists of tokenized words :return: a list of lists of tuples (word, POS tag) @@ -364,8 +353,7 @@ def get_ner( pos: bool = False, strategy: str = "simple", ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: output with part-of-speech tags.\ @@ -436,8 +424,7 @@ def get_ner( def segment(sentence: str) -> list[str]: - """ - Subword tokenize of PhayaThaiBERT, \ + """Subword tokenize of PhayaThaiBERT, \ sentencepiece from WangchanBERTa model with vocabulary expansion. :param str sentence: text to be tokenized diff --git a/pythainlp/soundex/__init__.py b/pythainlp/soundex/__init__.py index 6ff802204..4e002ebcd 100644 --- a/pythainlp/soundex/__init__.py +++ b/pythainlp/soundex/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex +"""Thai soundex Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ diff --git a/pythainlp/soundex/core.py b/pythainlp/soundex/core.py index e0435a92e..5446c3d77 100644 --- a/pythainlp/soundex/core.py +++ b/pythainlp/soundex/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex +"""Thai soundex Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ @@ -22,8 +21,7 @@ def soundex( text: str, engine: str = DEFAULT_SOUNDEX_ENGINE, length: int = 4 ) -> str: - """ - This function converts Thai text into phonetic code. + """This function converts Thai text into phonetic code. :param str text: word :param str engine: soundex engine diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py index 1885553a7..47f8aaddc 100644 --- a/pythainlp/soundex/lk82.py +++ b/pythainlp/soundex/lk82.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - LK82 system +"""Thai soundex - LK82 system Original paper: Vichit Lorchirachoonkul. 1982. A Thai soundex @@ -39,8 +38,7 @@ def lk82(text: str) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the Thai soundex algorithm named **LK82** [#lk82]_. :param str text: Thai word diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py index 924f9b58d..ef764eb52 100644 --- a/pythainlp/soundex/metasound.py +++ b/pythainlp/soundex/metasound.py @@ -1,14 +1,14 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - MetaSound system +"""Thai soundex - MetaSound system References: Snae & Brückner. (2009). Novel Phonetic Name Matching Algorithm with a Statistical Ontology for Analysing Names Given in Accordance with Thai Astrology. https://pdfs.semanticscholar.org/3983/963e87ddc6dfdbb291099aa3927a0e3e4ea6.pdf + """ from __future__ import annotations @@ -26,8 +26,7 @@ def metasound(text: str, length: int = 4) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the matching technique called **MetaSound** [#metasound]_ (combination between Soundex and Metaphone algorithms). MetaSound algorithm was developed specifically for the Thai language. diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py index ae74895ae..8adea3142 100644 --- a/pythainlp/soundex/prayut_and_somchaip.py +++ b/pythainlp/soundex/prayut_and_somchaip.py @@ -1,12 +1,15 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-English Cross-Language Transliterated Word Retrieval +"""Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique References: -Prayut Suwanvisat, Somchai Prasitjutrakul.Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf +Prayut Suwanvisat, Somchai Prasitjutrakul. +Thai-English Cross-Language Transliterated Word Retrieval using Soundex +Technique. In 1998 [cited 2022 Sep 8]. +Available from: +https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf """ from __future__ import annotations @@ -28,8 +31,7 @@ def prayut_and_somchaip(text: str, length: int = 4) -> str: - """ - This function converts English-Thai Cross-Language Transliterated Word into + """This function converts English-Thai Cross-Language Transliterated Word into phonetic code with the matching technique called **Soundex** [#prayut_and_somchaip]_. :param str text: English-Thai Cross-Language Transliterated Word diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py index a376d0715..6cf5e9924 100644 --- a/pythainlp/soundex/sound.py +++ b/pythainlp/soundex/sound.py @@ -14,8 +14,7 @@ def _clean_ipa(ipa: str) -> str: - """ - Clean IPA by removing tones and space between phonetic codes + """Clean IPA by removing tones and space between phonetic codes :param str ipa: IPA text :return: IPA with tones removed from the text @@ -35,8 +34,7 @@ def _clean_ipa(ipa: str) -> str: def word2audio(word: str) -> str: - """ - Convert word to IPA + """Convert word to IPA :param str word: Thai word :return: IPA with tones removed from the text @@ -59,8 +57,7 @@ def word2audio(word: str) -> str: def audio_vector(word: str) -> list[list[int]]: - """ - Convert audio to vector list + """Convert audio to vector list :param str word: Thai word :return: List of features from panphon @@ -78,8 +75,7 @@ def audio_vector(word: str) -> list[list[int]]: def word_approximation(word: str, list_word: list[str]) -> list[float]: - """ - Thai Word Approximation + """Thai Word Approximation :param str word: Thai word :param str list_word: Thai word diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py index caf6dd45a..cfb6a72f8 100644 --- a/pythainlp/soundex/udom83.py +++ b/pythainlp/soundex/udom83.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai soundex - Udom83 system +"""Thai soundex - Udom83 system Original paper: Wannee Udompanich. String searching for Thai alphabet @@ -50,8 +49,7 @@ def udom83(text: str) -> str: - """ - This function converts Thai text into phonetic code with the + """This function converts Thai text into phonetic code with the Thai soundex algorithm named **Udom83** [#udom83]_. :param str text: Thai word @@ -79,7 +77,6 @@ def udom83(text: str) -> str: udom83("ปัจจุบัน") # output: 'ป775300' """ - if not text or not isinstance(text, str): return "" diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 611a06230..8850ceb62 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Spell checking and correction. +"""Spell checking and correction. """ __all__ = [ diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index d99ce5154..2cca2dc81 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Spell checking functions +"""Spell checking functions """ from __future__ import annotations @@ -20,8 +19,7 @@ def default_spell_checker(): def spell(word: str, engine: str = "pn") -> list[str]: - """ - Provides a list of possible correct spellings of the given word. + """Provides a list of possible correct spellings of the given word. The list of words are from the words in the dictionary that incurs an edit distance value of 1 or 2. The result is a list of words sorted by their occurrences @@ -85,8 +83,7 @@ def spell(word: str, engine: str = "pn") -> list[str]: def correct(word: str, engine: str = "pn") -> str: - """ - Corrects the spelling of the given word by returning + """Corrects the spelling of the given word by returning the correctly spelled word. :param str word: word to correct spelling of @@ -140,8 +137,7 @@ def correct(word: str, engine: str = "pn") -> str: def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: - """ - Provides a list of possible correct spellings of sentence + """Provides a list of possible correct spellings of sentence :param List[str] list_words: list of words in sentence :param str engine: @@ -178,8 +174,7 @@ def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]: - """ - Corrects and returns the spelling of the given sentence + """Corrects and returns the spelling of the given sentence :param List[str] list_words: list of words in sentence :param str engine: diff --git a/pythainlp/spell/phunspell.py b/pythainlp/spell/phunspell.py index 26cb90f1e..c42fd54e3 100644 --- a/pythainlp/spell/phunspell.py +++ b/pythainlp/spell/phunspell.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Phunspell +"""Phunspell A pure Python spell checker utilizing spylls, a port of Hunspell. diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 12d2acc3e..695bc7cf1 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -1,5 +1,4 @@ -""" -Spell checker, using Peter Norvig algorithm. +"""Spell checker, using Peter Norvig algorithm. Spelling dictionary can be customized. Default spelling dictionary is based on Thai National Corpus. @@ -37,8 +36,7 @@ def _keep( max_len: int, dict_filter: Callable[[str], bool], ) -> bool: - """ - Checks whether a given word has the required minimum frequency min_freq + """Checks whether a given word has the required minimum frequency min_freq and its character length is between min_len and max_len (inclusive). """ if not word_freq or word_freq[1] < min_freq: @@ -52,8 +50,7 @@ def _keep( def _edits1(word: str) -> set[str]: - """ - Returns a set of words with an edit distance of 1 from the input word + """Returns a set of words with an edit distance of 1 from the input word """ splits = [(word[:i], word[i:]) for i in range(len(word) + 1)] deletes = [L + R[1:] for L, R in splits if R] @@ -65,8 +62,7 @@ def _edits1(word: str) -> set[str]: def _edits2(word: str) -> set[str]: - """ - Returns a set of words with an edit distance of 2 from the input word + """Returns a set of words with an edit distance of 2 from the input word """ return set(e2 for e1 in _edits1(word) for e2 in _edits1(e1)) @@ -78,8 +74,7 @@ def _convert_custom_dict( max_len: int, dict_filter: Callable[[str], bool] | None, ) -> list[tuple[str, int]]: - """ - Converts a custom dictionary to a list of (str, int) tuples + """Converts a custom dictionary to a list of (str, int) tuples """ if isinstance(custom_dict, dict): custom_dict = list(custom_dict.items()) @@ -121,8 +116,7 @@ def __init__( max_len: int = 40, dict_filter: Callable[[str], bool] | None = _is_thai_and_not_num, ): - """ - Initializes Peter Norvig's spell checker object. + """Initializes Peter Norvig's spell checker object. Spelling dictionary can be customized. By default, spelling dictionary is from `Thai National Corpus `_ @@ -170,8 +164,7 @@ def __init__( self.__WORDS_TOTAL = sum(self.__WORDS.values()) def dictionary(self) -> ItemsView[str, int]: - """ - Returns the spelling dictionary currently used by this spell checker + """Returns the spelling dictionary currently used by this spell checker :return: spelling dictionary of this instance :rtype: list[tuple[str, int]] @@ -190,8 +183,7 @@ def dictionary(self) -> ItemsView[str, int]: return self.__WORDS.items() def known(self, words: Iterable[str]) -> list[str]: - """ - Returns a list of given words found in the spelling dictionary + """Returns a list of given words found in the spelling dictionary :param list[str] words: A list of words to check if they exist in the spelling dictionary @@ -219,8 +211,7 @@ def known(self, words: Iterable[str]) -> list[str]: return list(w for w in words if w in self.__WORDS) def prob(self, word: str) -> float: - """ - Returns the probability of an input word, + """Returns the probability of an input word, according to the spelling dictionary :param str word: A word to check occurrence probability of @@ -247,8 +238,7 @@ def prob(self, word: str) -> float: return self.__WORDS[word] / self.__WORDS_TOTAL def freq(self, word: str) -> int: - """ - Returns the frequency of an input word, + """Returns the frequency of an input word, according to the spelling dictionary :param str word: A word to check frequency of @@ -271,8 +261,7 @@ def freq(self, word: str) -> int: return self.__WORDS[word] def spell(self, word: str) -> list[str]: - """ - Returns a list of all correctly-spelled words whose spelling + """Returns a list of all correctly-spelled words whose spelling is similar to the given word by edit distance metrics. The returned list of words will be sorted by decreasing order of word frequencies in the word spelling dictionary. @@ -321,8 +310,7 @@ def spell(self, word: str) -> list[str]: return candidates def correct(self, word: str) -> str: - """ - Returns the most possible word, using the probability from + """Returns the most possible word, using the probability from the spelling dictionary :param str word: A word to correct spelling of diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index d75b502f3..8ba10fa22 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -symspellpy +"""symspellpy symspellpy is a Python port of SymSpell v6.5. We used unigram & bigram from Thai National Corpus (TNC). diff --git a/pythainlp/spell/tltk.py b/pythainlp/spell/tltk.py index d21381675..225a20c02 100644 --- a/pythainlp/spell/tltk.py +++ b/pythainlp/spell/tltk.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -TLTK +"""TLTK Thai Language Toolkit diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 6cf81f878..116b949e7 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Two-stage Thai Misspelling Correction based on Pre-trained Language Models +"""Two-stage Thai Misspelling Correction based on Pre-trained Language Models :See Also: * Paper: \ diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index 7daa8c26e..92b4467c5 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -9,8 +9,7 @@ class FastTextEncoder: - """ - A class to load pre-trained FastText-like word embeddings, + """A class to load pre-trained FastText-like word embeddings, compute word and sentence vectors, and interact with an ONNX model for nearest neighbor suggestions. """ @@ -27,8 +26,7 @@ def __init__( minn=5, maxn=5, ): - """ - Initializes the FastTextEncoder, loading embeddings, vocabulary, + """Initializes the FastTextEncoder, loading embeddings, vocabulary, nearest neighbor model, and suggestion words list. Args: @@ -39,6 +37,7 @@ def __init__( nb_words (int): The number of words in the vocabulary (used as an offset for subword indices). minn (int): Minimum character length for subwords. maxn (int): Maximum character length for subwords. + """ try: import numpy as np @@ -185,8 +184,7 @@ def get_sentence_vector(self, line): # --- Nearest Neighbor Method --- def get_word_suggestion(self, list_word): - """ - Queries the ONNX model to find the nearest neighbor word(s) + """Queries the ONNX model to find the nearest neighbor word(s) for the given word or list of words. Args: @@ -196,6 +194,7 @@ def get_word_suggestion(self, list_word): Returns: str or list of str: The nearest neighbor word(s) from the pre-loaded suggestion list. + """ if isinstance(list_word, str): input_words = [list_word] @@ -246,8 +245,7 @@ def __init__(self): def get_words_spell_suggestion( list_words: str | list[str], ) -> list[str] | list[list[str]]: - """ - Get words spell suggestion + """Get words spell suggestion The function is designed to retrieve spelling suggestions \ for one or more input Thai words. diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py index b73c8ffff..6294498aa 100644 --- a/pythainlp/summarize/__init__.py +++ b/pythainlp/summarize/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text summarization +"""Text summarization """ __all__ = [ diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index 49cb824da..7a644c184 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text summarization and keyword extraction +"""Text summarization and keyword extraction """ from __future__ import annotations @@ -24,8 +23,7 @@ def summarize( engine: str = DEFAULT_SUMMARIZE_ENGINE, tokenizer: str = "newmm", ) -> list[str]: - """ - This function summarizes text based on frequency of words. + """This function summarizes text based on frequency of words. Under the hood, this function first tokenizes sentences from the given text with :func:`pythainlp.tokenize.sent_tokenize`. @@ -127,8 +125,7 @@ def extract_keywords( tokenizer: str = "newmm", stop_words: Iterable[str] | None = None, ) -> list[str]: - """ - This function returns most-relevant keywords (and/or keyphrases) from the input document. + """This function returns most-relevant keywords (and/or keyphrases) from the input document. Each algorithm may produce completely different keywords from each other, so please be careful when choosing the algorithm. diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index 93653274d..01c15c595 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Summarization by frequency of words +"""Summarization by frequency of words """ from __future__ import annotations diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index f589e1815..efed9e721 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Minimal re-implementation of KeyBERT. +"""Minimal re-implementation of KeyBERT. KeyBERT is a minimal and easy-to-use keyword extraction technique that leverages BERT embeddings to create keywords and keyphrases @@ -44,8 +43,7 @@ def extract_keywords( return_similarity=False, stop_words: Iterable[str] | None = None, ) -> list[str] | list[tuple[str, float]]: - """ - Extract Thai keywords and/or keyphrases with KeyBERT algorithm. + """Extract Thai keywords and/or keyphrases with KeyBERT algorithm. See https://github.com/MaartenGr/KeyBERT. :param str text: text to be summarized @@ -137,8 +135,7 @@ def extract_keywords( return [kw for kw, _ in keywords] def embed(self, docs: str | list[str]) -> np.ndarray: - """ - Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. + """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. """ embs = self.ft_pipeline(docs) if isinstance(docs, str) or len(docs) == 1: diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index ad226b7ab..bce375fea 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Summarization by mT5 model +"""Summarization by mT5 model """ from __future__ import annotations @@ -23,8 +22,7 @@ def __init__( skip_special_tokens: bool = True, pretrained_mt5_model_name: str = "", ): - """ - Initialize mT5 Summarizer. + """Initialize mT5 Summarizer. :param str model_size: Size of the model ("small", "base", "large", "xl", "xxl"). Default is "small". diff --git a/pythainlp/tag/__init__.py b/pythainlp/tag/__init__.py index 891ddd8d2..23b02980c 100644 --- a/pythainlp/tag/__init__.py +++ b/pythainlp/tag/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Linguistic and other taggers. +"""Linguistic and other taggers. Tagging each token in a sentence with supplementary information, such as its part-of-speech (POS) tag, and named entity (NE) tag. diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index bf1fdf685..d815d64ea 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Perceptron Tagger. +"""Perceptron Tagger. This tagger is a port of the Textblob Averaged Perceptron Tagger Author: Matthew Honnibal , @@ -25,8 +24,7 @@ class AveragedPerceptron: - """ - An averaged perceptron, as implemented by Matthew Honnibal. + """An averaged perceptron, as implemented by Matthew Honnibal. See more implementation details here: http://honnibal.wordpress.com/2013/09/11/a-good-part-of-speechpos-tagger-in-about-200-lines-of-python/ @@ -48,8 +46,7 @@ def __init__(self) -> None: self.i = 0 def predict(self, features: dict): - """ - Dot-product the features and current weights and return the best + """Dot-product the features and current weights and return the best label. """ scores = defaultdict(float) @@ -94,8 +91,7 @@ def average_weights(self) -> None: class PerceptronTagger: - """ - Greedy Averaged Perceptron tagger, as implemented by Matthew Honnibal. + """Greedy Averaged Perceptron tagger, as implemented by Matthew Honnibal. See more implementation details here: http://honnibal.wordpress.com/2013/09/11/a-good-part-of-speechpos-tagger-in-about-200-lines-of-python/ @@ -120,8 +116,7 @@ class PerceptronTagger: AP_MODEL_LOC = "" def __init__(self, path: str = "") -> None: - """ - :param str path: model path + """:param str path: model path """ self.model = AveragedPerceptron() self.tagdict = {} @@ -152,8 +147,7 @@ def train( save_loc: str | None = None, nr_iter: int = 5, ) -> None: - """ - Train a model from sentences, and save it at ``save_loc``. + """Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. @@ -200,8 +194,7 @@ def train( json.dump(data, f, ensure_ascii=False) def load(self, loc: str) -> None: - """ - Load a pickled model. + """Load a pickled model. :param str loc: model path """ try: @@ -216,8 +209,7 @@ def load(self, loc: str) -> None: self.model.classes = set(self.classes) def _normalize(self, word: str) -> str: - """ - Normalization used in pre-processing. + """Normalization used in pre-processing. - All words are lower cased - Digits in the range 1800-2100 are represented as !YEAR; @@ -237,8 +229,7 @@ def _normalize(self, word: str) -> str: def _get_features( self, i: int, word: str, context: list[str], prev: str, prev2: str ) -> dict: - """ - Map tokens into a feature representation, implemented as a + """Map tokens into a feature representation, implemented as a {hashable: float} dict. If the features change, a new model must be trained. """ diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index a4b4b8ee9..2a3938f14 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -32,8 +32,7 @@ def pre_process(words: list[str]) -> list[str]: - """ - Convert signs and symbols with their defined strings. + """Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, before the actual POS tagging. """ @@ -45,8 +44,7 @@ def pre_process(words: list[str]) -> list[str]: def post_process( word_tags: list[tuple[str, str]], to_ud: bool = False ) -> list[tuple[str, str]]: - """ - Convert defined strings back to corresponding signs and symbols. + """Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, after the POS tagging. """ diff --git a/pythainlp/tag/chunk.py b/pythainlp/tag/chunk.py index c950636b0..278da56a9 100644 --- a/pythainlp/tag/chunk.py +++ b/pythainlp/tag/chunk.py @@ -7,8 +7,7 @@ def chunk_parse( sent: list[tuple[str, str]], engine: str = "crf", corpus: str = "orchidpp" ) -> list[str]: - """ - This function parses Thai sentence to phrase structure in IOB format. + """This function parses Thai sentence to phrase structure in IOB format. :param list sent: list [(word, part-of-speech)] :param str engine: chunk parse engine (now, it has crf only) diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index c04a8f52a..1e949a817 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -13,8 +13,7 @@ def _is_stopword(word: str) -> bool: # check Thai stopword def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict: - """ - `tokens` = a POS-tagged sentence [(w1, t1), ...] + """`tokens` = a POS-tagged sentence [(w1, t1), ...] `index` = the index of the token we want to extract features for """ word, pos = tokens[index] diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py index f4753e54a..9b2143c0a 100644 --- a/pythainlp/tag/locations.py +++ b/pythainlp/tag/locations.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Recognizes locations in text +"""Recognizes locations in text """ from __future__ import annotations @@ -11,8 +10,7 @@ def tag_provinces(tokens: list[str]) -> list[tuple[str, str]]: - """ - This function recognizes Thailand provinces in text. + """This function recognizes Thailand provinces in text. Note that it uses exact match and considers no context. diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index 4639dafbb..71a0876c8 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -1,16 +1,14 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Named-entity recognizer +"""Named-entity recognizer """ from __future__ import annotations class NER: - """ - Class of named-entity recognizer + """Class of named-entity recognizer :param str engine: engine of named-entity recognizer :param str corpus: corpus @@ -64,8 +62,7 @@ def load_engine(self, engine: str, corpus: str) -> None: def tag( self, text, pos=False, tag=False ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: output with part-of-speech tags.\ @@ -99,8 +96,7 @@ def tag( class NNER: - """ - Nested Named Entity Recognition + """Nested Named Entity Recognition :param str engine: engine of nested named entity recognizer :param str corpus: corpus @@ -118,8 +114,7 @@ def load_engine(self, engine: str = "thai_nner") -> None: self.engine = Thai_NNER() def tag(self, text) -> tuple[list[str], list[dict]]: - """ - This function tags nested named entities. + """This function tags nested named entities. :param str text: text in Thai to be tagged diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 46e0f1005..69fcb0c07 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Data preprocessing for ORCHID corpus +"""Data preprocessing for ORCHID corpus """ from __future__ import annotations @@ -127,8 +126,7 @@ def ud_exception(w: str, tag: str) -> str: def pre_process(words: list[str]) -> list[str]: - """ - Convert signs and symbols with their defined strings. + """Convert signs and symbols with their defined strings. This function is to be used as a preprocessing step, before the actual POS tagging. """ @@ -140,8 +138,7 @@ def pre_process(words: list[str]) -> list[str]: def post_process( word_tags: list[tuple[str, str]], to_ud: bool = False ) -> list[tuple[str, str]]: - """ - Convert defined strings back to corresponding signs and symbols. + """Convert defined strings back to corresponding signs and symbols. This function is to be used as a post-processing step, after the actual POS tagging. """ diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index 2219fdd50..c856a864b 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Perceptron part-of-speech tagger +"""Perceptron part-of-speech tagger """ from __future__ import annotations @@ -70,8 +69,7 @@ def _tud_tagger(): def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: - """ - :param list words: a list of tokenized words + """:param list words: a list of tokenized words :param str corpus: corpus name (orchid, pud) :return: a list of tuples (word, POS tag) :rtype: list[tuple[str, str]] diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index eb1d8e385..6fbc3f2f0 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -7,8 +7,7 @@ def pos_tag( words: list[str], engine: str = "perceptron", corpus: str = "orchid" ) -> list[tuple[str, str]]: - """ - Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'. + """Marks words with part-of-speech (POS) tags, such as 'NOUN' and 'VERB'. :param list words: a list of tokenized words :param str engine: @@ -129,8 +128,7 @@ def pos_tag_sents( engine: str = "perceptron", corpus: str = "orchid", ) -> list[list[tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. + """Marks sentences with part-of-speech (POS) tags. :param list sentences: a list of lists of tokenized words :param str engine: @@ -178,21 +176,26 @@ def pos_tag_transformers( engine: str = "bert", corpus: str = "blackboard", ) -> list[list[tuple[str, str]]]: - """ - Marks sentences with part-of-speech (POS) tags. + """Marks sentences with part-of-speech (POS) tags. :param str sentence: a list of lists of tokenized words :param str engine: * *bert* - BERT: Bidirectional Encoder Representations from Transformers (default) - * *wangchanberta* - fine-tuned version of airesearch/wangchanberta-base-att-spm-uncased on pud corpus (support PUD cotpus only) + * *wangchanberta* - fine-tuned version of \ + airesearch/wangchanberta-base-att-spm-uncased on pud corpus \ + (support PUD cotpus only) * *phayathaibert* - fine-tuned version of clicknext/phayathaibert \ on blackboard corpus (support blackboard cotpus only) - * *mdeberta* - mDeBERTa: Multilingual Decoding-enhanced BERT with disentangled attention (support PUD corpus only) - :param str corpus: the corpus that is used to create the language model for tagger - * *blackboard* - `blackboard treebank (support bert engine only) `_ + * *mdeberta* - mDeBERTa: Multilingual Decoding-enhanced BERT \ + with disentangled attention (support PUD corpus only) + :param str corpus: the corpus that is used to create the language model + for tagger + * *blackboard* - `blackboard treebank (support bert engine only) \ + `_ * *pud* - `Parallel Universal Dependencies (PUD)\ `_ \ - treebanks, natively use Universal POS tags (support wangchanberta and mdeberta engine) + treebanks, natively use Universal POS tags \ + (support wangchanberta and mdeberta engine) :return: a list of lists of tuples (word, POS tag) :rtype: list[list[tuple[str, str]]] @@ -207,7 +210,6 @@ def pos_tag_transformers( # output: # [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] """ - try: from transformers import ( AutoModelForTokenClassification, diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index c432679d2..106cb82b1 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Named-entity recognizer +"""Named-entity recognizer """ from __future__ import annotations @@ -74,8 +73,7 @@ def _doc2features(doc, i) -> dict: class ThaiNameTagger: - """ - Thai named-entity recognizer or Thai NER. + """Thai named-entity recognizer or Thai NER. This function supports Thai NER 1.4 and 1.5 only. :param str version: Thai NER version. It supports Thai NER 1.4 & 1.5. @@ -91,8 +89,7 @@ class ThaiNameTagger: """ def __init__(self, version: str = "1.4") -> None: - """ - Thai named-entity recognizer. + """Thai named-entity recognizer. :param str version: Thai NER version. It's support Thai NER 1.4 & 1.5. @@ -112,8 +109,7 @@ def __init__(self, version: str = "1.4") -> None: def get_ner( self, text: str, pos: bool = True, tag: bool = False ) -> list[tuple[str, str]] | list[tuple[str, str, str]]: - """ - This function tags named-entities in text in IOB format. + """This function tags named-entities in text in IOB format. :param str text: text in Thai to be tagged :param bool pos: To include POS tags in the results (`True`) or diff --git a/pythainlp/tag/tltk.py b/pythainlp/tag/tltk.py index fdb65180d..c8cba01d1 100644 --- a/pythainlp/tag/tltk.py +++ b/pythainlp/tag/tltk.py @@ -28,8 +28,7 @@ def _post_process(text: str) -> str: def get_ner( text: str, pos: bool = True, tag: bool = False ) -> list[tuple[str, str]] | list[tuple[str, str, str]] | str: - """ - Named-entity recognizer from **TLTK** + """Named-entity recognizer from **TLTK** This function tags named-entities in text in IOB format. diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 23537eeec..ea3fa3c43 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unigram Part-Of-Speech tagger +"""Unigram Part-Of-Speech tagger """ from __future__ import annotations @@ -86,8 +85,7 @@ def _find_tag( def tag(words: list[str], corpus: str = "pud") -> list[tuple[str, str]]: - """ - :param list words: a list of tokenized words + """:param list words: a list of tokenized words :param str corpus: corpus name (orchid or pud) :return: a list of tuples (word, POS tag) :rtype: list[tuple[str, str]] diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 865bc0aa9..3f9867571 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Tokenizers at different levels of linguistic analysis. +"""Tokenizers at different levels of linguistic analysis. """ __all__ = [ diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index 8df8f53c0..2d8cadee7 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Utility functions for tokenize module. +"""Utility functions for tokenize module. """ from __future__ import annotations @@ -16,8 +15,7 @@ def apply_postprocessors( segments: list[str], postprocessors: Callable[[list[str]], list[str]] ) -> list[str]: - """ - A list of callables to apply to a raw segmentation result. + """A list of callables to apply to a raw segmentation result. """ for func in postprocessors: segments = func(segments) @@ -26,8 +24,7 @@ def apply_postprocessors( def rejoin_formatted_num(segments: list[str]) -> list[str]: - """ - Rejoin well-known formatted numeric that are over-tokenized. + """Rejoin well-known formatted numeric that are over-tokenized. The formatted numeric are numbers separated by ":", ",", or ".", such as time, decimal numbers, comma-added numbers, and IP addresses. @@ -75,8 +72,7 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: def strip_whitespace(segments: list[str]) -> list[str]: - """ - Strip whitespace(s) off each token and remove whitespace tokens. + """Strip whitespace(s) off each token and remove whitespace tokens. :param List[str] segments: result from word tokenizer :return: a list of tokens :rtype: List[str] diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py index 169c750cd..f7e65cbdd 100644 --- a/pythainlp/tokenize/attacut.py +++ b/pythainlp/tokenize/attacut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai +"""Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai :See Also: * `GitHub repository `_ @@ -30,8 +29,7 @@ def tokenize(self, text: str) -> list[str]: def segment(text: str, model: str = "attacut-sc") -> list[str]: - """ - Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai + """Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai :param str text: text to be tokenized to words :param str model: model of word tokenizer model :return: list of words, tokenized from the text diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py index 74e46b9c0..b958b9438 100644 --- a/pythainlp/tokenize/budoux.py +++ b/pythainlp/tokenize/budoux.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for BudouX tokenizer (https://github.com/google/budoux) +"""Wrapper for BudouX tokenizer (https://github.com/google/budoux) This module provides a small, defensive wrapper around the Python `budoux` package. The wrapper lazy-imports the package so importing diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index cb7d7c211..9259041dc 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Generic functions of tokenizers +"""Generic functions of tokenizers """ from __future__ import annotations @@ -30,8 +29,7 @@ def word_detokenize( segments: list[list[str]] | list[str], output: str = "str" ) -> list[str] | str: - """ - Word detokenizer. + """Word detokenizer. This function will detokenize the list of words in each sentence into text. @@ -103,8 +101,7 @@ def word_tokenize( keep_whitespace: bool = True, join_broken_num: bool = True, ) -> list[str]: - """ - Word tokenizer. + """Word tokenizer. Tokenizes running text into words (list of strings). @@ -366,8 +363,7 @@ def sent_tokenize( engine: str = DEFAULT_SENT_TOKENIZE_ENGINE, keep_whitespace: bool = True, ) -> list[str]: - """ - Sentence tokenizer. + """Sentence tokenizer. Tokenizes running text into "sentences". Supports both string and list of strings. @@ -432,7 +428,6 @@ def sent_tokenize( # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ ', 'และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค'] """ - if not text or not isinstance(text, (str, list)): return [] @@ -534,8 +529,7 @@ def paragraph_tokenize( paragraph_threshold: float = 0.5, style: str = "newline", ) -> list[list[str]]: - """ - Paragraph tokenizer. + """Paragraph tokenizer. Tokenizes text into paragraphs. @@ -602,8 +596,7 @@ def subword_tokenize( engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, ) -> list[str]: - """ - Subword tokenizer for tokenizing text into units smaller than syllables. + """Subword tokenizer for tokenizing text into units smaller than syllables. Tokenizes text into inseparable units of Thai contiguous characters, namely @@ -723,8 +716,7 @@ def syllable_tokenize( engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE, keep_whitespace: bool = True, ) -> list[str]: - """ - Syllable tokenizer + """Syllable tokenizer Tokenizes text into inseparable units of Thai syllables. @@ -755,8 +747,7 @@ def syllable_tokenize( def display_cell_tokenize(text: str) -> list[str]: - """ - Display cell tokenizer. + """Display cell tokenizer. Tokenizes Thai text into display cells without splitting tone marks. @@ -795,8 +786,7 @@ def display_cell_tokenize(text: str) -> list[str]: class Tokenizer: - """ - Tokenizer class for a custom tokenizer. + """Tokenizer class for a custom tokenizer. This class allows users to pre-define custom dictionary along with tokenizer and encapsulate them into one single object. @@ -868,8 +858,7 @@ def __init__( keep_whitespace: bool = True, join_broken_num: bool = True, ): - """ - Initialize tokenizer object. + """Initialize tokenizer object. :param str custom_dict: a file path, a list of vocaburaies* to be used to create a trie, or an instantiated @@ -896,8 +885,7 @@ def __init__( self.__join_broken_num = join_broken_num def word_tokenize(self, text: str) -> list[str]: - """ - Main tokenization function. + """Main tokenization function. :param str text: text to be tokenized :return: list of words, tokenized from the text @@ -912,8 +900,7 @@ def word_tokenize(self, text: str) -> list[str]: ) def set_tokenize_engine(self, engine: str) -> None: - """ - Set the tokenizer's engine. + """Set the tokenizer's engine. :param str engine: choose between different options of tokenizer engines (i.e. *newmm*, *mm*, *longest*, *deepcut*) diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 19bf255e6..b4cb9e0be 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -CRFCut - Thai sentence segmenter. +"""CRFCut - Thai sentence segmenter. Thai sentence segmentation using conditional random field, with default model trained on TED dataset @@ -130,8 +129,7 @@ def extract_features( doc: list[str], window: int = 2, max_n_gram: int = 3 ) -> list[list[str]]: - """ - Extract features for CRF by sliding `max_n_gram` of tokens + """Extract features for CRF by sliding `max_n_gram` of tokens for +/- `window` from the current token :param List[str] doc: tokens from which features are to be extracted @@ -187,8 +185,7 @@ def extract_features( def segment(text: str) -> list[str]: - """ - CRF-based sentence segmentation. + """CRF-based sentence segmentation. :param str text: text to be tokenized into sentences :return: list of words, tokenized from the text diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py index 0087fa10f..6e677c241 100644 --- a/pythainlp/tokenize/deepcut.py +++ b/pythainlp/tokenize/deepcut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for deepcut Thai word segmentation. deepcut is a +"""Wrapper for deepcut Thai word segmentation. deepcut is a Thai word segmentation library using 1D Convolution Neural Network. User need to install deepcut (and its dependency: tensorflow) by themselves. diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index cb0563ab1..9e97f92bf 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Segmenting text into Enhanced Thai Character Clusters (ETCCs) +"""Segmenting text into Enhanced Thai Character Clusters (ETCCs) Python implementation by Wannaphong Phatthiyaphaibun This implementation relies on a dictionary of ETCC created from etcc.txt @@ -54,8 +53,7 @@ def _cut_subword(tokens: list[str]) -> list[str]: def segment(text: str) -> list[str]: - """ - Segmenting text into ETCCs. + """Segmenting text into ETCCs. Enhanced Thai Character Cluster (ETCC) is a kind of subword unit. The concept was presented in Inrut, Jeeragone, Patiroop Yuanghirun, @@ -68,7 +66,6 @@ def segment(text: str) -> list[str]: :return: list of clusters, tokenized from the text :return: List[str] """ - if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index eacfb1494..4c11ce684 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2019 Ponrawee Prasertsom # SPDX-License-Identifier: Apache-2.0 -""" -🪿 Han-solo: Thai syllable segmenter +"""🪿 Han-solo: Thai syllable segmenter GitHub: https://github.com/PyThaiNLP/Han-solo """ diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index b777ca126..1059cd976 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Dictionary-based longest-matching Thai word segmentation. Implementation is based +"""Dictionary-based longest-matching Thai word segmentation. Implementation is based on the codes from Patorn Utenpattanun. :See Also: @@ -158,8 +157,7 @@ def tokenize(self, text: str) -> list[str]: def segment(text: str, custom_dict: Trie | None = None) -> list[str]: - """ - Dictionary-based longest matching word segmentation. + """Dictionary-based longest matching word segmentation. :param str text: text to be tokenized into words :param pythainlp.util.Trie custom_dict: dictionary for tokenization diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index d6f9c7840..3f88f5b6a 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Multi cut -- Thai word segmentation with maximum matching. +"""Multi cut -- Thai word segmentation with maximum matching. Original codes from Korakot Chaovavanich. :See Also: diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py index 75e42c17f..50f33d8ba 100644 --- a/pythainlp/tokenize/nercut.py +++ b/pythainlp/tokenize/nercut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -nercut 0.2 +"""nercut 0.2 Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are @@ -32,8 +31,7 @@ def segment( ], tagger=_thainer, ) -> list[str]: - """ - Dictionary-based maximal matching word segmentation, constrained by + """Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are parts of the same named-entity. diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 58dd70080..2bcbfa3be 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Dictionary-based maximal matching word segmentation, constrained by +"""Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries with improved rules. The codes are based on the notebooks created by Korakot Chaovavanich, diff --git a/pythainlp/tokenize/oskut.py b/pythainlp/tokenize/oskut.py index 58cb96344..82701f36d 100644 --- a/pythainlp/tokenize/oskut.py +++ b/pythainlp/tokenize/oskut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper OSKut (Out-of-domain StacKed cut for Word Segmentation). +"""Wrapper OSKut (Out-of-domain StacKed cut for Word Segmentation). Handling Cross- and Out-of-Domain Samples in Thai Word Segmentation Stacked Ensemble Framework and DeepCut as Baseline model (ACL 2021 Findings) diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py index 217577a64..1b8c52f58 100644 --- a/pythainlp/tokenize/pyicu.py +++ b/pythainlp/tokenize/pyicu.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for PyICU word segmentation. This wrapper module uses +"""Wrapper for PyICU word segmentation. This wrapper module uses :class:`icu.BreakIterator` with Thai as :class:`icu.Local` to locate boundaries between words in the text. @@ -29,8 +28,7 @@ def _gen_words(text: str) -> str: def segment(text: str) -> list[str]: - """ - :param str text: text to be tokenized into words + """:param str text: text to be tokenized into words :return: list of words, tokenized from the text """ if not text or not isinstance(text, str): diff --git a/pythainlp/tokenize/sefr_cut.py b/pythainlp/tokenize/sefr_cut.py index 7a123e541..3381fa692 100644 --- a/pythainlp/tokenize/sefr_cut.py +++ b/pythainlp/tokenize/sefr_cut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Wrapper for SEFR CUT Thai word segmentation. SEFR CUT is a +"""Wrapper for SEFR CUT Thai word segmentation. SEFR CUT is a Thai Word Segmentation Models using Stacked Ensemble. :See Also: diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py index e7abeabc3..ccaa56550 100644 --- a/pythainlp/tokenize/ssg.py +++ b/pythainlp/tokenize/ssg.py @@ -7,8 +7,7 @@ def segment(text: str) -> list[str]: - """ - Syllable tokenizer using ssg + """Syllable tokenizer using ssg """ if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py index 786050397..b23fd0586 100644 --- a/pythainlp/tokenize/tcc.py +++ b/pythainlp/tokenize/tcc.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of tokenizer according to Thai Character Clusters (TCCs) +"""The implementation of tokenizer according to Thai Character Clusters (TCCs) rules proposed by `Theeramunkong et al. 2000. \ `_ @@ -60,8 +59,7 @@ def tcc(text: str) -> str: - """ - TCC generator which generates Thai Character Clusters + """TCC generator which generates Thai Character Clusters :param str text: text to be tokenized into character clusters :return: subwords (character clusters) @@ -83,8 +81,7 @@ def tcc(text: str) -> str: def tcc_pos(text: str) -> set[int]: - """ - TCC positions + """TCC positions :param str text: text to be tokenized into character clusters :return: list of the ending position of subwords @@ -103,13 +100,11 @@ def tcc_pos(text: str) -> set[int]: def segment(text: str) -> list[str]: - """ - Subword segmentation + """Subword segmentation :param str text: text to be tokenized into character clusters :return: list of subwords (character clusters), tokenized from the text :rtype: list[str] """ - return list(tcc(text)) diff --git a/pythainlp/tokenize/tcc_p.py b/pythainlp/tokenize/tcc_p.py index 1d7fd7325..80199475b 100644 --- a/pythainlp/tokenize/tcc_p.py +++ b/pythainlp/tokenize/tcc_p.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of tokenizer according to Thai Character Clusters (TCCs) +"""The implementation of tokenizer according to Thai Character Clusters (TCCs) rules proposed by `Theeramunkong et al. 2000. \ `_ and improved rules that are used in newmm @@ -60,8 +59,7 @@ def tcc(text: str) -> str: - """ - TCC generator which generates Thai Character Clusters + """TCC generator which generates Thai Character Clusters :param str text: text to be tokenized into character clusters :return: subwords (character clusters) @@ -83,8 +81,7 @@ def tcc(text: str) -> str: def tcc_pos(text: str) -> set[int]: - """ - TCC positions + """TCC positions :param str text: text to be tokenized into character clusters :return: list of the ending position of subwords @@ -103,13 +100,11 @@ def tcc_pos(text: str) -> set[int]: def segment(text: str) -> list[str]: - """ - Subword segmentation + """Subword segmentation :param str text: text to be tokenized into character clusters :return: list of subwords (character clusters), tokenized from the text :rtype: list[str] """ - return list(tcc(text)) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 15bc321e7..35dda6ec0 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileCopyrightText: Copyright 2020 Nakhun Chumpolsathien # SPDX-License-Identifier: Apache-2.0 -""" -The implementation of sentence segmentator from Nakhun Chumpolsathien, 2020 +"""The implementation of sentence segmentator from Nakhun Chumpolsathien, 2020 original codes are from: https://github.com/nakhunchumpolsathien/ThaiSum Cite: diff --git a/pythainlp/tokenize/wtsplit.py b/pythainlp/tokenize/wtsplit.py index 80ff22800..c4d688c33 100644 --- a/pythainlp/tokenize/wtsplit.py +++ b/pythainlp/tokenize/wtsplit.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Where's the Point? Self-Supervised Multilingual Punctuation-Agnostic Sentence Segmentation +"""Where's the Point? Self-Supervised Multilingual Punctuation-Agnostic Sentence Segmentation GitHub: https://github.com/bminixhofer/wtpsplit """ diff --git a/pythainlp/tools/core.py b/pythainlp/tools/core.py index 62997e743..c4fc9e6c4 100644 --- a/pythainlp/tools/core.py +++ b/pythainlp/tools/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Generic support functions for PyThaiNLP. +"""Generic support functions for PyThaiNLP. """ from __future__ import annotations diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index 92aebb7a1..89c445eae 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -105,8 +105,7 @@ def find_misspell_candidates(char: str, verbose: bool = False): def misspell(sentence: str, ratio: float = 0.05): - """ - Simulate some misspellings of the input sentence. + """Simulate some misspellings of the input sentence. The number of misspelled locations is governed by ratio. :params str sentence: sentence to be misspelled diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index 49deb97af..9af51e60a 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PyThaiNLP data tools +"""PyThaiNLP data tools For text processing and text conversion, see pythainlp.util """ @@ -17,8 +16,7 @@ def get_full_data_path(path: str) -> str: - """ - This function joins path of :mod:`pythainlp` data directory and the + """This function joins path of :mod:`pythainlp` data directory and the given path, and returns the full path. :return: full path given the name of dataset @@ -36,8 +34,7 @@ def get_full_data_path(path: str) -> str: def get_pythainlp_data_path() -> str: - """ - Returns the full path where PyThaiNLP keeps its (downloaded) data. + """Returns the full path where PyThaiNLP keeps its (downloaded) data. If the directory does not yet exist, it will be created. The path can be specified through the environment variable :envvar:`PYTHAINLP_DATA_DIR`. By default, `~/pythainlp-data` @@ -63,8 +60,7 @@ def get_pythainlp_data_path() -> str: def get_pythainlp_path() -> str: - """ - This function returns full path of PyThaiNLP codes + """This function returns full path of PyThaiNLP codes :return: full path of :mod:`pythainlp` codes :rtype: str diff --git a/pythainlp/translate/__init__.py b/pythainlp/translate/__init__.py index 410d0d40b..f126f56db 100644 --- a/pythainlp/translate/__init__.py +++ b/pythainlp/translate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Language translation. +"""Language translation. """ __all__ = ["Translate", "ThZhTranslator", "ZhThTranslator", "word_translate"] diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index 493fffdbd..d49cbb289 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -5,8 +5,7 @@ class Translate: - """ - Machine Translation + """Machine Translation """ def __init__( @@ -16,8 +15,7 @@ def __init__( engine: str = "default", use_gpu: bool = False, ) -> None: - """ - :param str src_lang: source language + """:param str src_lang: source language :param str target_lang: target language :param str engine: machine translation engine :param bool use_gpu: load model using GPU (Default is False) @@ -85,8 +83,7 @@ def load_model(self): raise ValueError("Not support language!") def translate(self, text: str) -> str: - """ - Translate text + """Translate text :param str text: input text in source language :return: translated text in target language @@ -100,8 +97,7 @@ def translate(self, text: str) -> str: def word_translate( word: str, src: str, target: str, engine: str = "word2word" ) -> list[str] | None: - """ - Translate word from source language to target language. + """Translate word from source language to target language. :param str word: text :param str src: src language diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index e13e9264a..2a734b4dd 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -English-Thai Machine Translation +"""English-Thai Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute @@ -48,16 +47,14 @@ def _download_install(name: str) -> None: def download_model_all() -> None: - """ - Download all translation models in advance + """Download all translation models in advance """ _download_install(_EN_TH_MODEL_NAME) _download_install(_TH_EN_MODEL_NAME) class EnThTranslator: - """ - English-Thai Machine Translation + """English-Thai Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute @@ -89,8 +86,7 @@ def __init__(self, use_gpu: bool = False): self._model = self._model.cuda() def translate(self, text: str) -> str: - """ - Translate text from English to Thai + """Translate text from English to Thai :param str text: input text in source language :return: translated text in target language @@ -114,8 +110,7 @@ def translate(self, text: str) -> str: class ThEnTranslator: - """ - Thai-English Machine Translation + """Thai-English Machine Translation from VISTEC-depa Thailand Artificial Intelligence Research Institute @@ -152,8 +147,7 @@ def __init__(self, use_gpu: bool = False): self._model.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to English + """Translate text from Thai to English :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index da00ef6ba..b774b9a50 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -9,8 +9,7 @@ class Small100Translator: - """ - Machine Translation using small100 model + """Machine Translation using small100 model - Huggingface https://huggingface.co/alirezamsh/small100 @@ -31,8 +30,7 @@ def __init__( self.model = self.model.cuda() def translate(self, text: str, tgt_lang: str = "en") -> str: - """ - Translate text from X to X + """Translate text from X to X :param str text: input text in source language :param str tgt_lang: target language diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 44db1faa0..a408c7896 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai-French Machine Translation +"""Thai-French Machine Translation Trained by OPUS Corpus @@ -17,8 +16,7 @@ class ThFrTranslator: - """ - Thai-French Machine Translation + """Thai-French Machine Translation Trained by OPUS Corpus @@ -44,8 +42,7 @@ def __init__( self.model_thzh = self.model_thzh.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to French + """Translate text from Thai to French :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index e7d32e6c0..96d641179 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -64,10 +64,10 @@ class SMALL100Tokenizer(PreTrainedTokenizer): - """ - Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). + """Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. + Args: vocab_file (`str`): Path to the vocabulary file. @@ -101,6 +101,7 @@ class SMALL100Tokenizer(PreTrainedTokenizer): using forward-filtering-and-backward-sampling algorithm. - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. + Examples: ```python >>> from tokenization_small100 import SMALL100Tokenizer @@ -109,7 +110,9 @@ class SMALL100Tokenizer(PreTrainedTokenizer): >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria" >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt") >>> model(**model_inputs) # should work - ```""" + ``` + + """ vocab_files_names = VOCAB_FILES_NAMES max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES @@ -134,15 +137,12 @@ def __init__( num_madeup_words=8, **kwargs, ) -> None: - self.sp_model_kwargs = ( - {} if sp_model_kwargs is None else sp_model_kwargs - ) + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs self.language_codes = language_codes fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes] self.lang_code_to_token = { - lang_code: f"__{lang_code}__" - for lang_code in fairseq_language_code + lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code } kwargs["additional_special_tokens"] = kwargs.get( @@ -151,8 +151,7 @@ def __init__( kwargs["additional_special_tokens"] += [ self.get_lang_token(lang_code) for lang_code in fairseq_language_code - if self.get_lang_token(lang_code) - not in kwargs["additional_special_tokens"] + if self.get_lang_token(lang_code) not in kwargs["additional_special_tokens"] ] super().__init__( @@ -184,9 +183,7 @@ def __init__( lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code) } - self.id_to_lang_token = { - v: k for k, v in self.lang_token_to_id.items() - } + self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()} self._tgt_lang = tgt_lang if tgt_lang is not None else "en" self.cur_lang_id = self.get_lang_id(self._tgt_lang) @@ -196,11 +193,7 @@ def __init__( @property def vocab_size(self) -> int: - return ( - len(self.encoder) - + len(self.lang_token_to_id) - + self.num_madeup_words - ) + return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words @property def tgt_lang(self) -> str: @@ -235,20 +228,24 @@ def get_special_tokens_mask( token_ids_1: list[int] | None = None, already_has_special_tokens: bool = False, ) -> list[int]: - """ - Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding - special tokens using the tokenizer `prepare_for_model` method. + """Retrieve sequence IDs from a token list that has no special tokens + added. This method is called when adding special tokens using the + tokenizer `prepare_for_model` method. + Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not the token list is already formatted with special tokens for the model. + Whether or not the token list is already formatted with + special tokens for the model. + Returns: - `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. - """ + `List[int]`: A list of integers in the range [0, 1]: + 1 for a special token, 0 for a sequence token. + """ if already_has_special_tokens: return super().get_special_tokens_mask( token_ids_0=token_ids_0, @@ -270,41 +267,41 @@ def get_special_tokens_mask( def build_inputs_with_special_tokens( self, token_ids_0: list[int], token_ids_1: list[int] | None = None ) -> list[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. An MBART sequence has the following format, where `X` represents the sequence: + """Build model inputs from a sequence or a pair of sequence for + sequence classification tasks by concatenating and + adding special tokens. An MBART sequence has the following format, + where `X` represents the sequence: - `input_ids` (for encoder) `X [eos, src_lang_code]` - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]` - BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a - separator. + + BOS is never used. Pairs of sequences are not the expected use case, + but they will be handled without aseparator. + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. + Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + `List[int]`: List of [input IDs](../glossary#input-ids) with the + appropriate special tokens. + """ if token_ids_1 is None: if self.prefix_tokens is None: return token_ids_0 + self.suffix_tokens else: return self.prefix_tokens + token_ids_0 + self.suffix_tokens - # We don't expect to process pairs, but leave the pair logic for API consistency + # We don't expect to process pairs, + # but leave the pair logic for API consistency if self.prefix_tokens is None: return token_ids_0 + token_ids_1 + self.suffix_tokens else: - return ( - self.prefix_tokens - + token_ids_0 - + token_ids_1 - + self.suffix_tokens - ) + return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens def get_vocab(self) -> dict: - vocab = { - self.convert_ids_to_tokens(i): i for i in range(self.vocab_size) - } + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} vocab.update(self.added_tokens_encoder) return vocab @@ -364,11 +361,10 @@ def prepare_seq2seq_batch( def _build_translation_inputs( self, raw_inputs, tgt_lang: str | None, **extra_kwargs ): - """Used by translation pipeline, to prepare inputs for the generate function""" + """Used by translation pipeline, to prepare inputs for the generate + function""" if tgt_lang is None: - raise ValueError( - "Translation requires a `tgt_lang` for this model" - ) + raise ValueError("Translation requires a `tgt_lang` for this model") self.tgt_lang = tgt_lang inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs) return inputs @@ -381,7 +377,8 @@ def _switch_to_target_mode(self): self.suffix_tokens = [self.eos_token_id] def set_lang_special_tokens(self, src_lang: str) -> None: - """Reset the special tokens to the tgt lang setting. No prefix and suffix=[eos, tgt_lang_code].""" + """Reset the special tokens to the tgt lang setting. + No prefix and suffix=[eos, tgt_lang_code].""" lang_token = self.get_lang_token(src_lang) self.cur_lang_id = self.lang_token_to_id[lang_token] self.prefix_tokens = [self.cur_lang_id] diff --git a/pythainlp/translate/word2word_translate.py b/pythainlp/translate/word2word_translate.py index 1c603dc71..87bd7aca6 100644 --- a/pythainlp/translate/word2word_translate.py +++ b/pythainlp/translate/word2word_translate.py @@ -72,8 +72,7 @@ def translate(word: str, src: str, target: str) -> list[str] | None: - """ - Word translate + """Word translate :param str word: text :param str src: src language diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 2cf5c056b..84a3b75b5 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Lalita Chinese-Thai Machine Translation +"""Lalita Chinese-Thai Machine Translation from AI builder @@ -14,8 +13,7 @@ class ThZhTranslator: - """ - Thai-Chinese Machine Translation + """Thai-Chinese Machine Translation from Lalita @ AI builder @@ -38,8 +36,7 @@ def __init__( self.model_thzh = self.model_thzh.cuda() def translate(self, text: str) -> str: - """ - Translate text from Thai to Chinese + """Translate text from Thai to Chinese :param str text: input text in source language :return: translated text in target language @@ -67,8 +64,7 @@ def translate(self, text: str) -> str: class ZhThTranslator: - """ - Chinese-Thai Machine Translation + """Chinese-Thai Machine Translation from Lalita @ AI builder @@ -89,8 +85,7 @@ def __init__( self.model_zhth.cuda() def translate(self, text: str) -> str: - """ - Translate text from Chinese to Thai + """Translate text from Chinese to Thai :param str text: input text in source language :return: translated text in target language diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 466dd4d3d..bee56ad24 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliteration. +"""Transliteration. """ __all__ = [ diff --git a/pythainlp/transliterate/core.py b/pythainlp/transliterate/core.py index df96a5253..112d0bd4d 100644 --- a/pythainlp/transliterate/core.py +++ b/pythainlp/transliterate/core.py @@ -13,8 +13,7 @@ def romanize( engine: str = DEFAULT_ROMANIZE_ENGINE, fallback_engine: str = DEFAULT_ROMANIZE_ENGINE, ) -> str: - """ - This function renders Thai word in the Latin alphabet or "romanization", + """This function renders Thai word in the Latin alphabet or "romanization", using the Royal Thai General System of Transcription (RTGS) [#rtgs_transcription]_. RTGS is the official system published by the Royal Institute of Thailand. (Thai: ถอดเสียงภาษาไทยเป็นอักษรละติน) @@ -22,9 +21,11 @@ def romanize( :param str text: A Thai word to be romanized. \ The input should not include whitespace because \ the function is support subwords by spliting whitespace. - :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx, 'tltk', and 'lookup'. See more in options for engine section. - :param str fallback_engine: If engine equals 'lookup', use `fallback_engine` for words that are not in the transliteration dict. - No effect on other engines. Default to 'royin'. + :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx, + 'tltk', and 'lookup'. See more in options for engine section. + :param str fallback_engine: If engine equals 'lookup', + use `fallback_engine` for words that are not in the lookup dictionary. + No effect on other engines. Default to 'royin'. :return: A string of a Thai word rendered in the Latin alphabet. :rtype: str @@ -101,8 +102,7 @@ def select_romanize_engine(engine: str): def transliterate( text: str, engine: str = DEFAULT_TRANSLITERATE_ENGINE ) -> str: - """ - This function transliterates Thai text. + """This function transliterates Thai text. :param str text: Thai text to be transliterated :param str engine: 'icu', 'ipa', or 'thaig2p' (default) @@ -161,7 +161,6 @@ def transliterate( transliterate("ภาพยนตร์", engine="iso_11940") # output: 'p̣hāphyntr' """ - if not text or not isinstance(text, str): return "" @@ -186,8 +185,7 @@ def transliterate( def pronunciate(word: str, engine: str = DEFAULT_PRONUNCIATE_ENGINE) -> str: - """ - This function pronunciates Thai word. + """This function pronunciates Thai word. :param str word: Thai text to be pronunciated :param str engine: 'w2p' (default) diff --git a/pythainlp/transliterate/ipa.py b/pythainlp/transliterate/ipa.py index b1c6b6fa7..7432347f2 100644 --- a/pythainlp/transliterate/ipa.py +++ b/pythainlp/transliterate/ipa.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating text to International Phonetic Alphabet (IPA) +"""Transliterating text to International Phonetic Alphabet (IPA) Using epitran :See Also: diff --git a/pythainlp/transliterate/iso_11940.py b/pythainlp/transliterate/iso_11940.py index 326827f32..0141e9a2f 100644 --- a/pythainlp/transliterate/iso_11940.py +++ b/pythainlp/transliterate/iso_11940.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating Thai text using ISO 11940 +"""Transliterating Thai text using ISO 11940 :See Also: * `Wikipedia \ @@ -133,8 +132,7 @@ def transliterate(word: str) -> str: - """ - Use ISO 11940 for transliteration + """Use ISO 11940 for transliteration :param str text: Thai text to be transliterated. :return: A string indicating how the text should be pronounced, according to ISO 11940. """ diff --git a/pythainlp/transliterate/lookup.py b/pythainlp/transliterate/lookup.py index 5b89228bb..bc1e248dc 100644 --- a/pythainlp/transliterate/lookup.py +++ b/pythainlp/transliterate/lookup.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Look up romanized Thai words in a predefined dictionary compiled by Wannaphong, 2022. +"""Look up romanized Thai words in a predefined dictionary compiled by Wannaphong, 2022. Wannaphong Phatthiyaphaibun. (2022). wannaphong/thai-english-transliteration-dictionary: v1.4 (v1.4). @@ -23,8 +22,7 @@ def follow_rtgs(text: str) -> bool | None: - """ - Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). + """Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). :param str text: Text to look up. Must be a self-contained word. :return: True if text follows the definition by RTGS, False otherwise. `None` means unverified or unknown word. @@ -41,8 +39,7 @@ def follow_rtgs(text: str) -> bool | None: def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """ - Romanize one word. Look up first, call `fallback_func` if not found. + """Romanize one word. Look up first, call `fallback_func` if not found. """ try: # try to get 0-th idx of look up result, simply ignore other possible variations. @@ -57,8 +54,7 @@ def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: def romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """ - Render Thai words in Latin alphabet by looking up + """Render Thai words in Latin alphabet by looking up Thai-English transliteration dictionary. :param str text: Thai text to be romanized diff --git a/pythainlp/transliterate/pyicu.py b/pythainlp/transliterate/pyicu.py index c0f330116..4dc431e85 100644 --- a/pythainlp/transliterate/pyicu.py +++ b/pythainlp/transliterate/pyicu.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating text to International Phonetic Alphabet (IPA) +"""Transliterating text to International Phonetic Alphabet (IPA) Using International Components for Unicode (ICU) :See Also: @@ -18,8 +17,7 @@ def transliterate(text: str) -> str: - """ - Use ICU (International Components for Unicode) for transliteration + """Use ICU (International Components for Unicode) for transliteration :param str text: Thai text to be transliterated. :return: A string of Internaitonal Phonetic Alphabets indicating how the text should be pronounced. """ diff --git a/pythainlp/transliterate/royin.py b/pythainlp/transliterate/royin.py index 74e592ca0..d8b9e02ba 100644 --- a/pythainlp/transliterate/royin.py +++ b/pythainlp/transliterate/royin.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -The Royal Thai General System of Transcription (RTGS) +"""The Royal Thai General System of Transcription (RTGS) is the official system for rendering Thai words in the Latin alphabet. It was published by the Royal Institute of Thailand. @@ -138,8 +137,7 @@ def _normalize(word: str) -> str: - """ - Remove silence, no sound, and tonal characters. + """Remove silence, no sound, and tonal characters. ตัดอักษรที่ไม่ออกเสียง (การันต์ ไปยาลน้อย ไม้ยมก*) และวรรณยุกต์ทิ้ง """ @@ -291,8 +289,7 @@ def _romanize(word: str) -> str: def _should_add_syllable_separator( prev_word: str, curr_word: str, prev_romanized: str ) -> bool: - """ - Determine if 'a' should be added between two romanized syllables. + """Determine if 'a' should be added between two romanized syllables. This applies when: - Previous word has explicit vowel and ends with consonant diff --git a/pythainlp/transliterate/spoonerism.py b/pythainlp/transliterate/spoonerism.py index cc9a55d30..ad57eac4a 100644 --- a/pythainlp/transliterate/spoonerism.py +++ b/pythainlp/transliterate/spoonerism.py @@ -10,8 +10,7 @@ def puan(word: str, show_pronunciation: bool = True) -> str: - """ - Thai Spoonerism + """Thai Spoonerism This function converts Thai word to spoonerism word. diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index f4d55f48a..d7a2d6b0e 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Romanization of Thai words based on machine-learnt engine ("thai2rom") +"""Romanization of Thai words based on machine-learnt engine ("thai2rom") """ from __future__ import annotations @@ -22,8 +21,7 @@ class ThaiTransliterator: def __init__(self): - """ - Transliteration of Thai words. + """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ @@ -62,8 +60,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for PyTorch + """Prepare input sequence for PyTorch """ idxs = [] for ch in text: @@ -76,8 +73,7 @@ def _prepare_sequence_in(self, text: str): return tensor.to(device) def romanize(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ @@ -238,7 +234,6 @@ def __init__( def forward(self, input_character, last_hidden, encoder_outputs, mask): """Defines the forward computation of the decoder""" - # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) # encoder_outputs: (batch_size, sequence_len, hidden_dim) diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index 822129c6f..19ebb0468 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") +"""Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") """ from __future__ import annotations @@ -21,8 +20,7 @@ class ThaiTransliterator_ONNX: def __init__(self): - """ - Transliteration of Thai words. + """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ @@ -60,8 +58,7 @@ def __init__(self): ) def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for ONNX + """Prepare input sequence for ONNX """ idxs = [] for ch in text: @@ -73,8 +70,7 @@ def _prepare_sequence_in(self, text: str): return np.array(idxs) def romanize(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index 215b33328..dea1f3306 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Grapheme-to-Phoneme (Thai G2P) +"""Thai Grapheme-to-Phoneme (Thai G2P) GitHub : https://github.com/wannaphong/thai-g2p """ @@ -23,8 +22,7 @@ class ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self): @@ -63,8 +61,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """ - Prepare input sequence for PyTorch. + """Prepare input sequence for PyTorch. """ idxs = [] for ch in text: @@ -77,8 +74,7 @@ def _prepare_sequence_in(self, text: str): return tensor.to(device) def g2p(self, text: str) -> str: - """ - :param str text: Thai text to be romanized + """:param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. """ @@ -242,7 +238,6 @@ def __init__( def forward(self, input_character, last_hidden, encoder_outputs, mask): """ "Defines the forward computation of the decoder""" - # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) # encoder_outputs: (batch_size, sequence_len, hidden_dim) diff --git a/pythainlp/transliterate/thaig2p_v2.py b/pythainlp/transliterate/thaig2p_v2.py index b9d4ba617..aad18c0d4 100644 --- a/pythainlp/transliterate/thaig2p_v2.py +++ b/pythainlp/transliterate/thaig2p_v2.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Grapheme-to-Phoneme (Thai G2P) +"""Thai Grapheme-to-Phoneme (Thai G2P) huggingface: https://huggingface.co/pythainlp/thaig2p-v2.0 """ @@ -14,8 +13,7 @@ class ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self, device: str = "cpu"): diff --git a/pythainlp/transliterate/tltk.py b/pythainlp/transliterate/tltk.py index bbeb7ba9c..6f84d8613 100644 --- a/pythainlp/transliterate/tltk.py +++ b/pythainlp/transliterate/tltk.py @@ -12,8 +12,7 @@ def romanize(text: str) -> str: - """ - Transliterating thai text to the Latin alphabet using tltk. + """Transliterating thai text to the Latin alphabet using tltk. :param str text: Thai text to be romanized :return: A string of Thai words rendered in the Latin alphabet. diff --git a/pythainlp/transliterate/umt5_thaig2p.py b/pythainlp/transliterate/umt5_thaig2p.py index 868d4a8f4..4976fe547 100644 --- a/pythainlp/transliterate/umt5_thaig2p.py +++ b/pythainlp/transliterate/umt5_thaig2p.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -umt5-thai-g2p-v2-0.5k +"""umt5-thai-g2p-v2-0.5k huggingface: https://huggingface.co/B-K/umt5-thai-g2p-v2-0.5k """ @@ -14,8 +13,7 @@ class Umt5ThaiG2P: - """ - Latin transliteration of Thai words, using International Phonetic Alphabet + """Latin transliteration of Thai words, using International Phonetic Alphabet """ def __init__(self, device: str = "cpu"): diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 3880e76e8..5f5a9edfa 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Word-to-Phoneme (Thai W2P) +"""Thai Word-to-Phoneme (Thai W2P) GitHub : https://github.com/wannaphong/Thai_W2P """ @@ -194,8 +193,7 @@ def __call__(self, word: str) -> str: def pronunciate(text: str) -> str: - """ - Convert a Thai word to its pronunciation in Thai letters. + """Convert a Thai word to its pronunciation in Thai letters. Input should be one single word. diff --git a/pythainlp/transliterate/wunsen.py b/pythainlp/transliterate/wunsen.py index 75b98c5c6..ae503cd6d 100644 --- a/pythainlp/transliterate/wunsen.py +++ b/pythainlp/transliterate/wunsen.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text +"""Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text to Thai text By Wunsen @@ -17,8 +16,7 @@ class WunsenTransliterate: - """ - Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text + """Transliterating Japanese/Korean/Mandarin/Vietnamese romanization text to Thai text by Wunsen @@ -42,8 +40,7 @@ def transliterate( zh_sandhi: bool | None = None, system: str | None = None, ): - """ - Use Wunsen for transliteration + """Use Wunsen for transliteration :param str text: text to be transliterated to Thai text. :param str lang: source language diff --git a/pythainlp/ulmfit/__init__.py b/pythainlp/ulmfit/__init__.py index 307843ce5..ce9757727 100644 --- a/pythainlp/ulmfit/__init__.py +++ b/pythainlp/ulmfit/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Universal Language Model Fine-tuning for Text Classification (ULMFiT). +"""Universal Language Model Fine-tuning for Text Classification (ULMFiT). Code by Charin Polpanumas https://github.com/cstorm125/thai2fit/ diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index d30f34cd9..72d314b7a 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Universal Language Model Fine-tuning for Text Classification (ULMFiT). +"""Universal Language Model Fine-tuning for Text Classification (ULMFiT). """ from __future__ import annotations @@ -72,8 +71,7 @@ def process_thai( tok_func: Callable | None = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: - """ - Process Thai texts for models (with sparse features as default) + """Process Thai texts for models (with sparse features as default) :param str text: text to be cleaned :param list[func] pre_rules: rules to apply before tokenization. @@ -147,8 +145,7 @@ def process_thai( def document_vector(text: str, learn, data, agg: str = "mean"): - """ - This function vectorizes Thai input text into a 400 dimension vector using + """This function vectorizes Thai input text into a 400 dimension vector using :class:`fastai` language model and data bunch. :meth: `document_vector` get document vector using fastai language model @@ -187,7 +184,6 @@ def document_vector(text: str, learn, data, agg: str = "mean"): `_ """ - s = thai2fit_tokenizer().word_tokenize(text) t = torch.tensor(data.vocab.numericalize(s), requires_grad=False).to( device @@ -205,8 +201,7 @@ def document_vector(text: str, learn, data, agg: str = "mean"): def merge_wgts(em_sz, wgts, itos_pre, itos_new): - """ - This function is to insert new vocab into an existing model named `wgts` + """This function is to insert new vocab into an existing model named `wgts` and update the model's weights for new vocab with the average embedding. :meth: `merge_wgts` insert pretrained weights and vocab into a new set diff --git a/pythainlp/ulmfit/preprocess.py b/pythainlp/ulmfit/preprocess.py index 82b94a6c1..99f21c22c 100644 --- a/pythainlp/ulmfit/preprocess.py +++ b/pythainlp/ulmfit/preprocess.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Preprocessing for ULMFiT +"""Preprocessing for ULMFiT """ from __future__ import annotations @@ -21,8 +20,7 @@ def replace_url(text: str) -> str: - """ - Replace URL in `text` with TK_URL + """Replace URL in `text` with TK_URL :param str text: text to replace URL in @@ -40,8 +38,7 @@ def replace_url(text: str) -> str: def fix_html(text: str) -> str: - """ - Replace HTML strings in `test`. (codes from `fastai`) + """Replace HTML strings in `test`. (codes from `fastai`) :param str text: text to replace HTML strings in @@ -85,8 +82,7 @@ def spec_add_spaces(text: str) -> str: def replace_rep_after(text: str) -> str: - """ - Replace repetitions at the character level in `text` after the repeated character. + """Replace repetitions at the character level in `text` after the repeated character. This is to prevent cases such as 'น้อยยยยยยยย' becomes 'น้อ xxrep 8 ย' ; instead it will retain the word as 'น้อย xxrep 8' @@ -115,8 +111,7 @@ def _replace_rep(m): def replace_wrep_post(toks: Collection[str]) -> list[str]: - """ - Replace repetitive words after tokenization; + """Replace repetitive words after tokenization; fastai `replace_wrep` does not work well with Thai. :param list[str] toks: list of tokens @@ -150,13 +145,12 @@ def replace_wrep_post(toks: Collection[str]) -> list[str]: def rm_useless_newlines(text: str) -> str: - "Remove multiple newlines in `text`." - + """Remove multiple newlines in `text`.""" return re.sub(r"[\n]{2,}", " ", text) def rm_brackets(text: str) -> str: - "Remove all empty brackets and artifacts within brackets from `text`." + """Remove all empty brackets and artifacts within brackets from `text`.""" # remove empty brackets new_line = re.sub(r"\(\)", "", text) new_line = re.sub(r"\{\}", "", new_line) @@ -189,8 +183,7 @@ def rm_brackets(text: str) -> str: def ungroup_emoji(toks: Collection[str]) -> list[str]: - """ - Ungroup Zero Width Joiner (ZVJ) Emojis + """Ungroup Zero Width Joiner (ZVJ) Emojis See https://emojipedia.org/emoji-zwj-sequence/ """ @@ -204,16 +197,14 @@ def ungroup_emoji(toks: Collection[str]) -> list[str]: def lowercase_all(toks: Collection[str]) -> list[str]: - """ - Lowercase all English words; + """Lowercase all English words; English words in Thai texts don't usually have nuances of capitalization. """ return [tok.lower() for tok in toks] def replace_rep_nonum(text: str) -> str: - """ - Replace repetitions at the character level in `text` after the repetition. + """Replace repetitions at the character level in `text` after the repetition. This is done to prevent such case as 'น้อยยยยยยยย' becoming 'น้อ xxrep ย'; instead it will retain the word as 'น้อย xxrep ' @@ -242,8 +233,7 @@ def _replace_rep(m): def replace_wrep_post_nonum(toks: Collection[str]) -> list[str]: - """ - Replace reptitive words post tokenization; + """Replace reptitive words post tokenization; fastai `replace_wrep` does not work well with Thai. :param list[str] toks: list of tokens @@ -277,8 +267,7 @@ def replace_wrep_post_nonum(toks: Collection[str]) -> list[str]: def remove_space(toks: Collection[str]) -> list[str]: - """ - Do not include space for bag-of-word models. + """Do not include space for bag-of-word models. :param list[str] toks: list of tokens diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index 2b73dbf60..2ce042773 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Tokenzier classes for ULMFiT +"""Tokenzier classes for ULMFiT """ from __future__ import annotations @@ -26,8 +25,7 @@ def add_special_cases(self, toks: Collection[str]): class ThaiTokenizer(BaseTokenizer): - """ - Wrapper around a frozen newmm tokenizer to make it a + """Wrapper around a frozen newmm tokenizer to make it a :class:`fastai.BaseTokenizer`. (see: https://docs.fast.ai/text.transform#BaseTokenizer) """ @@ -37,8 +35,7 @@ def __init__(self, lang: str = "th"): @staticmethod def tokenizer(text: str) -> list[str]: - """ - This function tokenizes text using *newmm* engine and the dictionary + """This function tokenizes text using *newmm* engine and the dictionary specifically for `ulmfit` related functions (see: `Dictionary file (.txt) \ `_). diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index ffee432f3..071fc5f31 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Utility functions, like date conversion and digit conversion +"""Utility functions, like date conversion and digit conversion """ __all__ = [ @@ -110,10 +109,26 @@ ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa +from pythainlp.util.pronounce import ( + rhyme, + spelling, + thai_consonant_to_spelling, + tone_to_spelling, +) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) from pythainlp.util.strftime import thai_strftime + +# sound_syllable and pronounce have to be imported last, +# to prevent circular import issues. +# Other imports should be above this line, sorted. +from pythainlp.util.syllable import ( + sound_syllable, + syllable_length, + syllable_open_close_detector, + tone_detector, +) from pythainlp.util.thai import ( analyze_thai_text, count_thai_chars, @@ -128,19 +143,3 @@ from pythainlp.util.time import thaiword_to_time, time_to_thaiword from pythainlp.util.trie import Trie, dict_trie from pythainlp.util.wordtonum import text_to_num, thaiword_to_num, words_to_num - -# sound_syllable and pronounce have to be imported last, -# to prevent circular import issues. -# Other imports should be above this line, sorted. -from pythainlp.util.syllable import ( - sound_syllable, - syllable_length, - syllable_open_close_detector, - tone_detector, -) -from pythainlp.util.pronounce import ( - rhyme, - spelling, - thai_consonant_to_spelling, - tone_to_spelling, -) diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index 380344472..f52cffc0e 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai abbreviation tools +"""Thai abbreviation tools """ from __future__ import annotations @@ -11,8 +10,7 @@ def abbreviation_to_full_text( text: str, top_k: int = 2 ) -> list[tuple[str, float | None]]: - """ - This function converts Thai text (with abbreviation) to full text. + """This function converts Thai text (with abbreviation) to full text. This function uses KhamYo for handles abbreviations. See more `KhamYo `_. diff --git a/pythainlp/util/collate.py b/pythainlp/util/collate.py index 10bd9a511..6f6feb5e9 100644 --- a/pythainlp/util/collate.py +++ b/pythainlp/util/collate.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai collation (sorted according to Thai dictionary order) +"""Thai collation (sorted according to Thai dictionary order) Simple implementation using regular expressions """ @@ -25,8 +24,7 @@ def _thkey(word: str) -> str: def collate(data: Iterable, reverse: bool = False) -> list[str]: - """ - This function sorts strings (almost) according to Thai dictionary. + """This function sorts strings (almost) according to Thai dictionary. Important notes: this implementation ignores tone marks and symbols diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index 21327d1e6..70ce91508 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai date/time conversion. +"""Thai date/time conversion. Note: It does not take into account the change of new year's day in Thailand """ @@ -122,8 +121,7 @@ def convert_years(year: str, src="be", target="ad") -> str: - """ - Convert years + """Convert years :param int year: Year :param str src: The source year @@ -204,8 +202,7 @@ def thai_strptime( add_year: int | None = None, tzinfo=ZoneInfo("Asia/Bangkok"), ): - """ - Thai strptime + """Thai strptime :param str text: text :param str fmt: string containing date and time directives @@ -313,8 +310,7 @@ def thai_strptime( def now_reign_year() -> int: - """ - Return the reign year of the 10th King of Chakri dynasty. + """Return the reign year of the 10th King of Chakri dynasty. :return: reign year of the 10th King of Chakri dynasty. :rtype: int @@ -335,8 +331,7 @@ def now_reign_year() -> int: def reign_year_to_ad(reign_year: int, reign: int) -> int: - """ - Convert reign year to AD. + """Convert reign year to AD. Return AD year according to the reign year for the 7th to 10th King of Chakri dynasty, Thailand. @@ -373,8 +368,7 @@ def reign_year_to_ad(reign_year: int, reign: int) -> int: def thaiword_to_date(text: str, date: datetime = None) -> datetime | None: - """ - Convert Thai relative date to :class:`datetime.datetime`. + """Convert Thai relative date to :class:`datetime.datetime`. :param str text: Thai text containing relative date :param datetime.datetime date: date (default is datetime.datetime.now()) diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index ebd622069..370968972 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert digits +"""Convert digits """ from __future__ import annotations @@ -65,8 +64,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: - """ - This function converts Thai digits (i.e. ๑, ๓, ๑๐) to Arabic digits + """This function converts Thai digits (i.e. ๑, ๓, ๑๐) to Arabic digits (i.e. 1, 3, 10). :param str text: Text with Thai digits such as '๑', '๒', '๓' @@ -91,8 +89,7 @@ def thai_digit_to_arabic_digit(text: str) -> str: def arabic_digit_to_thai_digit(text: str) -> str: - """ - This function converts Arabic digits (i.e. 1, 3, 10) to Thai digits + """This function converts Arabic digits (i.e. 1, 3, 10) to Thai digits (i.e. ๑, ๓, ๑๐). :param str text: Text with Arabic digits such as '1', '2', '3' @@ -118,8 +115,7 @@ def arabic_digit_to_thai_digit(text: str) -> str: def digit_to_text(text: str) -> str: - """ - :param str text: Text with digits such as '1', '2', '๓', '๔' + """:param str text: Text with digits such as '1', '2', '๓', '๔' :return: Text with digits spelled out in Thai """ if not text or not isinstance(text, str): @@ -133,8 +129,7 @@ def digit_to_text(text: str) -> str: def text_to_arabic_digit(text: str) -> str: - """ - This function converts spelled out digits in Thai to Arabic digits. + """This function converts spelled out digits in Thai to Arabic digits. :param text: A digit spelled out in Thai :return: An Arabic digit such as '1', '2', '3' if the text is @@ -171,8 +166,7 @@ def text_to_arabic_digit(text: str) -> str: def text_to_thai_digit(text: str) -> str: - """ - This function converts spelled out digits in Thai to Thai digits. + """This function converts spelled out digits in Thai to Thai digits. :param text: A digit spelled out in Thai :return: A Thai digit such as '๑', '๒', '๓' if the text is digit diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index 0f324c5ea..985a965da 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert emojis +"""Convert emojis """ from __future__ import annotations @@ -1835,8 +1834,7 @@ def emoji_to_thai(text: str, delimiters=(_delimiter, _delimiter)) -> str: - """ - This function converts emojis to their Thai meanings + """This function converts emojis to their Thai meanings :param str text: Text with emojis :return: Text with emojis converted to their Thai meanings @@ -1857,7 +1855,6 @@ def emoji_to_thai(text: str, delimiters=(_delimiter, _delimiter)) -> str: emoji_to_thai("🇹🇭 นี่คือธงประเทศไทย") # output: :ธง_ไทย: นี่คือธงประเทศไทย """ - return _emoji_regex.sub( lambda match: delimiters[0] + _emoji_th[match.group(0)] diff --git a/pythainlp/util/encoding.py b/pythainlp/util/encoding.py index 3f0c164d0..0727c3f74 100644 --- a/pythainlp/util/encoding.py +++ b/pythainlp/util/encoding.py @@ -6,8 +6,7 @@ def tis620_to_utf8(text: str) -> str: - """ - Convert TIS-620 to UTF-8 + """Convert TIS-620 to UTF-8 :param str text: TIS-620 encoded text :return: UTF-8 encoded text @@ -25,8 +24,7 @@ def tis620_to_utf8(text: str) -> str: def to_idna(text: str) -> str: - """ - Encode text with IDNA, as used in Internationalized Domain Name (IDN). + """Encode text with IDNA, as used in Internationalized Domain Name (IDN). :param str text: Thai text :return: IDNA-encoded text diff --git a/pythainlp/util/keyboard.py b/pythainlp/util/keyboard.py index a86c2ce65..6ea667070 100644 --- a/pythainlp/util/keyboard.py +++ b/pythainlp/util/keyboard.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Functions related to keyboard layout. +"""Functions related to keyboard layout. """ from __future__ import annotations @@ -122,8 +121,7 @@ def eng_to_thai(text: str) -> str: - """ - Corrects the given text that was incorrectly typed using English-US + """Corrects the given text that was incorrectly typed using English-US Qwerty keyboard layout to the originally intended keyboard layout that is the Thai Kedmanee keyboard. @@ -145,8 +143,7 @@ def eng_to_thai(text: str) -> str: def thai_to_eng(text: str) -> str: - """ - Corrects the given text that was incorrectly typed using Thai Kedmanee + """Corrects the given text that was incorrectly typed using Thai Kedmanee keyboard layout to the originally intended keyboard layout that is the English-US Qwerty keyboard. @@ -168,8 +165,7 @@ def thai_to_eng(text: str) -> str: def thai_keyboard_dist(c1: str, c2: str, shift_dist: float = 0.0) -> float: - """ - Calculate Euclidean distance between two Thai characters + """Calculate Euclidean distance between two Thai characters according to their location on a Thai keyboard layout. A modified TIS 820-2531 standard keyboard layout, which is developed diff --git a/pythainlp/util/keywords.py b/pythainlp/util/keywords.py index a555c41f3..fb9b10a33 100644 --- a/pythainlp/util/keywords.py +++ b/pythainlp/util/keywords.py @@ -11,8 +11,7 @@ def rank(words: list[str], exclude_stopwords: bool = False) -> Counter: - """ - Count word frequencies given a list of Thai words with an option + """Count word frequencies given a list of Thai words with an option to exclude stopwords. :param list words: a list of words @@ -73,8 +72,7 @@ def rank(words: list[str], exclude_stopwords: bool = False) -> Counter: def find_keyword(word_list: list[str], min_len: int = 3) -> dict[str, int]: - """ - This function counts the frequencies of words in the list + """This function counts the frequencies of words in the list where stopword is excluded and returns a frequency dictionary. :param list word_list: a list of words diff --git a/pythainlp/util/lcs.py b/pythainlp/util/lcs.py index a6f29f8d4..e127b68a2 100644 --- a/pythainlp/util/lcs.py +++ b/pythainlp/util/lcs.py @@ -5,8 +5,7 @@ def longest_common_subsequence(str1: str, str2: str) -> str: - """ - Find the longest common subsequence between two strings. + """Find the longest common subsequence between two strings. :param str str1: The first string. :param str str2: The second string. diff --git a/pythainlp/util/morse.py b/pythainlp/util/morse.py index 6b5544d1c..73b8e1cd0 100644 --- a/pythainlp/util/morse.py +++ b/pythainlp/util/morse.py @@ -134,8 +134,7 @@ def morse_encode(text: str, lang: str = "th") -> str: - """ - Convert text to Morse code (support Thai and English) + """Convert text to Morse code (support Thai and English) :param str text: Text :param str lang: Language Code (*th* is Thai and *en* is English) @@ -166,8 +165,7 @@ def morse_encode(text: str, lang: str = "th") -> str: def morse_decode(morse_text: str, lang: str = "th") -> str: - """ - Simple Convert Morse code to text + """Simple Convert Morse code to text Thai still have some wrong character problem that\ can fix by spell corrector. diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index 7f9b29421..6251ee495 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Text normalization +"""Text normalization """ from __future__ import annotations @@ -57,8 +56,7 @@ def _last_char(matchobj): # to be used with _RE_NOREPEAT_TONEMARKS def remove_dangling(text: str) -> str: - """ - Remove Thai non-base characters at the beginning of text. + """Remove Thai non-base characters at the beginning of text. This is a common "typo", especially for input field in a form, as these non-base characters can be visually hidden from user @@ -85,8 +83,7 @@ def remove_dangling(text: str) -> str: def remove_dup_spaces(text: str) -> str: - """ - Remove duplicate spaces. Replace multiple spaces with one space. + """Remove duplicate spaces. Replace multiple spaces with one space. Multiple newline characters and empty lines will be replaced with one newline character. @@ -111,8 +108,7 @@ def remove_dup_spaces(text: str) -> str: def remove_tonemark(text: str) -> str: - """ - Remove all Thai tone marks from the text. + """Remove all Thai tone marks from the text. Thai script has four tone marks indicating four tones as follows: @@ -144,8 +140,7 @@ def remove_tonemark(text: str) -> str: def remove_zw(text: str) -> str: - """ - Remove zero-width characters. + """Remove zero-width characters. These non-visible characters may cause unexpected result from the user's point of view. Removing them can make string matching more robust. @@ -167,8 +162,7 @@ def remove_zw(text: str) -> str: def reorder_vowels(text: str) -> str: - """ - Reorder vowels and tone marks to the standard logical order/spelling. + """Reorder vowels and tone marks to the standard logical order/spelling. Characters in input text will be reordered/transformed, according to these rules: @@ -189,8 +183,7 @@ def reorder_vowels(text: str) -> str: def remove_repeat_vowels(text: str) -> str: - """ - Remove repeating vowels, tone marks, and signs. + """Remove repeating vowels, tone marks, and signs. This function will call reorder_vowels() first, to make sure that double Sara E will be converted to Sara Ae and not be removed. @@ -210,8 +203,7 @@ def remove_repeat_vowels(text: str) -> str: def normalize(text: str) -> str: - """ - Normalize and clean Thai text with normalizing rules as follows: + """Normalize and clean Thai text with normalizing rules as follows: * Remove zero-width spaces * Remove duplicate spaces @@ -252,8 +244,7 @@ def normalize(text: str) -> str: def expand_maiyamok(sent: str | list[str]) -> list[str]: - """ - Expand Maiyamok. + """Expand Maiyamok. Maiyamok (ๆ) (Unicode U+0E46) is a Thai character indicating word repetition. This function preprocesses Thai text by replacing @@ -312,8 +303,7 @@ def expand_maiyamok(sent: str | list[str]) -> list[str]: def maiyamok(sent: str | list[str]) -> list[str]: - """ - Expand Maiyamok. + """Expand Maiyamok. Deprecated. Use expand_maiyamok() instead. diff --git a/pythainlp/util/numtoword.py b/pythainlp/util/numtoword.py index 8f19367b2..6c7c2b6fa 100644 --- a/pythainlp/util/numtoword.py +++ b/pythainlp/util/numtoword.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert number value to Thai read out +"""Convert number value to Thai read out Adapted from http://justmindthought.blogspot.com/2012/12/code-php.html @@ -30,8 +29,7 @@ def bahttext(number: float) -> str: - """ - This function converts a number to Thai text and adds + """This function converts a number to Thai text and adds a suffix "บาท" (Baht). The precision will be fixed at two decimal places (0.00) to fits "สตางค์" (Satang) unit. @@ -80,8 +78,7 @@ def bahttext(number: float) -> str: def num_to_thaiword(number: int) -> str: - """ - This function converts number to Thai text + """This function converts number to Thai text :param int number: an integer number to be converted to Thai text :return: text representing the number in Thai @@ -98,7 +95,6 @@ def num_to_thaiword(number: int) -> str: num_to_thaiword(11) # output: สิบเอ็ด """ - output = "" number_temp = number if number is None: diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index c8c75d5f6..2fbc57bbe 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Phonemes util +"""Phonemes util """ from __future__ import annotations @@ -91,8 +90,7 @@ def nectec_to_ipa(pronunciation: str) -> str: - """ - Convert NECTEC system to IPA system + """Convert NECTEC system to IPA system :param str pronunciation: NECTEC phoneme :return: IPA that is converted @@ -109,10 +107,10 @@ def nectec_to_ipa(pronunciation: str) -> str: References ---------- - Pornpimon Palingoon, Sumonmas Thatphithakkul. Chapter 4 Speech processing \ and Speech corpus. In: Handbook of Thai Electronic Corpus. \ 1st ed. p. 122–56. + """ parts = pronunciation.split("-") ipa = [] @@ -205,8 +203,7 @@ def _ipa_cut(): def ipa_to_rtgs(ipa: str) -> str: - """ - Convert IPA system to The Royal Thai General System of Transcription (RTGS) + """Convert IPA system to The Royal Thai General System of Transcription (RTGS) Docs: https://en.wikipedia.org/wiki/Help:IPA/Thai @@ -246,8 +243,7 @@ def ipa_to_rtgs(ipa: str) -> str: def remove_tone_ipa(ipa: str) -> str: - """ - Remove Thai Tones from IPA system + """Remove Thai Tones from IPA system :param str ipa: IPA phoneme :return: IPA phoneme with tones removed diff --git a/pythainlp/util/pronounce.py b/pythainlp/util/pronounce.py index e3dd01002..41f14914c 100644 --- a/pythainlp/util/pronounce.py +++ b/pythainlp/util/pronounce.py @@ -16,8 +16,7 @@ def rhyme(word: str) -> list[str]: - """ - Find Thai rhyme + """Find Thai rhyme :param str word: A Thai word :return: All list Thai rhyme words @@ -84,8 +83,7 @@ def rhyme(word: str) -> list[str]: def thai_consonant_to_spelling(c: str) -> str: - """ - Thai consonants to spelling + """Thai consonants to spelling :param str c: A Thai consonant :return: spelling @@ -105,8 +103,7 @@ def thai_consonant_to_spelling(c: str) -> str: def tone_to_spelling(t: str) -> str: - """ - Thai tonemarks to spelling + """Thai tonemarks to spelling :param str t: A Thai tonemarks :return: spelling @@ -132,8 +129,7 @@ def tone_to_spelling(t: str) -> str: def spelling(word: str) -> list[str]: - """ - Thai word to spelling + """Thai word to spelling This funnction support Thai root word only. diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index 3fb4f004c..ad6349435 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Removement of repeated consonants at the end of words +"""Removement of repeated consonants at the end of words """ from __future__ import annotations @@ -26,8 +25,7 @@ def remove_trailing_repeat_consonants( custom_dict: Iterable[str] = [], has_dictionary_updated: bool = True, ) -> str: - """ - Remove repeating consonants at the last of the sentence. + """Remove repeating consonants at the last of the sentence. This function will remove the repeating consonants before a whitespace, new line or at the last @@ -102,8 +100,7 @@ def remove_trailing_repeat_consonants( def _remove_repeat_trailing_consonants_from_segment(segment: str) -> str: - """ - Remove repeating consonants at the last of the segment. + """Remove repeating consonants at the last of the segment. This function process only at the last of the given text. Details is same as remove_repeat_consonants(). @@ -154,8 +151,7 @@ def _remove_repeat_trailing_consonants_from_segment(segment: str) -> str: def _remove_all_last_consonants(text: str, dup: str) -> str: - """ - Reduce repeating characters at the end of the text. + """Reduce repeating characters at the end of the text. This function will remove the repeating characters at the last. The text just before the repeating characters will be returned. @@ -173,8 +169,7 @@ def _remove_all_last_consonants(text: str, dup: str) -> str: def _update_consonant_repeaters(custom_dict: Iterable[str]) -> None: - """ - Update dictionary of all words that has + """Update dictionary of all words that has repeating consonants at the end from the dictionary. Search all words in the dictionary that has more than 1 consonants @@ -197,8 +192,7 @@ def _update_consonant_repeaters(custom_dict: Iterable[str]) -> None: def _is_last_consonant_repeater(word: str) -> bool: - """ - Check if the word has repeating consonants at the end. + """Check if the word has repeating consonants at the end. This function checks if the word has more than 1 repeating consonants at the end. @@ -215,8 +209,7 @@ def _is_last_consonant_repeater(word: str) -> bool: def _find_longest_consonant_repeaters_match( segment_head: str, repeaters: list[str] ) -> tuple[str, int]: - """ - Find the longest word that matches the segment. + """Find the longest word that matches the segment. Find the longest word that matches the last of the segment from the given repeaters list. diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index a3228ed79..c1850829f 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -85,8 +85,7 @@ def _clean(w): def spell_syllable(text: str) -> list[str]: - """ - Spell out syllables in Thai word distribution form. + """Spell out syllables in Thai word distribution form. :param str s: Thai syllables only :return: List of spelled out syllables @@ -110,8 +109,7 @@ def spell_syllable(text: str) -> list[str]: def spell_word(text: str) -> list[str]: - """ - Spell out words in Thai word distribution form. + """Spell out words in Thai word distribution form. :param str w: Thai words only :return: List of spelled out words diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index 962d519da..21a18af24 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai date/time formatting. +"""Thai date/time formatting. """ from __future__ import annotations @@ -31,8 +30,7 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: - """ - Standard datetime.strftime() with normalization and exception handling. + """Standard datetime.strftime() with normalization and exception handling. """ str_ = "" try: @@ -58,8 +56,7 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: - """ - Conversion support for thai_strftime(). + """Conversion support for thai_strftime(). The fmt_char should be in _NEED_L10N when calling this function. """ @@ -152,8 +149,7 @@ def thai_strftime( fmt: str = "%-d %b %y", thaidigit: bool = False, ) -> str: - """ - Convert :class:`datetime.datetime` into Thai date and time format. + """Convert :class:`datetime.datetime` into Thai date and time format. The formatting directives are similar to :func:`datatime.strrftime`. diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index 0fd51aef9..e193e1a87 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Syllable tools +"""Syllable tools """ from __future__ import annotations @@ -62,8 +61,7 @@ def sound_syllable(syllable: str) -> str: - """ - Sound syllable classification + """Sound syllable classification This function is sound syllable classification. The syllable is a live syllable or dead syllable. @@ -158,8 +156,7 @@ def sound_syllable(syllable: str) -> str: def syllable_open_close_detector(syllable: str) -> str: - """ - Open/close Thai syllables detector + """Open/close Thai syllables detector This function is used for finding Thai syllables that are open or closed sound. @@ -190,8 +187,7 @@ def syllable_open_close_detector(syllable: str) -> str: def syllable_length(syllable: str) -> str: - """ - Thai syllable length + """Thai syllable length This function is used for finding syllable's length. (long or short) @@ -242,8 +238,7 @@ def _check_sonorant_syllable(syllable: str) -> bool: def tone_detector(syllable: str) -> str: - """ - Thai tone detector for syllables + """Thai tone detector for syllables Return tone of a syllable. diff --git a/pythainlp/util/thai.py b/pythainlp/util/thai.py index 4fb053828..a8f7c8550 100644 --- a/pythainlp/util/thai.py +++ b/pythainlp/util/thai.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Check if it is Thai text +"""Check if it is Thai text """ from __future__ import annotations @@ -205,7 +204,6 @@ def display_thai_char(ch: str) -> str: display_thai_char("้") # output: "_้" """ - if ( ch in thai_above_vowels or ch in thai_tonemarks @@ -218,8 +216,7 @@ def display_thai_char(ch: str) -> str: def thai_word_tone_detector(word: str) -> tuple[str, str]: - """ - Thai tone detector for word. + """Thai tone detector for word. It uses pythainlp.transliterate.pronunciate for converting word to\ pronunciation. @@ -248,8 +245,7 @@ def thai_word_tone_detector(word: str) -> tuple[str, str]: def count_thai_chars(text: str) -> dict: - """ - Count Thai characters by type + """Count Thai characters by type This function will give you numbers of Thai characters by type\ (consonants, vowels, lead_vowels, follow_vowels, above_vowels,\ @@ -319,8 +315,7 @@ def count_thai_chars(text: str) -> dict: def analyze_thai_text(text: str) -> dict: - """ - Analyzes a string of Thai text and returns a dictionaries, + """Analyzes a string of Thai text and returns a dictionaries, where each values represents a single classified character from the text. The function processes the text character by character and maps each Thai @@ -337,6 +332,7 @@ def analyze_thai_text(text: str) -> dict: >>> analyze_thai_text("เล่น") {'สระ เอ': 1, 'ล': 1, 'ไม้เอก': 1, 'น': 1} + """ results = defaultdict(int) diff --git a/pythainlp/util/thai_lunar_date.py b/pythainlp/util/thai_lunar_date.py index 92dd75890..e89002686 100644 --- a/pythainlp/util/thai_lunar_date.py +++ b/pythainlp/util/thai_lunar_date.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -This file is a port from +"""This file is a port from > https://gist.github.com/touchiep/99f4f5bb349d6b983ef78697630ab78e """ @@ -281,8 +280,7 @@ def last_day_in_year(year: int) -> int: def athikasurathin(year: int) -> bool: - """ - Check if a year is a leap year in the Thai lunar calendar + """Check if a year is a leap year in the Thai lunar calendar """ # Check divisibility by 400 (divisible by 400 is always a leap year) if year % 400 == 0: @@ -309,8 +307,7 @@ def number_day_in_year(year: int) -> int: def th_zodiac(year: int, output_type: int = 1) -> str | int: - """ - Thai Zodiac Year Name + """Thai Zodiac Year Name Converts a Gregorian year to its corresponding Zodiac name. :param int year: The Gregorian year. AD (Anno Domini) @@ -331,8 +328,7 @@ def th_zodiac(year: int, output_type: int = 1) -> str | int: def to_lunar_date(input_date: date) -> str: - """ - Convert the solar date to Thai Lunar Date + """Convert the solar date to Thai Lunar Date :param date input_date: date of the day. :return: Thai text lunar date diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index ae5fd9b90..b8d1a7aaf 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Spell out time as Thai words. +"""Spell out time as Thai words. Convert time string or time object to Thai words. """ @@ -159,8 +158,7 @@ def time_to_thaiword( fmt: str = "24h", precision: str | None = None, ) -> str: - """ - Spell out time as Thai words. + """Spell out time as Thai words. :param str time_data: time input, can be a datetime.time object \ or a datetime.datetime object \ @@ -236,8 +234,7 @@ def time_to_thaiword( def thaiword_to_time(text: str, padding: bool = True) -> str: - """ - Convert Thai time in words into time (H:M). + """Convert Thai time in words into time (H:M). :param str text: Thai time in words :param bool padding: Zero pad the hour if True diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index 869cad34d..e3e518dc7 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Trie data structure. +"""Trie data structure. Designed to be used for tokenizer's dictionary, but can be for other purposes. """ @@ -28,8 +27,7 @@ def __init__(self, words: Iterable[str]): self.add(word) def add(self, word: str) -> None: - """ - Add a word to the trie. + """Add a word to the trie. Spaces in front of and following the word will be removed. :param str text: a word @@ -46,8 +44,7 @@ def add(self, word: str) -> None: cur.end = True def remove(self, word: str) -> None: - """ - Remove a word from the trie. + """Remove a word from the trie. If the word is not found, do nothing. :param str text: a word @@ -72,8 +69,7 @@ def remove(self, word: str) -> None: del parent.children[ch] # remove from parent dict def prefixes(self, text: str) -> list[str]: - """ - List all possible words from first sequence of characters in a word. + """List all possible words from first sequence of characters in a word. :param str text: a word :return: a list of possible words @@ -101,8 +97,7 @@ def __len__(self) -> int: def dict_trie(dict_source: str | Iterable[str] | Trie) -> Trie: - """ - Create a dictionary trie from a file or an iterable. + """Create a dictionary trie from a file or an iterable. :param str|Iterable[str]|pythainlp.util.Trie dict_source: a path to dictionary file or a list of words or a pythainlp.util.Trie object diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 3087cd7d9..c213b6ec1 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Convert number in words to a computable number value +"""Convert number in words to a computable number value First version of the code adapted from Korakot Chaovavanich's notebook https://colab.research.google.com/drive/148WNIeclf0kOU6QxKd6pcfwpSs8l-VKD#scrollTo=EuVDd0nNuI8Q @@ -77,8 +76,7 @@ def _tokenizer_thaiwords(): def thaiword_to_num(word: str) -> int: - """ - Converts the spelled-out numerals in Thai scripts into an actual integer. + """Converts the spelled-out numerals in Thai scripts into an actual integer. :param str word: Spelled-out numerals in Thai scripts :return: Corresponding integer value of the input @@ -143,8 +141,7 @@ def _decimal_unit(words: list) -> float: def words_to_num(words: list) -> float: - """ - Thai Words to float + """Thai Words to float :param str text: Thai words :return: float of words @@ -175,8 +172,7 @@ def words_to_num(words: list) -> float: def text_to_num(text: str) -> list[str]: - """ - Thai text to list of Thai words with floating point numbers + """Thai text to list of Thai words with floating point numbers :param str text: Thai text with the spelled-out numerals :return: list of Thai words with float values of the input diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index f14322fc6..d07537f32 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -25,8 +25,7 @@ class ThaiNameTagger: def __init__( self, dataset_name: str = "thainer", grouped_entities: bool = True ): - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -57,8 +56,7 @@ def _clear_tag(self, tag): def get_ner( self, text: str, pos: bool = False, tag: bool = False ) -> list[tuple[str, str]] | str: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -136,8 +134,7 @@ class NamedEntityRecognition: def __init__( self, model: str = "pythainlp/thainer-corpus-v2-base-model" ) -> None: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -166,8 +163,7 @@ def _fix_span_error(self, words, ner): def get_ner( self, text: str, pos: bool = False, tag: bool = False ) -> list[tuple[str, str]] | str: - """ - This function tags named entities in text in IOB format. + """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand @@ -226,8 +222,7 @@ def get_ner( def segment(text: str) -> list[str]: - """ - Subword tokenize. SentencePiece from wangchanberta model. + """Subword tokenize. SentencePiece from wangchanberta model. :param str text: text to be tokenized :return: list of subwords diff --git a/pythainlp/word_vector/__init__.py b/pythainlp/word_vector/__init__.py index 7aa5165f5..00c8423ae 100644 --- a/pythainlp/word_vector/__init__.py +++ b/pythainlp/word_vector/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -thai2fit - Thai word vector. +"""thai2fit - Thai word vector. Initial code from https://github.com/cstorm125/thai2fit """ diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index c032a0dc5..9bfd19e0a 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -19,8 +19,7 @@ class WordVector: - """ - Word Vector class + """Word Vector class :param str model_name: model name @@ -32,8 +31,7 @@ class WordVector: """ def __init__(self, model_name: str = "thai2fit_wv") -> None: - """ - Word Vector class + """Word Vector class :param str model_name: model name @@ -46,8 +44,7 @@ def __init__(self, model_name: str = "thai2fit_wv") -> None: self.load_wordvector(model_name) def load_wordvector(self, model_name: str): - """ - Load word vector model. + """Load word vector model. :param str model_name: model name """ @@ -65,8 +62,7 @@ def load_wordvector(self, model_name: str): self.tokenize = word_tokenize def get_model(self) -> Word2VecKeyedVectors: - """ - Get word vector model. + """Get word vector model. :return: `gensim` word2vec model :rtype: gensim.models.keyedvectors.Word2VecKeyedVectors @@ -74,8 +70,7 @@ def get_model(self) -> Word2VecKeyedVectors: return self.model def doesnt_match(self, words: list[str]) -> str: - """ - This function returns one word that is mostly unrelated to other words + """This function returns one word that is mostly unrelated to other words in the list. We use the function :func:`doesnt_match` from :mod:`gensim`. @@ -114,8 +109,7 @@ def doesnt_match(self, words: list[str]) -> str: def most_similar_cosmul( self, positive: list[str], negative: list[str] ) -> list[tuple[str, float]]: - """ - This function finds the top-10 words that are most similar with respect + """This function finds the top-10 words that are most similar with respect to two lists of words labeled as positive and negative. The top-10 most similar words are obtained using multiplication combination objective from Omer Levy and Yoav Goldberg @@ -216,8 +210,7 @@ def most_similar_cosmul( ) def similarity(self, word1: str, word2: str) -> float: - """ - This function computes cosine similarity between two words. + """This function computes cosine similarity between two words. :param str word1: first word to be compared with :param str word2: second word to be compared with @@ -255,8 +248,7 @@ def similarity(self, word1: str, word2: str) -> float: return self.model.similarity(word1, word2) def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: - """ - This function converts a Thai sentence into vector. + """This function converts a Thai sentence into vector. Specifically, it first tokenizes that text and map each tokenized word with the word vectors from the model. Then, word vectors are aggregated into one vector of 300 dimension diff --git a/pythainlp/wsd/__init__.py b/pythainlp/wsd/__init__.py index aeb7cd113..1fce7736d 100644 --- a/pythainlp/wsd/__init__.py +++ b/pythainlp/wsd/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Thai Word Sense Disambiguation (WSD) +"""Thai Word Sense Disambiguation (WSD) """ __all__ = ["get_sense"] diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 8b947bf02..04e5da796 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -53,10 +53,9 @@ def get_sense( custom_dict: dict = dict(), custom_tokenizer: Tokenizer = _word_cut, ) -> list[tuple[str, float]]: - """ - Get word sense from the sentence. + """Get word sense from the sentence. This function will get definition and distance from context in sentence. - + :param str sentence: Thai sentence :param str word: Thai word :param str device: device for running model on. @@ -66,19 +65,19 @@ def get_sense( :return: a list of definitions and distances (1 - cos_sim) or \ an empty list (if word is not in the dictionary) :rtype: List[Tuple[str, float]] - + We get the ideas from `Context-Aware Semantic Similarity Measurement for \ Unsupervised Word Sense Disambiguation \ `_ to build get_sense function. Use Thai dictionary from wiktionary. See `thai_dict `_. - + Use sentence transformers model from \ `sentence-transformers/paraphrase-multilingual-mpnet-base-v2 \ `_ \ for unsupervised word sense disambiguation. - + :Example: :: diff --git a/tests/__init__.py b/tests/__init__.py index 97ba77207..e325bf4ed 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. +"""Unit test. Each file in tests/ is for each main package. """ diff --git a/tests/compact/__init__.py b/tests/compact/__init__.py index 3aa77b4a2..63bba7f50 100644 --- a/tests/compact/__init__.py +++ b/tests/compact/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. Compact version. +"""Unit test. Compact version. Test functions that require "compact" dependencies (see setup.py). """ diff --git a/tests/compact/test_cli.py b/tests/compact/test_cli.py index 6c3a2dbe5..38b4c6c70 100644 --- a/tests/compact/test_cli.py +++ b/tests/compact/test_cli.py @@ -2,13 +2,12 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.cli module. (compact) +"""Unit tests for pythainlp.cli module. (compact) """ import unittest -from pythainlp import __main__, cli +from pythainlp import cli from pythainlp.cli.misspell import App as MisspellApp diff --git a/tests/compact/testc_util.py b/tests/compact/testc_util.py index 007db4c15..1f6d06366 100644 --- a/tests/compact/testc_util.py +++ b/tests/compact/testc_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import unittest diff --git a/tests/core/__init__.py b/tests/core/__init__.py index 3aac3cb6c..ea3afeac0 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. +"""Unit test. Each file in tests/ is for each main package. """ diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index 29494d63e..f541b08ef 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -6,8 +6,6 @@ import os import unittest -from requests import Response - from pythainlp.corpus import ( countries, download, diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index e5672506d..b1cbb0956 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -6,8 +6,8 @@ import unittest from pythainlp.tokenize import ( - word_dict_trie, Tokenizer, + display_cell_tokenize, etcc, longest, multi_cut, @@ -18,8 +18,8 @@ tcc, tcc_p, word_detokenize, + word_dict_trie, word_tokenize, - display_cell_tokenize, ) from pythainlp.util import dict_trie @@ -411,7 +411,6 @@ def test_longest(self): def test_longest_custom_dict(self): """Test switching the custom dict on longest segment function""" - self.assertEqual( word_tokenize("ทดสอบ ทดสอบ", engine="longest"), ["ทดสอบ", " ", "ทดสอบ"], diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 19558ba92..97f48b437 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import os @@ -16,6 +15,7 @@ from pythainlp.corpus.common import _THAI_WORDS_FILENAME from pythainlp.util import ( Trie, + analyze_thai_text, arabic_digit_to_thai_digit, bahttext, collate, @@ -46,6 +46,7 @@ remove_trailing_repeat_consonants, remove_zw, sound_syllable, + spelling, syllable_length, syllable_open_close_detector, text_to_arabic_digit, @@ -66,8 +67,6 @@ to_lunar_date, tone_detector, words_to_num, - spelling, - analyze_thai_text, ) from pythainlp.util.morse import morse_decode, morse_encode diff --git a/tests/extra/__init__.py b/tests/extra/__init__.py index f8b6bcf16..4073d5b1c 100644 --- a/tests/extra/__init__.py +++ b/tests/extra/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit test. Extra version. +"""Unit test. Extra version. Test functions that require dependencies beyond "compact" (see setup.py). """ diff --git a/tests/extra/testx_cli.py b/tests/extra/testx_cli.py index fe6c8e56d..730dff76c 100644 --- a/tests/extra/testx_cli.py +++ b/tests/extra/testx_cli.py @@ -5,7 +5,7 @@ import unittest -from pythainlp import __main__, cli +from pythainlp import cli from pythainlp.cli.benchmark import App as BenchmarkApp from pythainlp.cli.data import App as DataApp from pythainlp.cli.tokenize import App as TokenizeApp diff --git a/tests/extra/testx_coref.py b/tests/extra/testx_coref.py index bb2e4d09b..749b9a773 100644 --- a/tests/extra/testx_coref.py +++ b/tests/extra/testx_coref.py @@ -5,8 +5,6 @@ import unittest -from pythainlp.coref import coreference_resolution - class CorefTestCaseX(unittest.TestCase): def test_coreference_resolution(self): diff --git a/tests/extra/testx_spell.py b/tests/extra/testx_spell.py index 644def075..ea8f65273 100644 --- a/tests/extra/testx_spell.py +++ b/tests/extra/testx_spell.py @@ -8,10 +8,10 @@ from pythainlp.spell import ( correct, correct_sent, + get_words_spell_suggestion, spell, spell_sent, symspellpy, - get_words_spell_suggestion, ) from ..core.test_spell import SENT_TOKS diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index e24e7eeb6..d2170e0dd 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -8,7 +8,6 @@ import unittest from pythainlp.tokenize import ( - word_dict_trie, attacut, deepcut, nercut, @@ -19,6 +18,7 @@ ssg, subword_tokenize, tltk, + word_dict_trie, word_tokenize, ) diff --git a/tests/extra/testx_util.py b/tests/extra/testx_util.py index be81edd08..9bc2d857e 100644 --- a/tests/extra/testx_util.py +++ b/tests/extra/testx_util.py @@ -3,8 +3,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -Unit tests for pythainlp.util module. +"""Unit tests for pythainlp.util module. """ import unittest From f10ef245f2fd6f23332ec7fa8db9c8bb8294ec14 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 13:08:04 +0000 Subject: [PATCH 4/5] Revert reordering of corpus.core imports --- pythainlp/corpus/__init__.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 48376ed6d..6271cd35b 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -87,7 +87,21 @@ def corpus_db_path() -> str: """ return _CORPUS_DB_PATH - +# DO NOT REORDER THESE pythainlp.corpus imports +from pythainlp.corpus.core import ( + download, + get_corpus, + get_corpus_as_is, + get_corpus_db, + get_corpus_db_detail, + get_corpus_default_db, + get_corpus_path, + get_hf_hub, + get_path_folder_corpus, + make_safe_directory_name, + path_pythainlp_corpus, + remove, +) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.common import ( countries, find_synonyms, @@ -105,20 +119,6 @@ def corpus_db_path() -> str: thai_words, thai_wsd_dict, ) -from pythainlp.corpus.core import ( - download, - get_corpus, - get_corpus_as_is, - get_corpus_db, - get_corpus_db_detail, - get_corpus_default_db, - get_corpus_path, - get_hf_hub, - get_path_folder_corpus, - make_safe_directory_name, - path_pythainlp_corpus, - remove, -) # these imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.icu import thai_icu_words from pythainlp.corpus.volubilis import thai_volubilis_words from pythainlp.corpus.wikipedia import thai_wikipedia_titles From d704a5015fbf7f51f707ea7c28433e4d51633dd6 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 13:12:41 +0000 Subject: [PATCH 5/5] Revert reordering of sound_syllable and pronounce --- pythainlp/corpus/__init__.py | 5 +++-- pythainlp/util/__init__.py | 32 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 6271cd35b..6d04c4593 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -87,7 +87,8 @@ def corpus_db_path() -> str: """ return _CORPUS_DB_PATH -# DO NOT REORDER THESE pythainlp.corpus imports +# DO NOT REORDER these pythainlp.corpus imports. +# These imports must come before other pythainlp.corpus.* imports from pythainlp.corpus.core import ( download, get_corpus, @@ -101,7 +102,7 @@ def corpus_db_path() -> str: make_safe_directory_name, path_pythainlp_corpus, remove, -) # these imports must come before other pythainlp.corpus.* imports +) from pythainlp.corpus.common import ( countries, find_synonyms, diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index 071fc5f31..3f1eb8520 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -109,26 +109,11 @@ ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa -from pythainlp.util.pronounce import ( - rhyme, - spelling, - thai_consonant_to_spelling, - tone_to_spelling, -) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) from pythainlp.util.strftime import thai_strftime -# sound_syllable and pronounce have to be imported last, -# to prevent circular import issues. -# Other imports should be above this line, sorted. -from pythainlp.util.syllable import ( - sound_syllable, - syllable_length, - syllable_open_close_detector, - tone_detector, -) from pythainlp.util.thai import ( analyze_thai_text, count_thai_chars, @@ -143,3 +128,20 @@ from pythainlp.util.time import thaiword_to_time, time_to_thaiword from pythainlp.util.trie import Trie, dict_trie from pythainlp.util.wordtonum import text_to_num, thaiword_to_num, words_to_num + +# DO NOT REORDER these imports. +# sound_syllable and pronounce have to be imported last, +# to prevent circular import issues. +# Other imports should be above this line, sorted. +from pythainlp.util.syllable import ( + sound_syllable, + syllable_length, + syllable_open_close_detector, + tone_detector, +) +from pythainlp.util.pronounce import ( + rhyme, + spelling, + thai_consonant_to_spelling, + tone_to_spelling, +)