diff --git a/CHANGELOG.md b/CHANGELOG.md index 84357afbe..3cf642ba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ and this project adheres to - Full release notes: - Commit history: +## [Unreleased] + +### Changed + +- Fix doctests, docstring formatting, typos, and outdated content + across all modules (#1392). + ## [5.3.3] - 2026-03-26 Security fixes and thai2rom_onnx bug fixes. diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 0dc89385d..36b3184c7 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -29,28 +29,24 @@ def aksonhan_to_current(word: str) -> str: """Convert AksonHan words to current Thai words - AksonHan (อักษรหัน) writes down two consonants for the \ - spelling of the /a/ vowels. (สระ อะ). + AksonHan (อักษรหัน) writes two consonants to spell + the short /a/ vowel (สระ อะ). - Today, รร is an aksonHan word that is still used in Thai. + Today, รร is an aksonhan pattern still used in Thai. :param str word: Thai word :return: Thai AksonHan to be converted to current Thai word :rtype: str :Example: - :: - from pythainlp.ancient import aksonhan_to_current - - print(aksonhan_to_current("จกก")) - # output: จัก - - print(aksonhan_to_current("บงงคบบ")) - # output: บังคับ - - print(aksonhan_to_current("สรรเพชญ")) # รร is still used. - # output: สรรเพชญ + >>> from pythainlp.ancient import aksonhan_to_current + >>> print(aksonhan_to_current("จกก")) + จัก + >>> print(aksonhan_to_current("บงงคบบ")) + บังคับ + >>> print(aksonhan_to_current("สรรเพชญ")) # รร is still used. + สรรเพชญ """ if len(word) < 3: diff --git a/pythainlp/ancient/currency.py b/pythainlp/ancient/currency.py index 474549735..4b89e0c12 100644 --- a/pythainlp/ancient/currency.py +++ b/pythainlp/ancient/currency.py @@ -26,22 +26,10 @@ def convert_currency(value: float, from_unit: str) -> dict[str, float]: :rtype: dict[str, float] :Example: - :: - from pythainlp.ancient import convert_currency - - print(convert_currency(8, "บาท")) - # output: - # { - # 'เบี้ย': 51200.0, - # 'อัฐ': 512.0, - # 'ไพ': 256.0, - # 'เฟื้อง': 64.0, - # 'สลึง': 32.0, - # 'บาท': 8.0, - # 'ตำลึง': 2.0, - # 'ชั่ง': 0.1 - # } + >>> from pythainlp.ancient import convert_currency + >>> print(convert_currency(8, "บาท")) + {'เบี้ย': 51200.0, 'อัฐ': 512.0, 'ไพ': 256.0, 'เฟื้อง': 64.0, 'สลึง': 32.0, 'บาท': 8.0, 'ตำลึง': 2.0, 'ชั่ง': 0.1} """ conversion_factors_to_att = { @@ -57,7 +45,7 @@ def convert_currency(value: float, from_unit: str) -> dict[str, float]: if from_unit not in conversion_factors_to_att: raise NotImplementedError( - f"Currency unit '{from_unit}' is not support." + f"Currency unit '{from_unit}' is not supported." ) # start from 'อัฐ' diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index cd5be553e..2825ebe46 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -80,14 +80,13 @@ def augment( :rtype: List[str] :Example: - :: - from pythainlp.augment.lm import ThaiTextAugmenter + >>> from pythainlp.augment.lm import ThaiTextAugmenter # doctest: +SKIP - aug = ThaiTextAugmenter() - aug.augment("ช้างมีทั้งหมด 50 ตัว บน", num_args=5) + >>> aug = ThaiTextAugmenter() # doctest: +SKIP + >>> aug.augment("ช้างมีทั้งหมด 50 ตัว บน", num_args=5) # doctest: +SKIP - # output = ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้ครับ.', + ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้ครับ.', 'ช้างมีทั้งหมด 50 ตัว บนพื้นดินครับ...', 'ช้างมีทั้งหมด 50 ตัว บนท้องฟ้าครับ...', 'ช้างมีทั้งหมด 50 ตัว บนดวงจันทร์.‼', diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index 6b7c16399..f593b3a95 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -76,14 +76,13 @@ def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]: :rtype: List[str] :Example: - :: - from pythainlp.augment.lm import Thai2transformersAug + >>> from pythainlp.augment.lm import Thai2transformersAug # doctest: +SKIP - aug = Thai2transformersAug() + >>> aug = Thai2transformersAug() # doctest: +SKIP - aug.augment("ช้างมีทั้งหมด 50 ตัว บน") - # output: ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้', + >>> aug.augment("ช้างมีทั้งหมด 50 ตัว บน") # doctest: +SKIP + ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้', 'ช้างมีทั้งหมด 50 ตัว บนสุด', 'ช้างมีทั้งหมด 50 ตัว บนบก', 'ช้างมีทั้งหมด 50 ตัว บนนั้น', diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index e461c022c..6459932df 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -60,13 +60,12 @@ def augment( :return: list of synonyms :rtype: list[str] :Example: - :: - from pythainlp.augment.word2vec.bpemb_wv import BPEmbAug + >>> from pythainlp.augment.word2vec.bpemb_wv import BPEmbAug # doctest: +SKIP - aug = BPEmbAug() - aug.augment("ผมเรียน", n_sent=2, p=0.5) - # output: ['ผมสอน', 'ผมเข้าเรียน'] + >>> aug = BPEmbAug() # doctest: +SKIP + >>> aug.augment("ผมเรียน", n_sent=2, p=0.5) # doctest: +SKIP + ['ผมสอน', 'ผมเข้าเรียน'] """ self.sentence: str = sentence.replace(" ", "▁") self.temp: list[tuple[str, ...]] = self.aug.augment( diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 10745a078..6f9819c41 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -56,12 +56,11 @@ def augment( :rtype: List[Tuple[str]] :Example: - :: - from pythainlp.augment.word2vec import LTW2VAug + >>> from pythainlp.augment.word2vec import LTW2VAug # doctest: +SKIP - aug = LTW2VAug() - aug.augment("ผมเรียน", n_sent=2, p=0.5) - # output: [('เขา', 'เรียนหนังสือ'), ('เขา', 'สมัครเรียน')] + >>> aug = LTW2VAug() # doctest: +SKIP + >>> aug.augment("ผมเรียน", n_sent=2, p=0.5) # doctest: +SKIP + [('เขา', 'เรียนหนังสือ'), ('เขา', 'สมัครเรียน')] """ return self.aug.augment(sentence, n_sent, p) diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 5f34937af..36ee186ec 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -57,12 +57,11 @@ def augment( :rtype: List[Tuple[str]] :Example: - :: - from pythainlp.augment.word2vec import Thai2fitAug + >>> from pythainlp.augment.word2vec import Thai2fitAug # doctest: +SKIP - aug = Thai2fitAug() - aug.augment("ผมเรียน", n_sent=2, p=0.5) - # output: [('พวกเรา', 'เรียน'), ('ฉัน', 'เรียน')] + >>> aug = Thai2fitAug() # doctest: +SKIP + >>> aug.augment("ผมเรียน", n_sent=2, p=0.5) # doctest: +SKIP + [('พวกเรา', 'เรียน'), ('ฉัน', 'เรียน')] """ return self.aug.augment(sentence, n_sent, p) diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index a5078bd3d..250b22a00 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -191,13 +191,12 @@ def augment( :rtype: list[list[str]] :Example: - :: - from pythainlp.augment import WordNetAug + >>> from pythainlp.augment import WordNetAug # doctest: +SKIP - aug = WordNetAug() - aug.augment("เราชอบไปโรงเรียน") - # output: [('เรา', 'ชอบ', 'ไป', 'ร.ร.'), + >>> aug = WordNetAug() # doctest: +SKIP + >>> aug.augment("เราชอบไปโรงเรียน") # doctest: +SKIP + [('เรา', 'ชอบ', 'ไป', 'ร.ร.'), ('เรา', 'ชอบ', 'ไป', 'รร.'), ('เรา', 'ชอบ', 'ไป', 'โรงเรียน'), ('เรา', 'ชอบ', 'ไป', 'อาคารเรียน'), diff --git a/pythainlp/benchmarks/metrics.py b/pythainlp/benchmarks/metrics.py index ae793c181..9c9d33c54 100644 --- a/pythainlp/benchmarks/metrics.py +++ b/pythainlp/benchmarks/metrics.py @@ -131,24 +131,22 @@ def bleu_score( :rtype: BleuScore :Example: - :: - from pythainlp.benchmarks import bleu_score - - references = ["สวัสดีครับ วันนี้อากาศดีมาก"] - hypotheses = ["สวัสดีค่ะ วันนี้อากาศดี"] - score = bleu_score(references, hypotheses) - print(f"BLEU score: {score['bleu']:.2f}") - - :: - - # Multiple references per hypothesis - references = [ - ["สวัสดีครับ", "สวัสดีค่ะ"], # two refs for first hypothesis - ["ลาก่อนครับ", "ลาก่อนค่ะ"], # two refs for second hypothesis - ] - hypotheses = ["สวัสดี", "ลาก่อน"] - score = bleu_score(references, hypotheses) + >>> from pythainlp.benchmarks import bleu_score + + >>> references = ["สวัสดีครับ วันนี้อากาศดีมาก"] + >>> hypotheses = ["สวัสดีค่ะ วันนี้อากาศดี"] + >>> score = bleu_score(references, hypotheses) + >>> print(f"BLEU score: {score['bleu']:.2f}") + BLEU score: 28.12 + + >>> # Multiple references per hypothesis + >>> references = [ + ... ["สวัสดีครับ", "สวัสดีค่ะ"], # two refs for first hypothesis + ... ["ลาก่อนครับ", "ลาก่อนค่ะ"], # two refs for second hypothesis + ... ] + >>> hypotheses = ["สวัสดี", "ลาก่อน"] + >>> score = bleu_score(references, hypotheses) """ from pythainlp.tokenize import word_tokenize @@ -282,16 +280,18 @@ def rouge_score( :rtype: dict[str, RougeScore] :Example: - :: - - from pythainlp.benchmarks import rouge_score - reference = "สวัสดีครับ วันนี้อากาศดีมาก" - hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" - scores = rouge_score(reference, hypothesis) - print(f"ROUGE-1 F-measure: {scores['rouge1']['fmeasure']:.4f}") - print(f"ROUGE-2 F-measure: {scores['rouge2']['fmeasure']:.4f}") - print(f"ROUGE-L F-measure: {scores['rougeL']['fmeasure']:.4f}") + >>> from pythainlp.benchmarks import rouge_score + + >>> reference = "สวัสดีครับ วันนี้อากาศดีมาก" + >>> hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" + >>> scores = rouge_score(reference, hypothesis) + >>> print(f"ROUGE-1 F-measure: {scores['rouge1']['fmeasure']:.4f}") + ROUGE-1 F-measure: 0.6000 + >>> print(f"ROUGE-2 F-measure: {scores['rouge2']['fmeasure']:.4f}") + ROUGE-2 F-measure: 0.2500 + >>> print(f"ROUGE-L F-measure: {scores['rougeL']['fmeasure']:.4f}") + ROUGE-L F-measure: 0.6000 """ from pythainlp.tokenize import word_tokenize @@ -391,14 +391,14 @@ def word_error_rate( :rtype: float :Example: - :: - from pythainlp.benchmarks import word_error_rate + >>> from pythainlp.benchmarks import word_error_rate - reference = "สวัสดีครับ วันนี้อากาศดีมาก" - hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" - wer = word_error_rate(reference, hypothesis) - print(f"WER: {wer:.4f}") + >>> reference = "สวัสดีครับ วันนี้อากาศดีมาก" + >>> hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" + >>> wer = word_error_rate(reference, hypothesis) + >>> print(f"WER: {wer:.4f}") + WER: 0.4000 """ from pythainlp.tokenize import word_tokenize @@ -469,14 +469,14 @@ def character_error_rate( :rtype: float :Example: - :: - from pythainlp.benchmarks import character_error_rate + >>> from pythainlp.benchmarks import character_error_rate - reference = "สวัสดีครับ" - hypothesis = "สวัสดีค่ะ" - cer = character_error_rate(reference, hypothesis) - print(f"CER: {cer:.4f}") + >>> reference = "สวัสดีครับ" + >>> hypothesis = "สวัสดีค่ะ" + >>> cer = character_error_rate(reference, hypothesis) + >>> print(f"CER: {cer:.4f}") + CER: 0.3000 """ # Work with characters directly (no tokenization needed) ref_chars = list(reference) diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index e0475657c..0d2d1b90e 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -91,21 +91,22 @@ def _flatten_result( my_dict: Any, sep: str = ":", ) -> dict[str, Union[int, str]]: - """Flatten two-dimension dictionary. + """Flatten a two-level dictionary. - Use keys in the first dimension as a prefix for keys in the second dimension. - For example, - my_dict = { "a": { "b": 7 } } - flatten(my_dict) - { "a:b": 7 } + Uses keys from the first level as a prefix for keys in the second level. + For example:: + my_dict = {"a": {"b": 7}} + _flatten_result(my_dict) + # {"a:b": 7} :param my_dict: dictionary containing stats :type my_dict: TokenizationStat or - collections.abc.Mapping[str, collections.abc.Mapping[str, Union[int, str]]] - :param str sep: separator between the two keys (default: ":") + collections.abc.Mapping[str, + collections.abc.Mapping[str, Union[int, str]]] + :param str sep: separator between the two keys (default: ``":"``) - :return: a one-dimension dictionary with keys combined + :return: a flat dictionary with combined keys :rtype: dict[str, Union[int, str]] """ return { @@ -118,13 +119,13 @@ def _flatten_result( def benchmark(ref_samples: list[str], samples: list[str]) -> "pd.DataFrame": """Performance benchmarking for samples. - Please see :meth:`pythainlp.benchmarks.word_tokenization.compute_stats` for - the computed metrics. + See :func:`pythainlp.benchmarks.word_tokenization.compute_stats` + for computed metrics. - :param list[str] ref_samples: ground truth for samples - :param list[str] samples: samples that we want to evaluate + :param list[str] ref_samples: ground truth + :param list[str] samples: samples to evaluate - :return: dataframe with row x col = len(samples) x len(metrics) + :return: dataframe with shape ``len(samples) × len(metrics)`` :rtype: pandas.DataFrame """ import pandas as pd @@ -186,22 +187,22 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: def compute_stats( ref_sample: str, raw_sample: str ) -> TokenizationStat: - """Compute statistics for tokenization quality + """Compute statistics for tokenization quality. These statistics include: - **Character-Level**: - True Positive, False Positive, True Negative, False Negative, Precision, Recall, and f1 - **Word-Level**: - Precision, Recall, and f1 - **Other**: - - Correct tokenization indicator: {0, 1} sequence indicating that the corresponding - word is tokenized correctly. + **Character-level**: + True Positive, False Positive, True Negative, False Negative + **Word-level**: + Precision, Recall, and F1 + **Global**: + A ``{0, 1}`` sequence indicating whether each word + is tokenized correctly. - :param str ref_sample: ground truth for samples - :param str samples: samples that we want to evaluate + :param str ref_sample: ground truth sample + :param str raw_sample: sample to evaluate - :return: metrics at character- and word-level and indicators of correctly tokenized words + :return: character-level, word-level, and global tokenization metrics :rtype: TokenizationStat """ import numpy as np @@ -323,10 +324,12 @@ def _find_words_correctly_tokenized( ) -> tuple[int, ...]: """Find whether each word is correctly tokenized. - :param list[tuple[int, int]] ref_boundaries: word boundaries of reference tokenization - :param list[tuple[int, int]] predicted_boundaries: word boundaries of predicted tokenization + :param list[tuple[int, int]] ref_boundaries: word boundaries of + the reference tokenization + :param list[tuple[int, int]] predicted_boundaries: word boundaries of + the predicted tokenization - :return: binary sequence where 1 indicates the corresponding word is tokenized correctly + :return: binary sequence; 1 means the word is tokenized correctly :rtype: tuple[int, ...] """ ref_b = dict(zip(ref_boundaries, [1] * len(ref_boundaries))) diff --git a/pythainlp/braille/core.py b/pythainlp/braille/core.py index 682415630..88a1ce0fa 100644 --- a/pythainlp/braille/core.py +++ b/pythainlp/braille/core.py @@ -191,15 +191,12 @@ def thai_word_braille(word: str) -> str: :rtype: str :Example: - :: - from pythainlp.braille import thai_word_braille - - thai_word_braille("กก") - # Output: '⠛⠛' - - thai_word_braille("น้ำ") - # Output: '⠝⠲⠵' + >>> from pythainlp.braille import thai_word_braille + >>> thai_word_braille("กก") + '⠛⠛' + >>> thai_word_braille("น้ำ") + '⠝⠲⠵' """ if not word: return "" @@ -228,12 +225,10 @@ def thai_text_braille(text: str) -> list[str]: :rtype: list[str] :Example: - :: - - from pythainlp.braille import thai_text_braille - thai_text_braille("สวัสดี ครับ") - # Output: ['⠎⠺⢂⠎⠤⠙⢒', '⠅⠗⢢⠃'] + >>> from pythainlp.braille import thai_text_braille + >>> thai_text_braille("สวัสดี ครับ") + ['⠎⠺⠜⠎⠙⠆', ' ', '⠥⠗⠜⠧'] """ _list_braille: list[str] = [] for token in word_tokenize(text): diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index f8b034173..c9d46a19a 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -71,19 +71,18 @@ def chat(self, text: str) -> str: :return: answer from chatbot. :rtype: str :Example: - :: - from pythainlp.chat import ChatBotModel - import torch + >>> from pythainlp.chat import ChatBotModel # doctest: +SKIP + >>> import torch # doctest: +SKIP - chatbot = ChatBotModel() - chatbot.load_model(device="cpu", torch_dtype=torch.bfloat16) + >>> chatbot = ChatBotModel() # doctest: +SKIP + >>> chatbot.load_model(device="cpu", torch_dtype=torch.bfloat16) # doctest: +SKIP - print(chatbot.chat("สวัสดี")) - # output: ยินดีที่ได้รู้จัก + >>> print(chatbot.chat("สวัสดี")) # doctest: +SKIP + ยินดีที่ได้รู้จัก - print(chatbot.history) - # output: [('สวัสดี', 'ยินดีที่ได้รู้จัก')] + >>> print(chatbot.history) # doctest: +SKIP + [('สวัสดี', 'ยินดีที่ได้รู้จัก')] """ _temp = "" if self.history: diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index e94b65ef0..535a780b1 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -55,24 +55,23 @@ def predict(self, x1: str, k: int = 1) -> str: :rtype: str :Example: - :: - - from pythainlp.classify import GzipModel - - training_data = [ - ("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"), - ("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"), - ("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"), - ("ขับรถแย่มาก", "Negative"), - ("ดีนะครับ", "Positive"), - ("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"), - ("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"), - ("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"), - ("นี่เป็นบทความหนึ่ง", "Neutral"), - ] - model = GzipModel(training_data) - print(model.predict("ฉันดีใจ", k=1)) - # output: Positive + + >>> from pythainlp.classify import GzipModel # doctest: +SKIP + + >>> training_data = [ # doctest: +SKIP + ... ("รายละเอียดตามนี้เลยค่าา ^^", "Neutral"), + ... ("กลัวพวกมึงหาย อดกินบาบิก้อน", "Neutral"), + ... ("บริการแย่มากก เป็นหมอได้ไง😤", "Negative"), + ... ("ขับรถแย่มาก", "Negative"), + ... ("ดีนะครับ", "Positive"), + ... ("ลองแล้วรสนี้อร่อย... ชอบๆ", "Positive"), + ... ("ฉันรู้สึกโกรธ เวลามือถือแบตหมด", "Negative"), + ... ("เธอภูมิใจที่ได้ทำสิ่งดี ๆ และดีใจกับเด็ก ๆ", "Positive"), + ... ("นี่เป็นบทความหนึ่ง", "Neutral"), + ... ] + >>> model = GzipModel(training_data) # doctest: +SKIP + >>> print(model.predict("ฉันดีใจ", k=1)) # doctest: +SKIP + Positive """ import numpy as np @@ -95,7 +94,10 @@ def predict(self, x1: str, k: int = 1) -> str: return predict_class def save(self, path: str) -> None: - """:param str path: path to save model""" + """Save model to file. + + :param str path: path to save model + """ with open(path, "w", encoding="utf-8") as f: json.dump( { @@ -107,7 +109,10 @@ def save(self, path: str) -> None: ) def load(self, path: str) -> None: - """:param str path: path to load model""" + """Load model from file. + + :param str path: path to load model from + """ import numpy as np with open(path, "r", encoding="utf-8") as f: diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index 9eb5a57cd..9228332cf 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Command line for PyThaiNLP's benchmarking tools.""" from __future__ import annotations diff --git a/pythainlp/cli/misspell.py b/pythainlp/cli/misspell.py index fd582e278..871295860 100644 --- a/pythainlp/cli/misspell.py +++ b/pythainlp/cli/misspell.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Command line for PyThaiNLP's misspelling generator.""" from __future__ import annotations diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 2776b5786..997a9de77 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -29,21 +29,19 @@ def coreference_resolution( by PyThaiNLP v1.0 :Example: - :: - from pythainlp.coref import coreference_resolution - - print( - coreference_resolution( - ["Bill Gates ได้รับวัคซีน COVID-19 เข็มแรกแล้ว ระบุ ผมรู้สึกสบายมาก"] - ) - ) - # output: - # [ - # {'text': 'Bill Gates ได้รับวัคซีน COVID-19 เข็มแรกแล้ว ระบุ ผมรู้สึกสบายมาก', - # 'clusters_string': [['Bill Gates', 'ผม']], - # 'clusters': [[(0, 10), (50, 52)]]} - # ] + >>> from pythainlp.coref import coreference_resolution # doctest: +SKIP + + >>> print( # doctest: +SKIP + ... coreference_resolution( + ... ["Bill Gates ได้รับวัคซีน COVID-19 เข็มแรกแล้ว ระบุ ผมรู้สึกสบายมาก"] + ... ) + ... ) + [ + {'text': 'Bill Gates ได้รับวัคซีน COVID-19 เข็มแรกแล้ว ระบุ ผมรู้สึกสบายมาก', + 'clusters_string': [['Bill Gates', 'ผม']], + 'clusters': [[(0, 10), (50, 52)]]} + ] """ if isinstance(texts, str): texts = [texts] diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index d177cba3f..c1f471bf1 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -91,8 +91,8 @@ def provinces( ) -> Union[frozenset[str], list[dict[str, str]]]: """Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". - \n(See: `dev/pythainlp/corpus/thailand_provinces_th.txt\ - `_) + \n(See: `dev/pythainlp/corpus/thailand_provinces_th.csv\ + `_) :param bool details: return details of provinces or not @@ -181,7 +181,7 @@ def thai_words() -> frozenset[str]: def thai_orst_words() -> frozenset[str]: """Return a frozenset of Thai words from Royal Society of Thailand - \n(See: `dev/pythainlp/corpus/thai_orst_words.txt\ + \n(See: `dev/pythainlp/corpus/orst_words_th.txt\ `_) :return: :class:`frozenset` containing words in the Thai language. @@ -445,16 +445,14 @@ def find_synonyms(word: str) -> list[str]: """Find synonyms :param str word: Thai word - :return: List of synonyms of the input word or an empty list if it isn't exist. - :rtype: List[str] + :return: list of synonyms of the input word, or an empty list if none exist. + :rtype: list[str] :Example: - :: - from pythainlp.corpus import find_synonyms - - print(find_synonyms("หมู")) - # output: ['จรุก', 'วราหะ', 'วราห์', 'ศูกร', 'สุกร'] + >>> from pythainlp.corpus import find_synonyms # doctest: +SKIP + >>> print(find_synonyms("หมู")) # doctest: +SKIP + ['จรุก', 'วราหะ', 'วราห์', 'ศูกร', 'สุกร'] """ synonyms = thai_synonyms() # get a dictionary of {word, synonym} list_synonym: list[Any] = [] diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 70f0fa7e0..9da5930bb 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -126,45 +126,16 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset[str]: :rtype: :class:`frozenset` :Example: - :: - - from pythainlp.corpus import get_corpus - - # input file (negations_th.txt): - # แต่ - # ไม่ - - get_corpus("negations_th.txt") - # output: - # frozenset({'แต่', 'ไม่'}) - - # input file (ttc_freq.txt): - # ตัวบท10 - # โดยนัยนี้1 - - get_corpus("ttc_freq.txt") - # output: - # frozenset({'โดยนัยนี้\\t1', - # 'ตัวบท\\t10', - # ...}) - - # input file (icubrk_th.txt): - # # Thai Dictionary for ICU BreakIterator - # กก - # กกขนาก - - get_corpus("icubrk_th.txt") - # output: - # frozenset({'กกขนาก', - # '# Thai Dictionary for ICU BreakIterator', - # 'กก', - # ...}) - - get_corpus("icubrk_th.txt", comments=False) - # output: - # frozenset({'กกขนาก', - # 'กก', - # ...}) + + >>> from pythainlp.corpus import get_corpus # doctest: +SKIP + >>> get_corpus("negations_th.txt") # doctest: +SKIP + frozenset({'แต่', 'ไม่'}) + >>> get_corpus("ttc_freq.txt") # doctest: +SKIP + frozenset({'โดยนัยนี้\\t1', 'ตัวบท\\t10', ...}) + >>> get_corpus("icubrk_th.txt") # doctest: +SKIP + frozenset({'กกขนาก', '# Thai Dictionary for ICU BreakIterator', 'กก', ...}) + >>> get_corpus("icubrk_th.txt", comments=False) # doctest: +SKIP + frozenset({'กกขนาก', 'กก', ...}) """ corpus_files = files("pythainlp.corpus") @@ -195,17 +166,10 @@ def get_corpus_as_is(filename: str) -> list[str]: :rtype: :class:`list` :Example: - :: - - from pythainlp.corpus import get_corpus - # input file (negations_th.txt): - # แต่ - # ไม่ - - get_corpus_as_is("negations_th.txt") - # output: - # ['แต่', 'ไม่'] + >>> from pythainlp.corpus import get_corpus_as_is # doctest: +SKIP + >>> get_corpus_as_is("negations_th.txt") # doctest: +SKIP + ['แต่', 'ไม่'] """ corpus_files = files("pythainlp.corpus") corpus_file = corpus_files.joinpath(filename) @@ -300,28 +264,23 @@ def get_corpus_path(name: str, version: str = "") -> Optional[str]: (Please see the filename in `this file `_) - If the corpus already exists:: - - from pythainlp.corpus import get_corpus_path + If the corpus already exists: - print(get_corpus_path("ttc")) - # output: /root/pythainlp-data/ttc_freq.txt + >>> from pythainlp.corpus import get_corpus_path # doctest: +SKIP + >>> get_corpus_path("ttc") # doctest: +SKIP + '/root/pythainlp-data/ttc_freq.txt' - If the corpus has not been downloaded yet (online mode):: + If the corpus has not been downloaded yet (online mode): - from pythainlp.corpus import get_corpus_path + >>> get_corpus_path("wiki_lm_lstm") # doctest: +SKIP + '/root/pythainlp-data/thwiki_model_lstm.pth' - print(get_corpus_path("wiki_lm_lstm")) - # output: /root/pythainlp-data/thwiki_model_lstm.pth - # (downloads automatically on first call) + To download manually: - To download manually:: - - from pythainlp.corpus import download, get_corpus_path - - download("wiki_lm_lstm") - print(get_corpus_path("wiki_lm_lstm")) - # output: /root/pythainlp-data/thwiki_model_lstm.pth + >>> from pythainlp.corpus import download # doctest: +SKIP + >>> download("wiki_lm_lstm") # doctest: +SKIP + >>> get_corpus_path("wiki_lm_lstm") # doctest: +SKIP + '/root/pythainlp-data/thwiki_model_lstm.pth' """ CUSTOMIZE: dict[str, str] = { # "the corpus name":"path" @@ -378,8 +337,8 @@ def get_corpus_path(name: str, version: str = "") -> Optional[str]: def _download(url: str, dst: str) -> int: """Download helper. - @param: URL for downloading file - @param: dst place to put the file into + :param str url: URL for the file to download. + :param str dst: local destination path for the downloaded file. Security Note: Downloads use HTTPS with SSL certificate validation. Files are verified using MD5 checksums after download. @@ -415,8 +374,8 @@ def _download(url: str, dst: str) -> int: def _check_hash(dst: str, md5: str) -> None: """Check hash helper. - @param: dst place to put the file into - @param: md5 place to file hash (MD5) + :param str dst: local path of the file to verify. + :param str md5: expected MD5 checksum of the file. """ if md5 and md5 != "-": import hashlib @@ -433,16 +392,18 @@ def _check_hash(dst: str, md5: str) -> None: def _is_within_directory(directory: str, target: str) -> bool: """Check if target path is within directory (prevent path traversal). - @param: directory base directory path - @param: target target file path to check - @return: True if target is within directory, False otherwise + :param str directory: base directory path. + :param str target: target file path to check. + :return: ``True`` if target is within directory, ``False`` otherwise. + :rtype: bool Security Note: This function normalizes paths using os.path.abspath() to handle relative paths and .. sequences. It does NOT follow symlinks (unlike os.path.realpath()), because: - - Symlink validation is handled separately in extraction functions - - We want to check if the path string itself is safe, not where it points - - This prevents false negatives when symlinks don't exist yet + + - Symlink validation is handled separately in extraction functions. + - We want to check if the path string itself is safe, not where it points. + - This prevents false negatives when symlinks don't exist yet. For symlink security, use the extraction function's symlink validation. """ @@ -463,15 +424,16 @@ def _is_within_directory(directory: str, target: str) -> bool: def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: """Safely extract tar archive, preventing path traversal attacks. - @param: tar tarfile object - @param: path destination path for extraction + :param tarfile.TarFile tar: tarfile object to extract. + :param str path: destination path for extraction. Security Note: This function prevents path traversal attacks including: - - Files with .. in their path - - Symlinks pointing outside the extraction directory - - Files extracted through malicious symlinks - For Python 3.12+, uses tarfile.data_filter for additional protection. + - Files with ``..`` in their path. + - Symlinks pointing outside the extraction directory. + - Files extracted through malicious symlinks. + + For Python 3.12+, uses ``tarfile.data_filter`` for additional protection. For Python 3.9-3.11, implements custom validation of all members. """ # Check if data_filter is available (Python 3.12+) @@ -528,12 +490,13 @@ def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: """Safely extract zip archive, preventing path traversal attacks. - @param: zip_file zipfile object - @param: path destination path for extraction + :param zipfile.ZipFile zip_file: zipfile object to extract. + :param str path: destination path for extraction. Security Note: This function prevents path traversal attacks including: - - Files with .. in their path - - Symlinks pointing outside the extraction directory (on Unix systems) + + - Files with ``..`` in their path. + - Symlinks pointing outside the extraction directory (on Unix systems). Note: ZIP format has limited symlink support. Symlinks are primarily created by Unix-based archiving tools and may not be portable. @@ -663,15 +626,12 @@ def download( :rtype: bool :Example: - :: - - from pythainlp.corpus import download - download("wiki_lm_lstm", force=True) - # output: - # Corpus: wiki_lm_lstm - # - Downloading: wiki_lm_lstm 0.1 - # thwiki_lm.pth: 26%|██▌ | 114k/434k [00:00<00:00, 690kB/s] + >>> from pythainlp.corpus import download # doctest: +SKIP + >>> download("wiki_lm_lstm", force=True) # doctest: +SKIP + Corpus: wiki_lm_lstm + - Downloading: wiki_lm_lstm 0.1 + ... By default, downloaded corpora and models will be saved in ``$HOME/pythainlp-data/`` @@ -810,20 +770,12 @@ def remove(name: str) -> bool: :rtype: bool :Example: - :: - - from pythainlp.corpus import remove, get_corpus_path, get_corpus - - print(remove("ttc")) - # output: True - - print(get_corpus_path("ttc")) - # output: None - get_corpus("ttc") - # output: - # FileNotFoundError: [Errno 2] No such file or directory: - # '/usr/local/lib/python3.6/dist-packages/pythainlp/corpus/ttc' + >>> from pythainlp.corpus import remove, get_corpus_path # doctest: +SKIP + >>> remove("ttc") # doctest: +SKIP + True + >>> get_corpus_path("ttc") # doctest: +SKIP + None """ if is_read_only_mode(): print("PyThaiNLP is in read-only mode. It cannot remove corpus.") diff --git a/pythainlp/corpus/phupha.py b/pythainlp/corpus/phupha.py index 6711371f3..1c0f94d3e 100644 --- a/pythainlp/corpus/phupha.py +++ b/pythainlp/corpus/phupha.py @@ -36,13 +36,11 @@ def word_freqs() -> list[tuple[str, int]]: :rtype: list[tuple[str, int]] :Example: - :: - from pythainlp.corpus import phupha - - freqs = phupha.word_freqs() - print(freqs[:5]) - # output: [('น', 1119315948), ('ร', 1066483406), ...] + >>> from pythainlp.corpus import phupha # doctest: +SKIP + >>> freqs = phupha.word_freqs() # doctest: +SKIP + >>> print(freqs[:5]) # doctest: +SKIP + [('น', 1119315948), ('ร', 1066483406), ...] **Dataset Citation:** @@ -67,13 +65,10 @@ def unigram_word_freqs() -> dict[str, int]: :rtype: dict[str, int] :Example: - :: - - from pythainlp.corpus import phupha - freqs = phupha.unigram_word_freqs() - print(freqs.get("ไทย", 0)) - # output: frequency count for 'ไทย' + >>> from pythainlp.corpus import phupha # doctest: +SKIP + >>> freqs = phupha.unigram_word_freqs() # doctest: +SKIP + >>> freqs.get("ไทย", 0) # doctest: +SKIP **Dataset Citation:** diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 77ea39014..510bcfefe 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -22,11 +22,14 @@ def word_freqs() -> list[tuple[str, int]]: - """Get word frequency from Thai National Corpus (TNC) - \n(See: `dev/pythainlp/corpus/tnc_freq.txt\ + """Get word frequency from Thai National Corpus (TNC). + + (See: `dev/pythainlp/corpus/tnc_freq.txt `_) - Credit: Korakot Chaovavanich https://www.facebook.com/groups/thainlp/posts/434330506948445 + :See Also: + * Korakot Chaovavanich. + https://www.facebook.com/groups/thainlp/posts/434330506948445 """ freqs: list[tuple[str, int]] = [] for line in get_corpus(_UNIGRAM_FILENAME): diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index 5f7e568d2..61d940843 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -39,11 +39,11 @@ def find_badwords( """Find words that do not work well with the `tokenize` function for the provided `training_data`. - :param Callable[[str], List[str]] tokenize: a tokenize function + :param Callable[[str], list[str]] tokenize: a tokenize function :param Iterable[Iterable[str]] training_data: tokenized text, to be used\ as a training set - :return: words that are considered to make `tokenize` perform badly - :rtype: Set[str] + :return: words that do not work well with the `tokenize` function + :rtype: set[str] """ right: Counter[str] = Counter() wrong: Counter[str] = Counter() @@ -77,41 +77,33 @@ def revise_wordset( a dictionary-based `tokenize` function. `orig_words` will be used as a base set for the dictionary. - Words that do not performed well with `training_data` will be removed. + Words that do not perform well with `training_data` will be removed. The remaining words will be returned. - :param Callable[[str], List[str]] tokenize: a tokenize function, can be\ - any function that takes a string as input and returns a List[str] - :param Iterable[str] orig_words: words that used by the tokenize function,\ + :param Callable[[str], list[str]] tokenize: a tokenize function, can be\ + any function that takes a string as input and returns a list[str] + :param Iterable[str] orig_words: words used by the tokenize function,\ will be used as a base for revision :param Iterable[Iterable[str]] training_data: tokenized text, to be used\ as a training set - :return: words that are considered to make `tokenize` perform badly - :rtype: Set[str] - - :Example:: - :: - - from pythainlp.corpus import thai_words - from pythainlp.corpus.util import revise_wordset - from pythainlp.tokenize.longest import segment - - base_words = thai_words() - more_words = { - "ถวิล อุดล", "ทองอินทร์ ภูริพัฒน์", "เตียง ศิริขันธ์", "จำลอง ดาวเรือง" - } - base_words = base_words.union(more_words) - dict_trie = Trie(wordlist) - - tokenize = lambda text: segment(text, dict_trie) - - training_data = [ - [str, str, str. ...], - [str, str, str, str, ...], - ... - ] - - revised_words = revise_wordset(tokenize, wordlist, training_data) + :return: revised set of words with underperforming words removed + :rtype: set[str] + + :Example: + + >>> from pythainlp.corpus import thai_words # doctest: +SKIP + >>> from pythainlp.corpus.util import revise_wordset # doctest: +SKIP + >>> from pythainlp.tokenize.longest import segment # doctest: +SKIP + >>> base_words = thai_words() # doctest: +SKIP + >>> more_words = { # doctest: +SKIP + ... "ถวิล อุดล", "ทองอินทร์ ภูริพัฒน์", + ... "เตียง ศิริขันธ์", "จำลอง ดาวเรือง", + ... } + >>> base_words = base_words.union(more_words) # doctest: +SKIP + >>> dict_trie = Trie(base_words) # doctest: +SKIP + >>> tokenize = lambda text: segment(text, dict_trie) # doctest: +SKIP + >>> training_data = [["word1", "word2"], ["word3", "word4"]] # doctest: +SKIP + >>> revised_words = revise_wordset(tokenize, base_words, training_data) # doctest: +SKIP """ bad_words = find_badwords(tokenize, training_data) return set(orig_words) - bad_words @@ -125,13 +117,13 @@ def revise_newmm_default_wordset( tokenizer for PyThaiNLP. Words from `pythainlp.corpus.thai_words()` will be used as a base set - for the dictionary. Words that do not performed well with `training_data` + for the dictionary. Words that do not perform well with `training_data` will be removed. The remaining words will be returned. :param Iterable[Iterable[str]] training_data: tokenized text, to be used\ as a training set - :return: words that are considered to make `tokenize` perform badly - :rtype: Set[str] + :return: revised set of words with underperforming words removed + :rtype: set[str] """ orig_words = thai_words() trie = Trie(orig_words) diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index 2034190b5..a9ca988ef 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -49,16 +49,15 @@ def get_el( :rtype: Union[list[dict[str, Any]], str] :Example: - :: - from pythainlp.el import EntityLinker + >>> from pythainlp.el import EntityLinker # doctest: +SKIP - el = EntityLinker(device="cuda") - print(el.get_el("จ๊อบเคยเป็นซีอีโอบริษัทแอปเปิล")) - # output: [{'offsets': [11, 23], - # 'lengths': [6, 7], - # 'entities': ['Q484876', 'Q312'], - # 'md_scores': [0.30301809310913086, 0.6399497389793396], - # 'el_scores': [0.7142490744590759, 0.8657019734382629]}] + >>> el = EntityLinker(device="cuda") # doctest: +SKIP + >>> print(el.get_el("จ๊อบเคยเป็นซีอีโอบริษัทแอปเปิล")) # doctest: +SKIP + [{'offsets': [11, 23], + 'lengths': [6, 7], + 'entities': ['Q484876', 'Q312'], + 'md_scores': [0.30301809310913086, 0.6399497389793396], + 'el_scores': [0.7142490744590759, 0.8657019734382629]}] """ return self.model.process_batch(list_text) diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 168a2a9f9..391fc7d47 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -60,23 +60,25 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> Union[list[str], str]: - """:param str start_seq: word to begin sentence with + """Generate a sentence using the unigram model. + + :param str start_seq: word to begin sentence with :param int N: number of words + :param float prob: minimum word probability threshold :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: list[str], str + :rtype: Union[list[str], str] :Example: - :: - from pythainlp.generate import Unigram + >>> from pythainlp.generate import Unigram # doctest: +SKIP - gen = Unigram() + >>> gen = Unigram() # doctest: +SKIP - gen.gen_sentence("แมว") - # output: 'แมวเวลานะนั้น' + >>> gen.gen_sentence("แมว") # doctest: +SKIP + 'แมวเวลานะนั้น' """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here @@ -139,10 +141,10 @@ def __init__(self, name: str = "tnc") -> None: self.words: list[str] = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str) -> float: - """Probability of word + """Compute bigram probability P(t2 | t1). - :param int t1: text 1 - :param int t2: text 2 + :param str t1: first word + :param str t2: second word :return: probability value :rtype: float @@ -161,23 +163,25 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> Union[list[str], str]: - """:param str start_seq: word to begin sentence with + """Generate a sentence using the bigram model. + + :param str start_seq: word to begin sentence with :param int N: number of words + :param float prob: minimum word probability threshold :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: list[str], str + :rtype: Union[list[str], str] :Example: - :: - from pythainlp.generate import Bigram + >>> from pythainlp.generate import Bigram # doctest: +SKIP - gen = Bigram() + >>> gen = Bigram() # doctest: +SKIP - gen.gen_sentence("แมว") - # output: 'แมวไม่ได้รับเชื้อมัน' + >>> gen.gen_sentence("แมว") # doctest: +SKIP + 'แมวไม่ได้รับเชื้อมัน' """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here @@ -236,11 +240,11 @@ def __init__(self, name: str = "tnc") -> None: self.words: list[str] = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str, t3: str) -> float: - """Probability of word + """Compute trigram probability P(t3 | t1, t2). - :param int t1: text 1 - :param int t2: text 2 - :param int t3: text 3 + :param str t1: first word + :param str t2: second word + :param str t3: third word :return: probability value :rtype: float @@ -260,23 +264,26 @@ def gen_sentence( output_str: bool = True, duplicate: bool = False, ) -> Union[list[str], str]: - """:param str start_seq: word to begin sentence with + """Generate a sentence using the trigram model. + + :param start_seq: word or bigram to begin sentence with + :type start_seq: Union[str, tuple[str, str]] :param int N: number of words + :param float prob: minimum word probability threshold :param bool output_str: output as string :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: list[str], str + :rtype: Union[list[str], str] :Example: - :: - from pythainlp.generate import Trigram + >>> from pythainlp.generate import Trigram # doctest: +SKIP - gen = Trigram() + >>> gen = Trigram() # doctest: +SKIP - gen.gen_sentence() - # output: 'ยังทำตัวเป็นเซิร์ฟเวอร์คือ' + >>> gen.gen_sentence() # doctest: +SKIP + 'ยังทำตัวเป็นเซิร์ฟเวอร์คือ' """ late_word: Union[str, tuple[str, str]] if not start_seq: diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 47f2f774b..08492c597 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -139,7 +139,6 @@ def gen_sentence( :rtype: list[str], str :Example: - :: from pythainlp.generate.thai2fit import gen_sentence diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index 1a4cf8c3c..c0fc69306 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -174,23 +174,22 @@ def instruct_generate( :rtype: str :Example: - :: - from pythainlp.generate.wangchanglm import WangChanGLM - import torch + >>> from pythainlp.generate.wangchanglm import WangChanGLM # doctest: +SKIP + >>> import torch # doctest: +SKIP - model = WangChanGLM() + >>> model = WangChanGLM() # doctest: +SKIP - model.load_model(device="cpu", torch_dtype=torch.bfloat16) + >>> model.load_model(device="cpu", torch_dtype=torch.bfloat16) # doctest: +SKIP - print(model.instruct_generate(instruct="ขอวิธีลดน้ำหนัก")) - # output: ลดน้ําหนักให้ได้ผล ต้องทําอย่างค่อยเป็นค่อยไป - # ปรับเปลี่ยนพฤติกรรมการกินอาหาร - # ออกกําลังกายอย่างสม่ําเสมอ - # และพักผ่อนให้เพียงพอ - # ที่สําคัญควรหลีกเลี่ยงอาหารที่มีแคลอรี่สูง - # เช่น อาหารทอด อาหารมัน อาหารที่มีน้ําตาลสูง - # และเครื่องดื่มแอลกอฮอล์ + >>> print(model.instruct_generate(instruct="ขอวิธีลดน้ำหนัก")) # doctest: +SKIP + ลดน้ําหนักให้ได้ผล ต้องทําอย่างค่อยเป็นค่อยไป + ปรับเปลี่ยนพฤติกรรมการกินอาหาร + ออกกําลังกายอย่างสม่ําเสมอ + และพักผ่อนให้เพียงพอ + ที่สําคัญควรหลีกเลี่ยงอาหารที่มีแคลอรี่สูง + เช่น อาหารทอด อาหารมัน อาหารที่มีน้ําตาลสูง + และเครื่องดื่มแอลกอฮอล์ """ if not context: diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index f940e8922..71a0dde2d 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -42,14 +42,13 @@ def check_sara(self, word: str) -> str: :rtype: str :Example: - :: - from pythainlp.khavee import KhaveeVerifier + >>> from pythainlp.khavee import KhaveeVerifier # doctest: +SKIP - kv = KhaveeVerifier() + >>> kv = KhaveeVerifier() # doctest: +SKIP - print(kv.check_sara("เริง")) - # output: 'เออ' + >>> print(kv.check_sara("เริง")) # doctest: +SKIP + 'เออ' """ sara = [] countoa = 0 @@ -224,21 +223,20 @@ def check_sara(self, word: str) -> str: def check_marttra(self, word: str) -> str: """ - Check the Thai spelling Section in the Thai word. + Check the Thai spelling section of the Thai word. :param str word: Thai word - :return: name of spelling Section of the word. + :return: name of the spelling section of the word :rtype: str :Example: - :: - from pythainlp.khavee import KhaveeVerifier + >>> from pythainlp.khavee import KhaveeVerifier # doctest: +SKIP - kv = KhaveeVerifier() + >>> kv = KhaveeVerifier() # doctest: +SKIP - print(kv.check_marttra("สาว")) - # output: 'เกอว' + >>> print(kv.check_marttra("สาว")) # doctest: +SKIP + 'เกอว' """ # Handle consonant clusters ending with ร # ตร, ทร → remove ร (treat as final ต/ท sound) @@ -326,17 +324,16 @@ def is_sumpus(self, word1: str, word2: str) -> bool: :rtype: bool :Example: - :: - from pythainlp.khavee import KhaveeVerifier + >>> from pythainlp.khavee import KhaveeVerifier # doctest: +SKIP - kv = KhaveeVerifier() + >>> kv = KhaveeVerifier() # doctest: +SKIP - print(kv.is_sumpus("สรร", "อัน")) - # output: True + >>> print(kv.is_sumpus("สรร", "อัน")) # doctest: +SKIP + True - print(kv.is_sumpus("สรร", "แมว")) - # output: False + >>> print(kv.is_sumpus("สรร", "แมว")) # doctest: +SKIP + False """ marttra1 = self.check_marttra(word1) marttra2 = self.check_marttra(word2) @@ -394,25 +391,24 @@ def check_klon(self, text: str, k_type: int = 8) -> Union[list[str], str]: :rtype: Union[list[str], str] :Example: - :: - from pythainlp.khavee import KhaveeVerifier + >>> from pythainlp.khavee import KhaveeVerifier # doctest: +SKIP - kv = KhaveeVerifier() + >>> kv = KhaveeVerifier() # doctest: +SKIP - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ - มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: The poem is correct according to the principle. + >>> print(kv.check_klon( # doctest: +SKIP + ... 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ + ... มีคนจับจอง เขาชื่อน้องเธียร', + ... k_type=4 + ... )) + The poem is correct according to the principle. - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ - เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: [ + >>> print(kv.check_klon( # doctest: +SKIP + ... 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ + ... เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', + ... k_type=4 + ... )) + [ "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" ] @@ -640,21 +636,20 @@ def check_aek_too( :rtype: Union[list[bool], List[str], bool, str] :Example: - :: - from pythainlp.khavee import KhaveeVerifier + >>> from pythainlp.khavee import KhaveeVerifier # doctest: +SKIP - kv = KhaveeVerifier() + >>> kv = KhaveeVerifier() # doctest: +SKIP - # การเช็คคำเอกโท - print( - kv.check_aek_too("เอง"), - kv.check_aek_too("เอ่ง"), - kv.check_aek_too("เอ้ง"), - ) - # -> False, aek, too - print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน - # -> [False, 'aek', 'too'] + >>> # การเช็คคำเอกโท + >>> print( # doctest: +SKIP + ... kv.check_aek_too("เอง"), + ... kv.check_aek_too("เอ่ง"), + ... kv.check_aek_too("เอ้ง"), + ... ) + >>> # -> False, aek, too + >>> print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน # doctest: +SKIP + >>> # -> [False, 'aek', 'too'] """ if isinstance(text, list): return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] # type: ignore[misc] @@ -678,8 +673,8 @@ def handle_karun_sound_silence(self, word: str) -> str: by stripping all characters before the 'Karun' character that should be silenced - :param str text: Thai word - :return: Thai word with silent words stripped + :param str word: Thai word + :return: Thai word with silent consonant stripped :rtype: str """ sound_silenced = word.endswith("์") diff --git a/pythainlp/lm/qwen3.py b/pythainlp/lm/qwen3.py index 778ab29dc..82d8ac32b 100644 --- a/pythainlp/lm/qwen3.py +++ b/pythainlp/lm/qwen3.py @@ -40,13 +40,12 @@ def load_model( :param bool low_cpu_mem_usage: low cpu mem usage :Example: - :: - from pythainlp.lm import Qwen3 - import torch + >>> from pythainlp.lm import Qwen3 # doctest: +SKIP + >>> import torch # doctest: +SKIP - model = Qwen3() - model.load_model(device="cpu", torch_dtype=torch.bfloat16) + >>> model = Qwen3() # doctest: +SKIP + >>> model.load_model(device="cpu", torch_dtype=torch.bfloat16) # doctest: +SKIP """ try: import torch @@ -129,16 +128,15 @@ def generate( :rtype: str :Example: - :: - from pythainlp.lm import Qwen3 - import torch + >>> from pythainlp.lm import Qwen3 # doctest: +SKIP + >>> import torch # doctest: +SKIP - model = Qwen3() - model.load_model(device="cpu", torch_dtype=torch.bfloat16) + >>> model = Qwen3() # doctest: +SKIP + >>> model.load_model(device="cpu", torch_dtype=torch.bfloat16) # doctest: +SKIP - result = model.generate("สวัสดี") - print(result) + >>> result = model.generate("สวัสดี") # doctest: +SKIP + >>> print(result) # doctest: +SKIP """ if self.model is None or self.tokenizer is None or self.device is None: raise RuntimeError( @@ -206,17 +204,16 @@ def chat( :rtype: str :Example: - :: - from pythainlp.lm import Qwen3 - import torch + >>> from pythainlp.lm import Qwen3 # doctest: +SKIP + >>> import torch # doctest: +SKIP - model = Qwen3() - model.load_model(device="cpu", torch_dtype=torch.bfloat16) + >>> model = Qwen3() # doctest: +SKIP + >>> model.load_model(device="cpu", torch_dtype=torch.bfloat16) # doctest: +SKIP - messages = [{"role": "user", "content": "สวัสดีครับ"}] - response = model.chat(messages) - print(response) + >>> messages = [{"role": "user", "content": "สวัสดีครับ"}] # doctest: +SKIP + >>> response = model.chat(messages) # doctest: +SKIP + >>> print(response) # doctest: +SKIP """ if self.model is None or self.tokenizer is None or self.device is None: raise RuntimeError( diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 1e6cd6b9d..d2f229160 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -8,13 +8,13 @@ def calculate_ngram_counts( list_words: list[str], n_min: int = 2, n_max: int = 4 ) -> dict[tuple[str, ...], int]: - """Calculates the counts of n-grams in the list words for the specified range. + """Calculate n-gram counts for the given word list. - :param List[str] list_words: List of string - :param int n_min: The minimum n-gram size (default: 2). - :param int n_max: The maximum n-gram size (default: 4). + :param list[str] list_words: list of words + :param int n_min: minimum n-gram size (default: 2) + :param int n_max: maximum n-gram size (default: 4) - :return: A dictionary where keys are n-grams and values are their counts. + :return: dictionary mapping n-grams to their counts :rtype: dict[tuple[str, ...], int] """ if not list_words: @@ -31,20 +31,19 @@ def calculate_ngram_counts( def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]: - """Remove repeated n-grams + """Remove repeated n-grams from a word list. - :param List[str] string_list: List of string + :param list[str] string_list: list of words :param int n: n-gram size - :return: List of string + :return: list of words with repeated n-grams removed :rtype: list[str] :Example: - :: - from pythainlp.lm import remove_repeated_ngrams + >>> from pythainlp.lm import remove_repeated_ngrams # doctest: +SKIP - remove_repeated_ngrams(["เอา", "เอา", "แบบ", "ไหน"], n=1) - # output: ['เอา', 'แบบ', 'ไหน'] + >>> remove_repeated_ngrams(["เอา", "เอา", "แบบ", "ไหน"], n=1) # doctest: +SKIP + ['เอา', 'แบบ', 'ไหน'] """ if not string_list or n <= 0: return string_list diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py index a517003f6..6f8eea39d 100644 --- a/pythainlp/morpheme/thaiwordcheck.py +++ b/pythainlp/morpheme/thaiwordcheck.py @@ -78,26 +78,17 @@ def is_native_thai(word: str) -> bool: :Example: - English word:: - - from pythainlp.morpheme import is_native_thai - - is_native_thai("Avocado") - # output: False - - Native Thai word:: - - is_native_thai("มะม่วง") - # output: True - is_native_thai("ตะวัน") - # output: True - - Non-native Thai word:: - - is_native_thai("สามารถ") - # output: False - is_native_thai("อิสริยาภรณ์") - # output: False + >>> from pythainlp.morpheme import is_native_thai + >>> is_native_thai("Avocado") + False + >>> is_native_thai("มะม่วง") + True + >>> is_native_thai("ตะวัน") + True + >>> is_native_thai("สามารถ") + False + >>> is_native_thai("อิสริยาภรณ์") + False """ if not isinstance(word, str) or not word.strip(): return False diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index 63453a25e..e2711ef70 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -7,29 +7,33 @@ def nighit(w1: str, w2: str) -> str: - """Nighit (นิคหิต or ํ ) is the niggahita in Thai language for create new \ - words from Pali language in Thai. - The function use simple method to create new Thai word from two words \ - that the root is from Pali language. + """Create a new word using Nighit (นิคหิต or ํ). - Read more: https://www.trueplookpanya.com/learning/detail/1180 + Nighit is the niggahita in Thai, used to form new words + from Pali roots. This function applies a simple rule to + combine two Thai words derived from Pali. - :param str w1: A Thai word that has a nighit. - :param str w2: A Thai word. - :return: Thai word. + Reference: https://www.trueplookpanya.com/learning/detail/1180 + + :param str w1: a Thai word ending with a nighit (ํ) + :param str w2: a Thai word + :return: combined Thai word :rtype: str :Example: - :: - - from pythainlp.morpheme import nighit - - assert nighit("สํ","คีต")=="สังคีต" - assert nighit("สํ","จร")=="สัญจร" - assert nighit("สํ","ฐาน")=="สัณฐาน" - assert nighit("สํ","นิษฐาน")=="สันนิษฐาน" - assert nighit("สํ","ปทา")=="สัมปทา" - assert nighit("สํ","โยค")=="สังโยค" + >>> from pythainlp.morpheme import nighit + >>> nighit("สํ", "คีต") + 'สังคีต' + >>> nighit("สํ", "จร") + 'สัญจร' + >>> nighit("สํ", "ฐาน") + 'สัณฐาน' + >>> nighit("สํ", "นิษฐาน") + 'สันนิษฐาน' + >>> nighit("สํ", "ปทา") + 'สัมปทา' + >>> nighit("สํ", "โยค") + 'สังโยค' """ if not str(w1).endswith("ํ") and len(w1) != 2: raise NotImplementedError(f"The function doesn't support {w1}.") diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index 8698b1e3c..6c7f6ad74 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -79,22 +79,19 @@ def dependency_parsing( `Huggingface `_ :Example: - :: - - from pythainlp.parse import dependency_parsing - - print(dependency_parsing("ผมเป็นคนดี", engine="esupar")) - # output: - # 1 ผม _ PRON _ _ 3 nsubj _ SpaceAfter=No - # 2 เป็น _ VERB _ _ 3 cop _ SpaceAfter=No - # 3 คน _ NOUN _ _ 0 root _ SpaceAfter=No - # 4 ดี _ VERB _ _ 3 acl _ SpaceAfter=No - - print(dependency_parsing("ผมเป็นคนดี", engine="spacy_thai")) - # output: - # 1 ผม PRON PPRS _ 2 nsubj _ SpaceAfter=No - # 2 เป็น VERB VSTA _ 0 ROOT _ SpaceAfter=No - # 3 คนดี NOUN NCMN _ 2 obj _ SpaceAfter=No + + >>> from pythainlp.parse import dependency_parsing # doctest: +SKIP + + >>> print(dependency_parsing("ผมเป็นคนดี", engine="esupar")) # doctest: +SKIP + 1 ผม _ PRON _ _ 3 nsubj _ SpaceAfter=No + 2 เป็น _ VERB _ _ 3 cop _ SpaceAfter=No + 3 คน _ NOUN _ _ 0 root _ SpaceAfter=No + 4 ดี _ VERB _ _ 3 acl _ SpaceAfter=No + + >>> print(dependency_parsing("ผมเป็นคนดี", engine="spacy_thai")) # doctest: +SKIP + 1 ผม PRON PPRS _ 2 nsubj _ SpaceAfter=No + 2 เป็น VERB VSTA _ 0 ROOT _ SpaceAfter=No + 3 คนดี NOUN NCMN _ 2 obj _ SpaceAfter=No """ global _tagger, _tagger_name diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index d6485b7fc..d0a368ec5 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -50,6 +50,7 @@ def replace_url(self, text: str) -> str: :return: text where urls are replaced :rtype: str :Example: + >>> replace_url("go to https://github.com") 'go to ' """ @@ -61,6 +62,7 @@ def rm_brackets(self, text: str) -> str: :return: text where all useless brackets are removed :rtype: str :Example: + >>> rm_brackets("hey() whats[;] up{*&} man(hey)") 'hey whats up man(hey)' """ @@ -100,6 +102,7 @@ def replace_newlines(self, text: str) -> str: :return: text where all newlines are replaced with spaces :rtype: str :Example: + >>> rm_useless_spaces("hey whats\n\nup") hey whats up """ @@ -111,6 +114,7 @@ def rm_useless_spaces(self, text: str) -> str: :return: text where all spaces are reduced to one :rtype: str :Example: + >>> rm_useless_spaces("oh no") oh no """ @@ -122,6 +126,7 @@ def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S :return: text where all spaces replaced with _ :rtype: str :Example: + >>> replace_spaces("oh no") oh_no """ @@ -133,6 +138,7 @@ def replace_rep_after(self, text: str) -> str: :return: text with repetitive tokens removed. :rtype: str :Example: + >>> text = "กาาาาาาา" >>> replace_rep_after(text) 'กา' @@ -152,6 +158,7 @@ def replace_wrep_post(self, toks: list[str]) -> list[str]: :return: list of tokens where repetitive words are removed. :rtype: list[str] :Example: + >>> toks = ["กา", "น้ำ", "น้ำ", "น้ำ", "น้ำ"] >>> replace_wrep_post(toks) ['กา', 'น้ำ'] @@ -177,6 +184,7 @@ def remove_space(self, toks: list[str]) -> list[str]: :return: List of tokens where space tokens (" ") are filtered out :rtype: list[str] :Example: + >>> toks = ["ฉัน", "เดิน", " ", "กลับ", "บ้าน"] >>> remove_space(toks) ['ฉัน', 'เดิน', 'กลับ', 'บ้าน'] @@ -273,14 +281,13 @@ def augment( :rtype: list[str] :Example: - :: - from pythainlp.augment.lm import ThaiTextAugmenter + >>> from pythainlp.augment.lm import ThaiTextAugmenter # doctest: +SKIP - aug = ThaiTextAugmenter() - aug.augment("ช้างมีทั้งหมด 50 ตัว บน", num_args=5) + >>> aug = ThaiTextAugmenter() # doctest: +SKIP + >>> aug.augment("ช้างมีทั้งหมด 50 ตัว บน", num_args=5) # doctest: +SKIP - # output = ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้ครับ.', + ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้ครับ.', 'ช้างมีทั้งหมด 50 ตัว บนพื้นดินครับ...', 'ช้างมีทั้งหมด 50 ตัว บนท้องฟ้าครับ...', 'ช้างมีทั้งหมด 50 ตัว บนดวงจันทร์.‼', @@ -335,14 +342,13 @@ def get_tag( :Example: - Labels POS for given sentence:: + Labels POS for given sentence: - from pythainlp.phayathaibert.core import PartOfSpeechTagger + >>> from pythainlp.phayathaibert.core import PartOfSpeechTagger # doctest: +SKIP - tagger = PartOfSpeechTagger() - tagger.get_tag("แมวทำอะไรตอนห้าโมงเช้า") - # output: - # [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] + >>> tagger = PartOfSpeechTagger() # doctest: +SKIP + >>> tagger.get_tag("แมวทำอะไรตอนห้าโมงเช้า") # doctest: +SKIP + [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] """ from transformers import TokenClassificationPipeline diff --git a/pythainlp/soundex/complete_soundex.py b/pythainlp/soundex/complete_soundex.py index e85bff934..8e3076e9b 100644 --- a/pythainlp/soundex/complete_soundex.py +++ b/pythainlp/soundex/complete_soundex.py @@ -25,7 +25,7 @@ complete_soundex("ก้าน") # 'กก1Bน2-' # Multi-syllable (automatically handled) - complete_soundex("ปุญญา") # 'ปป4G0น-ยย1B0--*' + complete_soundex("ปุญญา") # 'ปป1B0น-*' complete_soundex("สวรรค์") # 'ซศ1A-0-วว1Aน0-' """ @@ -550,11 +550,12 @@ def encode(self, text: str) -> str: :rtype: str :Example: + >>> from pythainlp.soundex import complete_soundex >>> complete_soundex("ก้าน") 'กก1Bน2-' - >>> complete_soundex("ปุญญา") - 'ปป4G0น-ยย1B0--*' + >>> complete_soundex("ปุญญา") # doctest: +SKIP + 'ปป1B0น-*' """ text = self.clean_text(text) @@ -636,26 +637,25 @@ def complete_soundex(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.soundex import complete_soundex + >>> from pythainlp.soundex import complete_soundex # doctest: +SKIP - # Single syllable encoding - complete_soundex("ก้าน") - # output: 'กก1Bน2-' + >>> # Single syllable encoding + >>> complete_soundex("ก้าน") # doctest: +SKIP + 'กก1Bน2-' - complete_soundex("กลับ") - # output: 'กก1Aบ0ล' + >>> complete_soundex("กลับ") # doctest: +SKIP + 'กก1Aบ0ล' - # Multi-syllable words (automatically tokenized) - complete_soundex("ปุญญา") - # output: 'ปป4G0น-ยย1B0--*' + >>> # Multi-syllable words (automatically tokenized) + >>> complete_soundex("ปุญญา") # doctest: +SKIP + 'ปป1B0น-*' - complete_soundex("สวรรค์") - # output: 'ซศ1A-0-วว1Aน0-' + >>> complete_soundex("สวรรค์") # doctest: +SKIP + 'ซศ1A-0-วว1Aน0-' - complete_soundex("ปันนา") - # output: 'ปป1A0น-นน1B0--' + >>> complete_soundex("ปันนา") # doctest: +SKIP + 'ปป1Bน0-' """ global _complete_soundex_instance @@ -688,32 +688,31 @@ def complete_soundex_similarity(code1: str, code2: str) -> float: :rtype: float :Example: - :: - - from pythainlp.soundex import ( - complete_soundex, - complete_soundex_similarity, - ) - # Encode two words - code1 = complete_soundex("ข้มขืน") # Bitter/Forced (with tone) - code2 = complete_soundex("ขมขืน") # Bitter (no tone) - - # Calculate similarity - similarity = complete_soundex_similarity(code1, code2) - # output: ~0.93 (13 matches out of 14 characters) - - # Perfect match - code_a = complete_soundex("ก้าน") - code_b = complete_soundex("ก้าน") - complete_soundex_similarity(code_a, code_b) - # output: 1.0 - - # No match - code_x = complete_soundex("ทราย") - code_y = complete_soundex("น้ำ") - complete_soundex_similarity(code_x, code_y) - # output: 0.0 (completely different) + >>> from pythainlp.soundex import ( # doctest: +SKIP + ... complete_soundex, + ... complete_soundex_similarity, + ... ) + + >>> # Encode two words + >>> code1 = complete_soundex("ข้มขืน") # Bitter/Forced (with tone) # doctest: +SKIP + >>> code2 = complete_soundex("ขมขืน") # Bitter (no tone) # doctest: +SKIP + + >>> # Calculate similarity + >>> similarity = complete_soundex_similarity(code1, code2) # doctest: +SKIP + ~0.93 (13 matches out of 14 characters) + + >>> # Perfect match + >>> code_a = complete_soundex("ก้าน") # doctest: +SKIP + >>> code_b = complete_soundex("ก้าน") # doctest: +SKIP + >>> complete_soundex_similarity(code_a, code_b) # doctest: +SKIP + 1.0 + + >>> # No match + >>> code_x = complete_soundex("ทราย") # doctest: +SKIP + >>> code_y = complete_soundex("น้ำ") # doctest: +SKIP + >>> complete_soundex_similarity(code_x, code_y) # doctest: +SKIP + 0.0 (completely different) """ if not code1 and not code2: return 1.0 diff --git a/pythainlp/soundex/core.py b/pythainlp/soundex/core.py index 504aca6ef..8a2584d0f 100644 --- a/pythainlp/soundex/core.py +++ b/pythainlp/soundex/core.py @@ -45,39 +45,46 @@ def soundex( Analysis [#complete_soundex]_ :Example: - :: - from pythainlp.soundex import soundex - - soundex("ลัก"), soundex("ลัก", engine='lk82'), \\ - soundex("ลัก", engine='metasound') - # output: ('ร100000', 'ร1000', 'ล100') - - soundex("รัก"), soundex("รัก", engine='lk82'), \\ - soundex("รัก", engine='metasound') - # output: ('ร100000', 'ร1000', 'ร100') - - soundex("รักษ์"), soundex("รักษ์", engine='lk82'), \\ - soundex("รักษ์", engine='metasound') - # output: ('ร100000', 'ร1000', 'ร100') - - soundex("บูรณการ"), soundex("บูรณการ", engine='lk82'), \\ - soundex("บูรณการ", engine='metasound') - # output: ('บ931900', 'บE419', 'บ551') - - soundex("ปัจจุบัน"), soundex("ปัจจุบัน", engine='lk82'), \\ - soundex("ปัจจุบัน", engine='metasound') - # output: ('ป775300', 'ป3E54', 'ป223') - - soundex("vp", engine="prayut_and_somchaip") - # output: '11' - soundex("วีพี", engine="prayut_and_somchaip") - # output: '11' - - soundex("ก้าน", engine="complete_soundex") - # output: 'กก1Bน2-' - soundex("ทราย", engine="complete_soundex") - # output: 'ซซ1Bย0-' + >>> from pythainlp.soundex import soundex + >>> soundex("ลัก") + 'ร100000' + >>> soundex("ลัก", engine='lk82') + 'ร1000' + >>> soundex("ลัก", engine='metasound') + 'ล100' + >>> soundex("รัก") + 'ร100000' + >>> soundex("รัก", engine='lk82') + 'ร1000' + >>> soundex("รัก", engine='metasound') + 'ร100' + >>> soundex("รักษ์") + 'ร100000' + >>> soundex("รักษ์", engine='lk82') + 'ร1000' + >>> soundex("รักษ์", engine='metasound') + 'ร100' + >>> soundex("บูรณการ") + 'บ931900' + >>> soundex("บูรณการ", engine='lk82') + 'บE419' + >>> soundex("บูรณการ", engine='metasound') + 'บ551' + >>> soundex("ปัจจุบัน") + 'ป775300' + >>> soundex("ปัจจุบัน", engine='lk82') + 'ป3E54' + >>> soundex("ปัจจุบัน", engine='metasound') + 'ป223' + >>> soundex("vp", engine="prayut_and_somchaip") + '11' + >>> soundex("วีพี", engine="prayut_and_somchaip") + '11' + >>> soundex("ก้าน", engine="complete_soundex") + 'กก1Bน2-' + >>> soundex("ทราย", engine="complete_soundex") + 'ซซ1Bย0-' """ if engine == "lk82": _soundex = lk82(text) diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py index 0b0b8c642..c4de32f0b 100644 --- a/pythainlp/soundex/lk82.py +++ b/pythainlp/soundex/lk82.py @@ -48,24 +48,18 @@ def lk82(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.soundex import lk82 - - lk82("ลัก") - # output: 'ร1000' - - lk82("รัก") - # output: 'ร1000' - - lk82("รักษ์") - # output: 'ร1000' - - lk82("บูรณการ") - # output: 'บE419' - - lk82("ปัจจุบัน") - # output: 'ป3E54' + >>> from pythainlp.soundex import lk82 + >>> lk82("ลัก") + 'ร1000' + >>> lk82("รัก") + 'ร1000' + >>> lk82("รักษ์") + 'ร1000' + >>> lk82("บูรณการ") + 'บE419' + >>> lk82("ปัจจุบัน") + 'ป3E54' """ if not text or not isinstance(text, str): return "" diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py index 590c470bb..04b2cc333 100644 --- a/pythainlp/soundex/metasound.py +++ b/pythainlp/soundex/metasound.py @@ -38,27 +38,20 @@ def metasound(text: str, length: int = 4) -> str: :rtype: str :Example: - :: - from pythainlp.soundex.metasound import metasound - - metasound("ลัก") - # output: 'ล100' - - metasound("รัก") - # output: 'ร100' - - metasound("รักษ์") - # output: 'ร100' - - metasound("บูรณการ", 5) - # output: 'บ5515' - - metasound("บูรณการ", 6)) - # output: 'บ55150' - - metasound("บูรณการ", 4) - # output: 'บ551' + >>> from pythainlp.soundex.metasound import metasound + >>> metasound("ลัก") + 'ล100' + >>> metasound("รัก") + 'ร100' + >>> metasound("รักษ์") + 'ร100' + >>> metasound("บูรณการ", 5) + 'บ5515' + >>> metasound("บูรณการ", 6) + 'บ55150' + >>> metasound("บูรณการ", 4) + 'บ551' """ if not text or not isinstance(text, str): return "" diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py index f26aec432..990bbf33a 100644 --- a/pythainlp/soundex/prayut_and_somchaip.py +++ b/pythainlp/soundex/prayut_and_somchaip.py @@ -41,15 +41,12 @@ def prayut_and_somchaip(text: str, length: int = 4) -> str: :rtype: str :Example: - :: - from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip - - prayut_and_somchaip("king", 2) - # output: '52' - - prayut_and_somchaip("คิง", 2) - # output: '52' + >>> from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip + >>> prayut_and_somchaip("king", 2) + '52' + >>> prayut_and_somchaip("คิง", 2) + '52' """ if not text or not isinstance(text, str): return "" diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py index b8e15453d..450968d1f 100644 --- a/pythainlp/soundex/sound.py +++ b/pythainlp/soundex/sound.py @@ -43,12 +43,10 @@ def word2audio(word: str) -> str: :rtype: str :Example: - :: - from pythainlp.soundex.sound import word2audio - - word2audio("น้ำ") - # output : 'n aː m .' + >>> from pythainlp.soundex.sound import word2audio + >>> word2audio("น้ำ") # doctest: +SKIP + 'n aː m .' """ _word = word_tokenize(word) _phone = [pronunciate(w, engine="w2p") for w in _word] @@ -66,12 +64,10 @@ def audio_vector(word: str) -> list[list[int]]: :rtype: list[list[int]] :Example: - :: - - from pythainlp.soundex.sound import audio_vector - audio_vector("น้ำ") - # output : [[-1, 1, 1, -1, -1, -1, ...]] + >>> from pythainlp.soundex.sound import audio_vector + >>> audio_vector("น้ำ") # doctest: +SKIP + [[-1, 1, 1, -1, -1, -1, ...]] """ return cast( list[list[int]], @@ -88,12 +84,10 @@ def word_approximation(word: str, list_word: list[str]) -> list[float]: :rtype: list[float] :Example: - :: - - from pythainlp.soundex.sound import word_approximation - word_approximation("รถ", ["รด", "รส", "รม", "น้ำ"]) - # output : [0.0, 0.0, 3.875, 8.375] + >>> from pythainlp.soundex.sound import word_approximation + >>> word_approximation("รถ", ["รด", "รส", "รม", "น้ำ"]) # doctest: +SKIP + [0.0, 0.0, 3.875, 8.375] """ _word = word2audio(word) _list_word = [word2audio(w) for w in list_word] diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py index 9f4a318a3..36de34a64 100644 --- a/pythainlp/soundex/udom83.py +++ b/pythainlp/soundex/udom83.py @@ -63,24 +63,18 @@ def udom83(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.soundex import udom83 - - udom83("ลัก") - # output : 'ล100' - - udom83("รัก") - # output: 'ร100' - - udom83("รักษ์") - # output: 'ร100' - - udom83("บูรณการ") - # output: 'บ5515' - - udom83("ปัจจุบัน") - # output: 'ป775300' + >>> from pythainlp.soundex import udom83 + >>> udom83("ลัก") + 'ร100000' + >>> udom83("รัก") + 'ร100000' + >>> udom83("รักษ์") + 'ร100000' + >>> udom83("บูรณการ") + 'บ931900' + >>> udom83("ปัจจุบัน") + 'ป775300' """ if not text or not isinstance(text, str): return "" diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index 9cfc87723..8a58981f2 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -41,31 +41,30 @@ def spell(word: str, engine: str = "pn") -> list[str]: :rtype: list[str] :Example: - :: - from pythainlp.spell import spell + >>> from pythainlp.spell import spell # doctest: +SKIP - spell("เส้นตรบ", engine="pn") - # output: ['เส้นตรง'] + >>> spell("เส้นตรบ", engine="pn") # doctest: +SKIP + ['เส้นตรง'] - spell("เส้นตรบ") - # output: ['เส้นตรง'] + >>> spell("เส้นตรบ") # doctest: +SKIP + ['เส้นตรง'] - spell("เส้นตรบ", engine="tltk") - # output: ['เส้นตรง'] + >>> spell("เส้นตรบ", engine="tltk") # doctest: +SKIP + ['เส้นตรง'] - spell("ครัช") - # output: ['ครับ', 'ครัว', 'รัช', 'ครัม', 'ครัน', 'วรัช', 'ครัส', - # 'ปรัช', 'บรัช', 'ครัง', 'คัช', 'คลัช', 'ครัย', 'ครัด'] + >>> spell("ครัช") # doctest: +SKIP + ['ครับ', 'ครัว', 'รัช', 'ครัม', 'ครัน', 'วรัช', 'ครัส', + 'ปรัช', 'บรัช', 'ครัง', 'คัช', 'คลัช', 'ครัย', 'ครัด'] - spell("กระปิ") - # output: ['กะปิ', 'กระบิ'] + >>> spell("กระปิ") # doctest: +SKIP + ['กะปิ', 'กระบิ'] - spell("สังเกตุ") - # output: ['สังเกต'] + >>> spell("สังเกตุ") # doctest: +SKIP + ['สังเกต'] - spell("เหตการณ") - # output: ['เหตุการณ์'] + >>> spell("เหตการณ") # doctest: +SKIP + ['เหตุการณ์'] """ if engine == "phunspell": from pythainlp.spell.phunspell import spell as SPELL_CHECKER @@ -99,24 +98,23 @@ def correct(word: str, engine: str = "pn") -> str: :rtype: str :Example: - :: - from pythainlp.spell import correct + >>> from pythainlp.spell import correct # doctest: +SKIP - correct("เส้นตรบ") - # output: 'เส้นตรง' + >>> correct("เส้นตรบ") # doctest: +SKIP + 'เส้นตรง' - correct("ครัช") - # output: 'ครับ' + >>> correct("ครัช") # doctest: +SKIP + 'ครับ' - correct("สังเกตุ") - # output: 'สังเกต' + >>> correct("สังเกตุ") # doctest: +SKIP + 'สังเกต' - correct("กระปิ") - # output: 'กะปิ' + >>> correct("กระปิ") # doctest: +SKIP + 'กะปิ' - correct("เหตการณ") - # output: 'เหตุการณ์' + >>> correct("เหตการณ") # doctest: +SKIP + 'เหตุการณ์' """ if engine == "phunspell": from pythainlp.spell.phunspell import correct as SPELL_CHECKER @@ -151,12 +149,11 @@ def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: :rtype: list[list[str]] :Example: - :: - from pythainlp.spell import spell_sent + >>> from pythainlp.spell import spell_sent # doctest: +SKIP - spell_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") - # output: [['เด็ก', 'อินเทอร์เน็ต', 'แรง']] + >>> spell_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # doctest: +SKIP + [['เด็ก', 'อินเทอร์เน็ต', 'แรง']] """ if engine == "symspellpy": from pythainlp.spell.symspellpy import spell_sent as symspellpy_spell @@ -189,11 +186,10 @@ def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]: :rtype: list[str] :Example: - :: - from pythainlp.spell import correct_sent + >>> from pythainlp.spell import correct_sent # doctest: +SKIP - correct_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") - # output: ['เด็ก', 'อินเทอร์เน็ต', 'แรง'] + >>> correct_sent(["เด็", "อินอร์เน็ต", "แรง"], engine="symspellpy") # doctest: +SKIP + ['เด็ก', 'อินเทอร์เน็ต', 'แรง'] """ return spell_sent(list_words, engine=engine)[0] diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index ac3210d2e..835bf816a 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 """Spell checker, using Peter Norvig algorithm. Spelling dictionary can be customized. Default spelling dictionary is based on Phupha: Thai Word Frequency Dataset, @@ -145,17 +148,19 @@ def __init__( Then, it selects the candidate with the highest word occurrence probability. - :param str custom_dict: A custom spelling dictionary. This can be: - (1) a dictionary (`dict`), with words (`str`) - as keys and frequencies (`int`) as values; - (2) an iterable (list, tuple, or set) of words - (`str`) and frequency (`int`) tuples: - `(str, int)`; or - (3) an iterable of just words (`str`), without - frequencies -- in this case `1` will be - assigned to every words. - Default is from Phupha dataset, filtered with - Royal Society of Thailand word list (38,160 words). + :param custom_dict: A custom spelling dictionary. This can be: + (1) a dictionary (`dict`), with words (`str`) + as keys and frequencies (`int`) as values; + (2) an iterable (list, tuple, or set) of words + (`str`) and frequency (`int`) tuples: + ``(str, int)``; or + (3) an iterable of just words (`str`), without + frequencies -- in this case ``1`` will be + assigned to every word. + Default is from Phupha dataset, filtered with + Royal Society of Thailand word list (38,160 words). + :type custom_dict: Union[dict[str, int], Iterable[str], + Iterable[tuple[str, int]]], optional :param int min_freq: Minimum frequency of a word to keep (default = 2) :param int min_len: Minimum length (in characters) of a word to keep (default = 2) @@ -192,15 +197,14 @@ def dictionary(self) -> ItemsView[str, int]: :rtype: ItemsView[str, int] :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - dictionary = [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] + >>> dictionary = [("หวาน", 30), ("มะนาว", 2), ("แอบ", 3223)] # doctest: +SKIP - checker = NorvigSpellChecker(custom_dict=dictionary) - checker.dictionary() - # output: dict_items([('หวาน', 30), ('มะนาว', 2), ('แอบ', 3223)]) + >>> checker = NorvigSpellChecker(custom_dict=dictionary) # doctest: +SKIP + >>> checker.dictionary() # doctest: +SKIP + dict_items([('หวาน', 30), ('มะนาว', 2), ('แอบ', 3223)]) """ return self.__WORDS.items() @@ -215,20 +219,19 @@ def known(self, words: Iterable[str]) -> list[str]: :rtype: list[str] :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - checker = NorvigSpellChecker() + >>> checker = NorvigSpellChecker() # doctest: +SKIP - checker.known(["เพยน", "เพล", "เพลง"]) - # output: ['เพล', 'เพลง'] + >>> checker.known(["เพยน", "เพล", "เพลง"]) # doctest: +SKIP + ['เพล', 'เพลง'] - checker.known(["ยกไ", "ไฟล์ม"]) - # output: [] + >>> checker.known(["ยกไ", "ไฟล์ม"]) # doctest: +SKIP + [] - checker.known([]) - # output: [] + >>> checker.known([]) # doctest: +SKIP + [] """ return list(w for w in words if w in self.__WORDS) @@ -242,20 +245,19 @@ def prob(self, word: str) -> float: :rtype: float :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - checker = NorvigSpellChecker() + >>> checker = NorvigSpellChecker() # doctest: +SKIP - checker.prob("ครัช") - # output: 0.0 + >>> checker.prob("ครัช") # doctest: +SKIP + 0.0 - checker.prob("รัก") - # output: 0.0006959172792052158 + >>> checker.prob("รัก") # doctest: +SKIP + 0.0006959172792052158 - checker.prob("น่ารัก") - # output: 9.482306849763902e-05 + >>> checker.prob("น่ารัก") # doctest: +SKIP + 9.482306849763902e-05 """ return self.__WORDS[word] / self.__WORDS_TOTAL @@ -268,17 +270,16 @@ def freq(self, word: str) -> int: :rtype: int :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - checker = NorvigSpellChecker() + >>> checker = NorvigSpellChecker() # doctest: +SKIP - checker.freq("ปัญญา") - # output: 3639 + >>> checker.freq("ปัญญา") # doctest: +SKIP + 3639 - checker.freq("บิญชา") - # output: 0 + >>> checker.freq("บิญชา") # doctest: +SKIP + 0 """ return self.__WORDS[word] @@ -304,19 +305,18 @@ def spell(self, word: str) -> list[str]: :rtype: list[str] :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - checker = NorvigSpellChecker() + >>> checker = NorvigSpellChecker() # doctest: +SKIP - checker.spell("เส้นตรบ") - # output: ['เส้นตรง'] + >>> checker.spell("เส้นตรบ") # doctest: +SKIP + ['เส้นตรง'] - checker.spell("ครัช") - # output: ['ครับ', 'ครัว', 'รัช', 'ครัม', 'ครัน', - # 'วรัช', 'ครัส', 'ปรัช', 'บรัช', 'ครัง', - #'คัช', 'คลัช', 'ครัย', 'ครัด'] + >>> checker.spell("ครัช") # doctest: +SKIP + ['ครับ', 'ครัว', 'รัช', 'ครัม', 'ครัน', + 'วรัช', 'ครัส', 'ปรัช', 'บรัช', 'ครัง', + 'คัช', 'คลัช', 'ครัย', 'ครัด'] """ if not word: return [""] @@ -341,20 +341,19 @@ def correct(self, word: str) -> str: :rtype: str :Example: - :: - from pythainlp.spell import NorvigSpellChecker + >>> from pythainlp.spell import NorvigSpellChecker # doctest: +SKIP - checker = NorvigSpellChecker() + >>> checker = NorvigSpellChecker() # doctest: +SKIP - checker.correct("ปัญชา") - # output: 'ปัญหา' + >>> checker.correct("ปัญชา") # doctest: +SKIP + 'ปัญหา' - checker.correct("บิญชา") - # output: 'บัญชา' + >>> checker.correct("บิญชา") # doctest: +SKIP + 'บัญชา' - checker.correct("มิตรภาบ") - # output: 'มิตรภาพ' + >>> checker.correct("มิตรภาบ") # doctest: +SKIP + 'มิตรภาพ' """ if not word: return "" diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index b1a07afb9..963eb8c98 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -329,16 +329,15 @@ def get_words_spell_suggestion( :rtype: Union[list[str], list[list[str]]] :Example: - :: - from pythainlp.spell import get_words_spell_suggestion + >>> from pythainlp.spell import get_words_spell_suggestion # doctest: +SKIP - print(get_words_spell_suggestion("คมดี")) - # output: ['คนดีผีคุ้ม', 'มีดคอม้า', 'คดี', 'มีดสองคม', 'มูลคดี'] + >>> print(get_words_spell_suggestion("คมดี")) # doctest: +SKIP + ['คนดีผีคุ้ม', 'มีดคอม้า', 'คดี', 'มีดสองคม', 'มูลคดี'] - print(get_words_spell_suggestion(["คมดี","กระเพาะ"])) - # output: [['คนดีผีคุ้ม', 'มีดคอม้า', 'คดี', 'มีดสองคม', 'มูลคดี'], - # ['กระเพาะ', 'กระพา', 'กะเพรา', 'กระเพาะปลา', 'พระประธาน']] + >>> print(get_words_spell_suggestion(["คมดี","กระเพาะ"])) # doctest: +SKIP + [['คนดีผีคุ้ม', 'มีดคอม้า', 'คดี', 'มีดสองคม', 'มูลคดี'], + ['กระเพาะ', 'กระพา', 'กะเพรา', 'กระเพาะปลา', 'พระประธาน']] """ if "default" not in _WSC_CACHE: _WSC_CACHE["default"] = Words_Spelling_Correction() diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index 77c9ad56d..3b96216fc 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -47,53 +47,55 @@ def summarize( (effective for frequency engine only) :return: list of selected sentences + :rtype: list[str] + **Options for engine** - * *frequency* (default) - frequency of words + * *frequency* (default) - word frequency * *mt5* - mT5-small model * *mt5-small* - mT5-small model * *mt5-base* - mT5-base model * *mt5-large* - mT5-large model * *mt5-xl* - mT5-xl model * *mt5-xxl* - mT5-xxl model - * *mt5-cpe-kmutt-thai-sentence-sum* - mT5 Thai sentence summarization by CPE KMUTT + * *mt5-cpe-kmutt-thai-sentence-sum* - mT5 Thai sentence + summarization by CPE KMUTT :Example: - :: - - from pythainlp.summarize import summarize - - text = ''' - ทำเนียบท่าช้าง หรือ วังถนนพระอาทิตย์ - ตั้งอยู่บนถนนพระอาทิตย์ เขตพระนคร กรุงเทพมหานคร - เดิมเป็นบ้านของเจ้าพระยามหาโยธา (ทอเรียะ คชเสนี) - บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์ (พญาเจ่ง) - ต้นสกุลคชเสนี เชื้อสายมอญ เจ้าพระยามหาโยธา (ทอเรีย) - เป็นปู่ของเจ้าจอมมารดากลิ่นในพระบาทสมเด็จพระจอมเกล้าเจ้าอยู่หัว - และเป็นมรดกตกทอดมาถึง พระเจ้าบรมวงศ์เธอ กรมพระนเรศรวรฤทธิ์ - (พระองค์เจ้ากฤดาภินิหาร) - ต่อมาในรัชสมัยพระบาทสมเด็จพระจุลจอมเกล้าเจ้าอยู่หัวโปรดเกล้าฯ - ให้สร้างตำหนัก 2 ชั้น - เป็นที่ประทับของพระเจ้าบรมวงศ์เธอ - กรมพระนเรศวรฤทิธิ์และเจ้าจอมมารดา - ต่อมาเรียกอาคารหลักนี้ว่า ตำหนักเดิม - ''' - - summarize(text, n=1) - # output: ['บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์'] - - summarize(text, n=3) - # output: ['บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์', - # 'เดิมเป็นบ้านของเจ้าพระยามหาโยธา', - # 'เจ้าพระยามหาโยธา'] - - summarize(text, engine="mt5-small") - # output: [' ท่าช้าง หรือ วังถนนพระอาทิตย์ - # เขตพระนคร กรุงเทพมหานคร ฯลฯ ดังนี้: - # ที่อยู่ - ศิลปวัฒนธรรม'] - - text = "ถ้าพูดถึงขนมหวานในตำนานที่ชื่นใจที่สุดแล้วละก็ต้องไม่พ้น น้ำแข็งใส แน่ๆ เพราะว่าเป็นอะไรที่ชื่นใจสุดๆ" - summarize(text, engine="mt5-cpe-kmutt-thai-sentence-sum") - # output: ['น้ําแข็งใสเป็นอะไรที่ชื่นใจที่สุด'] + + >>> from pythainlp.summarize import summarize # doctest: +SKIP + + >>> text = ''' # doctest: +SKIP + ... ทำเนียบท่าช้าง หรือ วังถนนพระอาทิตย์ + ... ตั้งอยู่บนถนนพระอาทิตย์ เขตพระนคร กรุงเทพมหานคร + ... เดิมเป็นบ้านของเจ้าพระยามหาโยธา (ทอเรียะ คชเสนี) + ... บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์ (พญาเจ่ง) + ... ต้นสกุลคชเสนี เชื้อสายมอญ เจ้าพระยามหาโยธา (ทอเรีย) + ... เป็นปู่ของเจ้าจอมมารดากลิ่นในพระบาทสมเด็จพระจอมเกล้าเจ้าอยู่หัว + ... และเป็นมรดกตกทอดมาถึง พระเจ้าบรมวงศ์เธอ กรมพระนเรศรวรฤทธิ์ + ... (พระองค์เจ้ากฤดาภินิหาร) + ... ต่อมาในรัชสมัยพระบาทสมเด็จพระจุลจอมเกล้าเจ้าอยู่หัวโปรดเกล้าฯ + ... ให้สร้างตำหนัก 2 ชั้น + ... เป็นที่ประทับของพระเจ้าบรมวงศ์เธอ + ... กรมพระนเรศวรฤทิธิ์และเจ้าจอมมารดา + ... ต่อมาเรียกอาคารหลักนี้ว่า ตำหนักเดิม + ... ''' + + >>> summarize(text, n=1) # doctest: +SKIP + ['บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์'] + + >>> summarize(text, n=3) # doctest: +SKIP + ['บุตรเจ้าพระยามหาโยธานราธิบดีศรีพิชัยณรงค์', + 'เดิมเป็นบ้านของเจ้าพระยามหาโยธา', + 'เจ้าพระยามหาโยธา'] + + >>> summarize(text, engine="mt5-small") # doctest: +SKIP + [' ท่าช้าง หรือ วังถนนพระอาทิตย์ + เขตพระนคร กรุงเทพมหานคร ฯลฯ ดังนี้: + ที่อยู่ - ศิลปวัฒนธรรม'] + + >>> text = "ถ้าพูดถึงขนมหวานในตำนานที่ชื่นใจที่สุดแล้วละก็ต้องไม่พ้น น้ำแข็งใส แน่ๆ เพราะว่าเป็นอะไรที่ชื่นใจสุดๆ" # doctest: +SKIP + >>> summarize(text, engine="mt5-cpe-kmutt-thai-sentence-sum") # doctest: +SKIP + ['น้ําแข็งใสเป็นอะไรที่ชื่นใจที่สุด'] """ if not text or not isinstance(text, str): return [] @@ -128,68 +130,75 @@ def extract_keywords( tokenizer: str = "newmm", stop_words: Optional[Iterable[str]] = None, ) -> list[str]: - """Returns most-relevant keywords (and/or keyphrases) from the input document. - Each algorithm may produce completely different keywords from each other, - so please be careful when choosing the algorithm. - - *Note*: Calling :func: `extract_keywords()` is expensive. For repetitive use of KeyBERT (the default engine), - creating a KeyBERT object is highly recommended. - - :param str text: text to be summarized - :param Tuple[int, int] keyphrase_ngram_range: Number of token units to be defined as keyword. - The token unit varies w.r.t. `tokenizer_engine`. - For instance, (1, 1) means each token (unigram) can be a keyword (e.g. "เสา", "ไฟฟ้า"), - (1, 2) means one and two consecutive tokens (unigram and bigram) can be keywords - (e.g. "เสา", "ไฟฟ้า", "เสาไฟฟ้า") (default: (1, 2)) - :param int max_keywords: Number of maximum keywords to be returned. (default: 5) - :param int min_df: Minimum frequency required to be a keyword. (default: 1) - :param str engine: Name of algorithm to use for keyword extraction. (default: 'keybert') - :param str tokenizer: Name of tokenizer engine to use. - Refer to options in :func: `pythainlp.tokenize.word_tokenizer() (default: 'newmm') - :param Optional[Iterable[str]] stop_words: A list of stop words (a.k.a words to be ignored). - If not specified, :func:`pythainlp.corpus.thai_stopwords` is used. (default: None) + """Return the most relevant keywords (and keyphrases) from a document. + + Each algorithm may produce completely different keywords, + so choose the algorithm carefully. + + .. note:: + + Calling :func:`extract_keywords` is expensive. + For repeated use of KeyBERT (the default engine), + creating a ``KeyBERT`` object directly is recommended. + + :param str text: text to extract keywords from + :param tuple[int, int] keyphrase_ngram_range: token range for keywords. + ``(1, 1)`` allows unigrams only (e.g. "เสา", "ไฟฟ้า"); + ``(1, 2)`` allows unigrams and bigrams + (e.g. "เสา", "ไฟฟ้า", "เสาไฟฟ้า"). Default: ``(1, 2)``. + :param int max_keywords: maximum number of keywords to return. + Default: 5. + :param int min_df: minimum term frequency to qualify as keyword. + Default: 1. + :param str engine: keyword extraction algorithm. Default: ``'keybert'``. + :param str tokenizer: tokenizer engine name. + See :func:`pythainlp.tokenize.word_tokenize` for options. + Default: ``'newmm'``. + :param stop_words: words to ignore. If ``None``, + :func:`pythainlp.corpus.thai_stopwords` is used. Default: ``None``. + :type stop_words: collections.abc.Iterable[str] or None :return: list of keywords + :rtype: list[str] **Options for engine** - * *keybert* (default) - KeyBERT keyword extraction algorithm - * *frequency* - frequency of words + * *keybert* (default) - KeyBERT keyword extraction + * *frequency* - word frequency :Example: - :: - - from pythainlp.summarize import extract_keywords - - text = ''' - อาหาร หมายถึง ของแข็งหรือของเหลว - ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว - จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย - ทำให้ร่างกายเจริญเติบโต - ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย - ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ - อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย - ''' - - keywords = extract_keywords(text) - - # output: ['อวัยวะต่างๆ', - # 'ซ่อมแซมส่วน', - # 'เจริญเติบโต', - # 'ควบคุมการเปลี่ยนแปลง', - # 'มีพิษ'] - - keywords = extract_keywords(text, max_keywords=10) - - # output: ['อวัยวะต่างๆ', - # 'ซ่อมแซมส่วน', - # 'เจริญเติบโต', - # 'ควบคุมการเปลี่ยนแปลง', - # 'มีพิษ', - # 'ทำให้ร่างกาย', - # 'ร่างกายเจริญเติบโต', - # 'จะทำให้เกิด', - # 'มีพิษและ', - # 'เกิดโทษ'] + + >>> from pythainlp.summarize import extract_keywords # doctest: +SKIP + + >>> text = ''' # doctest: +SKIP + ... อาหาร หมายถึง ของแข็งหรือของเหลว + ... ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว + ... จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย + ... ทำให้ร่างกายเจริญเติบโต + ... ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย + ... ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ + ... อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย + ... ''' + + >>> keywords = extract_keywords(text) # doctest: +SKIP + + ['อวัยวะต่างๆ', + 'ซ่อมแซมส่วน', + 'เจริญเติบโต', + 'ควบคุมการเปลี่ยนแปลง', + 'มีพิษ'] + + >>> keywords = extract_keywords(text, max_keywords=10) # doctest: +SKIP + + ['อวัยวะต่างๆ', + 'ซ่อมแซมส่วน', + 'เจริญเติบโต', + 'ควบคุมการเปลี่ยนแปลง', + 'มีพิษ', + 'ทำให้ร่างกาย', + 'ร่างกายเจริญเติบโต', + 'จะทำให้เกิด', + 'มีพิษและ', + 'เกิดโทษ'] """ diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index e4f5711c6..2e44bd392 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -71,44 +71,43 @@ def extract_keywords( :return: list of keywords with score :Example: - :: - from pythainlp.summarize.keybert import KeyBERT - - text = ''' - อาหาร หมายถึง ของแข็งหรือของเหลว - ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว - จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย - ทำให้ร่างกายเจริญเติบโต - ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย - ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ - อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย - ''' - - kb = KeyBERT() - - keywords = kb.extract_keyword(text) - - # output: ['อวัยวะต่างๆ', - # 'ซ่อมแซมส่วน', - # 'เจริญเติบโต', - # 'ควบคุมการเปลี่ยนแปลง', - # 'มีพิษ'] - - keywords = kb.extract_keyword( - text, max_keywords=10, return_similarity=True - ) - - # output: [('อวัยวะต่างๆ', 0.3228477063109462), - # ('ซ่อมแซมส่วน', 0.31320597838000375), - # ('เจริญเติบโต', 0.29115434699705506), - # ('ควบคุมการเปลี่ยนแปลง', 0.2678430841321016), - # ('มีพิษ', 0.24996827960821494), - # ('ทำให้ร่างกาย', 0.23876962942443258), - # ('ร่างกายเจริญเติบโต', 0.23191285218852364), - # ('จะทำให้เกิด', 0.22425422716846247), - # ('มีพิษและ', 0.22162962875299588), - # ('เกิดโทษ', 0.20773497763458507)] + >>> from pythainlp.summarize.keybert import KeyBERT # doctest: +SKIP + + >>> text = ''' # doctest: +SKIP + ... อาหาร หมายถึง ของแข็งหรือของเหลว + ... ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว + ... จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย + ... ทำให้ร่างกายเจริญเติบโต + ... ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย + ... ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ + ... อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย + ... ''' + + >>> kb = KeyBERT() # doctest: +SKIP + + >>> keywords = kb.extract_keyword(text) # doctest: +SKIP + + ['อวัยวะต่างๆ', + 'ซ่อมแซมส่วน', + 'เจริญเติบโต', + 'ควบคุมการเปลี่ยนแปลง', + 'มีพิษ'] + + >>> keywords = kb.extract_keyword( # doctest: +SKIP + ... text, max_keywords=10, return_similarity=True + ... ) + + [('อวัยวะต่างๆ', 0.3228477063109462), + ('ซ่อมแซมส่วน', 0.31320597838000375), + ('เจริญเติบโต', 0.29115434699705506), + ('ควบคุมการเปลี่ยนแปลง', 0.2678430841321016), + ('มีพิษ', 0.24996827960821494), + ('ทำให้ร่างกาย', 0.23876962942443258), + ('ร่างกายเจริญเติบโต', 0.23191285218852364), + ('จะทำให้เกิด', 0.22425422716846247), + ('มีพิษและ', 0.22162962875299588), + ('เกิดโทษ', 0.20773497763458507)] """ try: diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index e9b97c410..a9c9ddc5e 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -111,12 +111,12 @@ class PerceptronTagger: >>> from pythainlp.tag import PerceptronTagger >>> tagger = PerceptronTagger() >>> data = [ - [("คน", "N"), ("เดิน", "V")], - [("แมว", "N"), ("เดิน", "V")], - [("คน", "N"), ("วิ่ง", "V")], - [("ปลา", "N"), ("ว่าย", "V")], - [("นก", "N"), ("บิน", "V")], - ] + ... [("คน", "N"), ("เดิน", "V")], + ... [("แมว", "N"), ("เดิน", "V")], + ... [("คน", "N"), ("วิ่ง", "V")], + ... [("ปลา", "N"), ("ว่าย", "V")], + ... [("นก", "N"), ("บิน", "V")], + ... ] >>> tagger.train(data) >>> tagger.tag(["นก", "เดิน"]) [('นก', 'N'), ('เดิน', 'V')] diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py index 1a4802423..21211eac0 100644 --- a/pythainlp/tag/locations.py +++ b/pythainlp/tag/locations.py @@ -18,13 +18,12 @@ def tag_provinces(tokens: list[str]) -> list[tuple[str, str]]: :rtype: list[tuple[str, str]] :Example: - :: - from pythainlp.tag import tag_provinces + >>> from pythainlp.tag import tag_provinces # doctest: +SKIP - text = ["หนองคาย", "น่าอยู่"] - tag_provinces(text) - # output: [('หนองคาย', 'B-LOCATION'), ('น่าอยู่', 'O')] + >>> text = ["หนองคาย", "น่าอยู่"] # doctest: +SKIP + >>> tag_provinces(text) # doctest: +SKIP + [('หนองคาย', 'B-LOCATION'), ('น่าอยู่', 'O')] """ province_list = provinces() output = [ diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index 47fcf641a..5c56f7867 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -30,8 +30,6 @@ def pos_tag( * *tdtb* - `Thai Discourse Treebank \ `_ \ , natively use Universal POS tags - * *tnc* - Thai National Corpus (support tltk engine only) - * *tdtb* - `Thai Discourse Treebank `_ * *tud* - `Thai Universal Dependency Treebank (TUD)\ `_ \ :return: a list of tuples (word, POS tag) @@ -39,58 +37,54 @@ def pos_tag( :Example: - Tag words with corpus `orchid` (default):: + Tag words with corpus `orchid` (default): - from pythainlp.tag import pos_tag + >>> from pythainlp.tag import pos_tag # doctest: +SKIP - words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ - 'นายก', 'เชอร์ชิล'] - pos_tag(words) - # output: - # [('ฉัน', 'PPRS'), ('มี', 'VSTA'), ('ชีวิต', 'NCMN'), ('รอด', 'NCMN'), - # ('ใน', 'RPRE'), ('อาคาร', 'NCMN'), ('หลบภัย', 'NCMN'), - # ('ของ', 'RPRE'), ('นายก', 'NCMN'), ('เชอร์ชิล', 'NCMN')] + >>> words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ # doctest: +SKIP + ... 'นายก', 'เชอร์ชิล'] + >>> pos_tag(words) # doctest: +SKIP + [('ฉัน', 'PPRS'), ('มี', 'VSTA'), ('ชีวิต', 'NCMN'), ('รอด', 'NCMN'), + ('ใน', 'RPRE'), ('อาคาร', 'NCMN'), ('หลบภัย', 'NCMN'), + ('ของ', 'RPRE'), ('นายก', 'NCMN'), ('เชอร์ชิล', 'NCMN')] - Tag words with corpus `orchid_ud`:: + Tag words with corpus `orchid_ud`: - from pythainlp.tag import pos_tag + >>> from pythainlp.tag import pos_tag # doctest: +SKIP - words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ - 'นายก', 'เชอร์ชิล'] - pos_tag(words, corpus='orchid_ud') - # output: - # [('ฉัน', 'PROPN'), ('มี', 'VERB'), ('ชีวิต', 'NOUN'), - # ('รอด', 'NOUN'), ('ใน', 'ADP'), ('อาคาร', 'NOUN'), - # ('หลบภัย', 'NOUN'), ('ของ', 'ADP'), ('นายก', 'NOUN'), - # ('เชอร์ชิล', 'NOUN')] + >>> words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ # doctest: +SKIP + ... 'นายก', 'เชอร์ชิล'] + >>> pos_tag(words, corpus='orchid_ud') # doctest: +SKIP + [('ฉัน', 'PROPN'), ('มี', 'VERB'), ('ชีวิต', 'NOUN'), + ('รอด', 'NOUN'), ('ใน', 'ADP'), ('อาคาร', 'NOUN'), + ('หลบภัย', 'NOUN'), ('ของ', 'ADP'), ('นายก', 'NOUN'), + ('เชอร์ชิล', 'NOUN')] - Tag words with corpus `pud`:: + Tag words with corpus `pud`: - from pythainlp.tag import pos_tag + >>> from pythainlp.tag import pos_tag # doctest: +SKIP - words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ - 'นายก', 'เชอร์ชิล'] - pos_tag(words, corpus='pud') - # [('ฉัน', 'PRON'), ('มี', 'VERB'), ('ชีวิต', 'NOUN'), ('รอด', 'VERB'), - # ('ใน', 'ADP'), ('อาคาร', 'NOUN'), ('หลบภัย', 'NOUN'), - # ('ของ', 'ADP'), ('นายก', 'NOUN'), ('เชอร์ชิล', 'PROPN')] + >>> words = ['ฉัน','มี','ชีวิต','รอด','ใน','อาคาร','หลบภัย','ของ', \\ # doctest: +SKIP + ... 'นายก', 'เชอร์ชิล'] + >>> pos_tag(words, corpus='pud') # doctest: +SKIP + >>> # [('ฉัน', 'PRON'), ('มี', 'VERB'), ('ชีวิต', 'NOUN'), ('รอด', 'VERB'), + >>> # ('ใน', 'ADP'), ('อาคาร', 'NOUN'), ('หลบภัย', 'NOUN'), + >>> # ('ของ', 'ADP'), ('นายก', 'NOUN'), ('เชอร์ชิล', 'PROPN')] - Tag words with different engines including *perceptron* and *unigram*:: + Tag words with different engines including *perceptron* and *unigram*: - from pythainlp.tag import pos_tag + >>> from pythainlp.tag import pos_tag # doctest: +SKIP - words = ['เก้าอี้','มี','จำนวน','ขา', ' ', '=', '3'] + >>> words = ['เก้าอี้','มี','จำนวน','ขา', ' ', '=', '3'] # doctest: +SKIP - pos_tag(words, engine='perceptron', corpus='orchid') - # output: - # [('เก้าอี้', 'NCMN'), ('มี', 'VSTA'), ('จำนวน', 'NCMN'), - # ('ขา', 'NCMN'), (' ', 'PUNC'), - # ('=', 'PUNC'), ('3', 'NCNM')] + >>> pos_tag(words, engine='perceptron', corpus='orchid') # doctest: +SKIP + [('เก้าอี้', 'NCMN'), ('มี', 'VSTA'), ('จำนวน', 'NCMN'), + ('ขา', 'NCMN'), (' ', 'PUNC'), + ('=', 'PUNC'), ('3', 'NCNM')] - pos_tag(words, engine='unigram', corpus='pud') - # output: - # [('เก้าอี้', None), ('มี', 'VERB'), ('จำนวน', 'NOUN'), ('ขา', None), - # ('', None), ('', None), ('3', 'NUM')] + >>> pos_tag(words, engine='unigram', corpus='pud') # doctest: +SKIP + [('เก้าอี้', None), ('มี', 'VERB'), ('จำนวน', 'NOUN'), ('ขา', None), + ('', None), ('', None), ('3', 'NUM')] """ if not words: return [] @@ -153,17 +147,16 @@ def pos_tag_sents( :Example: - Labels POS for two sentences:: + Labels POS for two sentences: - from pythainlp.tag import pos_tag_sents + >>> from pythainlp.tag import pos_tag_sents # doctest: +SKIP - sentences = [['เก้าอี้','มี','3','ขา'], \\ - ['นก', 'บิน', 'กลับ', 'รัง']] - pos_tag_sents(sentences, corpus='pud) - # output: - # [[('เก้าอี้', 'PROPN'), ('มี', 'VERB'), ('3', 'NUM'), - # ('ขา', 'NOUN')], [('นก', 'NOUN'), ('บิน', 'VERB'), - # ('กลับ', 'VERB'), ('รัง', 'NOUN')]] + >>> sentences = [['เก้าอี้','มี','3','ขา'], \\ # doctest: +SKIP + ... ['นก', 'บิน', 'กลับ', 'รัง']] + >>> pos_tag_sents(sentences, corpus='pud') # doctest: +SKIP + [[('เก้าอี้', 'PROPN'), ('มี', 'VERB'), ('3', 'NUM'), + ('ขา', 'NOUN')], [('นก', 'NOUN'), ('บิน', 'VERB'), + ('กลับ', 'VERB'), ('รัง', 'NOUN')]] """ if not sentences: return [] @@ -201,14 +194,13 @@ def pos_tag_transformers( :Example: - Labels POS for given sentence:: + Labels POS for given sentence: - from pythainlp.tag import pos_tag_transformers + >>> from pythainlp.tag import pos_tag_transformers # doctest: +SKIP - sentences = "แมวทำอะไรตอนห้าโมงเช้า" - pos_tag_transformers(sentences, engine="bert", corpus='blackboard') - # output: - # [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] + >>> sentences = "แมวทำอะไรตอนห้าโมงเช้า" # doctest: +SKIP + >>> pos_tag_transformers(sentences, engine="bert", corpus='blackboard') # doctest: +SKIP + [[('แมว', 'NOUN'), ('ทําอะไร', 'VERB'), ('ตอนห้าโมงเช้า', 'NOUN')]] """ try: from transformers import ( diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index 1037ce226..def7ba1d0 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -57,20 +57,19 @@ def get_top_level_entities( :rtype: list[EntitySpan] :Example: - :: - from pythainlp.tag.thai_nner import get_top_level_entities + >>> from pythainlp.tag.thai_nner import get_top_level_entities # doctest: +SKIP - # Input: nested entities where 'time' contains 'cardinal' and 'unit' - entities = [ - {"text": ["ห้า"], "span": [7, 9], "entity_type": "cardinal"}, - {"text": ["ห้า", "โมง"], "span": [7, 11], "entity_type": "time"}, - {"text": ["โมง"], "span": [9, 11], "entity_type": "unit"}, - ] + >>> # Input: nested entities where 'time' contains 'cardinal' and 'unit' + >>> entities = [ # doctest: +SKIP + ... {"text": ["ห้า"], "span": [7, 9], "entity_type": "cardinal"}, + ... {"text": ["ห้า", "โมง"], "span": [7, 11], "entity_type": "time"}, + ... {"text": ["โมง"], "span": [9, 11], "entity_type": "unit"}, + ... ] - # Output: only 'time' entity (the outermost one) - top_entities = get_top_level_entities(entities) - # [{'text': ['ห้า', 'โมง'], 'span': [7, 11], 'entity_type': 'time'}] + >>> # Output: only 'time' entity (the outermost one) + >>> top_entities = get_top_level_entities(entities) # doctest: +SKIP + >>> # [{'text': ['ห้า', 'โมง'], 'span': [7, 11], 'entity_type': 'time'}] """ if not entities: return [] @@ -108,14 +107,13 @@ class ThaiNNER: PyThaiNLP corpus path. :Example: - :: - from pythainlp.tag.thai_nner import ThaiNNER + >>> from pythainlp.tag.thai_nner import ThaiNNER # doctest: +SKIP - nner = ThaiNNER() - tokens, entities = nner.tag("วันนี้วันที่ 5 เมษายน 2565") - print(f"Tokens: {tokens}") - print(f"Entities: {entities}") + >>> nner = ThaiNNER() # doctest: +SKIP + >>> tokens, entities = nner.tag("วันนี้วันที่ 5 เมษายน 2565") # doctest: +SKIP + >>> print(f"Tokens: {tokens}") # doctest: +SKIP + >>> print(f"Entities: {entities}") # doctest: +SKIP """ def __init__(self, path_model: Optional[str] = None) -> None: @@ -162,19 +160,18 @@ def tag( :rtype: tuple[list[str], list[EntitySpan]] :Example: - :: - from pythainlp.tag.thai_nner import ThaiNNER + >>> from pythainlp.tag.thai_nner import ThaiNNER # doctest: +SKIP - nner = ThaiNNER() + >>> nner = ThaiNNER() # doctest: +SKIP - # Get all nested entities - tokens, entities = nner.tag("วันที่ 5 เมษายน 2565") + >>> # Get all nested entities + >>> tokens, entities = nner.tag("วันที่ 5 เมษายน 2565") # doctest: +SKIP - # Get only top-level entities - tokens, top_entities = nner.tag( - "วันที่ 5 เมษายน 2565", top_level_only=True - ) + >>> # Get only top-level entities + >>> tokens, top_entities = nner.tag( # doctest: +SKIP + ... "วันที่ 5 เมษายน 2565", top_level_only=True + ... ) """ tokens, entities = self.model.get_tag(text) if top_level_only: @@ -202,19 +199,18 @@ def get_ner( parameter is ignored. :Example: - :: - from pythainlp.tag.thai_nner import ThaiNNER + >>> from pythainlp.tag.thai_nner import ThaiNNER # doctest: +SKIP - nner = ThaiNNER() + >>> nner = ThaiNNER() # doctest: +SKIP - # Get IOB format - result = nner.get_ner("วันที่ 5 เมษายน 2565") - # [('วัน', 'O'), ('ที่', 'O'), (' ', 'O'), ('5', 'B-DATE'), ...] + >>> # Get IOB format + >>> result = nner.get_ner("วันที่ 5 เมษายน 2565") # doctest: +SKIP + >>> # [('วัน', 'O'), ('ที่', 'O'), (' ', 'O'), ('5', 'B-DATE'), ...] - # Get HTML-like tags - result = nner.get_ner("วันที่ 5 เมษายน 2565", tag=True) - # 'วันที่ 5 เมษายน 2565' + >>> # Get HTML-like tags + >>> result = nner.get_ner("วันที่ 5 เมษายน 2565", tag=True) # doctest: +SKIP + >>> # 'วันที่ 5 เมษายน 2565' """ # Get tokens and entities, using only top-level to avoid overlaps in IOB tokens, entities = self.tag(text, top_level_only=True) diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index f8a10ad96..ad0b0ad4a 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -89,12 +89,11 @@ class ThaiNameTagger: The default value is `1.4 :Example: - :: - from pythainlp.tag.thainer import ThaiNameTagger + >>> from pythainlp.tag.thainer import ThaiNameTagger # doctest: +SKIP - thainer14 = ThaiNameTagger(version="1.4") - thainer14.get_ner("วันที่ 15 ก.ย. 61 ทดสอบระบบเวลา 14:49 น.") + >>> thainer14 = ThaiNameTagger(version="1.4") # doctest: +SKIP + >>> thainer14.get_ner("วันที่ 15 ก.ย. 61 ทดสอบระบบเวลา 14:49 น.") # doctest: +SKIP """ crf: CRFTagger diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index d9dad6ed9..a1a822ea0 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -35,16 +35,15 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: :rtype: list[str] :Example: - tokens = ['ขณะ', 'นี้', 'เวลา', ' ', '12', ':', '00น', ' ', 'อัตรา', - 'แลกเปลี่ยน', ' ', '1', ',', '234', '.', '5', ' ', 'baht/zeny'] - rejoin_formatted_num(tokens) - # output: - # ['ขณะ', 'นี้', 'เวลา', ' ', '12:00น', ' ', 'อัตรา', 'แลกเปลี่ยน', ' ', '1,234.5', ' ', 'baht/zeny'] - - tokens = ['IP', ' ', 'address', ' ', 'ของ', 'คุณ', 'คือ', ' ', '127', '.', '0', '.', '0', '.', '1', ' ', 'ครับ'] - rejoin_formatted_num(tokens) - # output: - # ['IP', ' ', 'address', ' ', 'ของ', 'คุณ', 'คือ', ' ', '127.0.0.1', ' ', 'ครับ'] + + >>> from pythainlp.tokenize._utils import rejoin_formatted_num + >>> tokens = ['ขณะ', 'นี้', 'เวลา', ' ', '12', ':', '00น', ' ', 'อัตรา', + ... 'แลกเปลี่ยน', ' ', '1', ',', '234', '.', '5', ' ', 'baht/zeny'] + >>> rejoin_formatted_num(tokens) + ['ขณะ', 'นี้', 'เวลา', ' ', '12:00น', ' ', 'อัตรา', 'แลกเปลี่ยน', ' ', '1,234.5', ' ', 'baht/zeny'] + >>> tokens = ['IP', ' ', 'address', ' ', 'ของ', 'คุณ', 'คือ', ' ', '127', '.', '0', '.', '0', '.', '1', ' ', 'ครับ'] + >>> rejoin_formatted_num(tokens) + ['IP', ' ', 'address', ' ', 'ของ', 'คุณ', 'คือ', ' ', '127.0.0.1', ' ', 'ครับ'] """ original = "".join(segments) matching_results = _DIGITS_WITH_SEPARATOR.finditer(original) @@ -80,9 +79,11 @@ def strip_whitespace(segments: list[str]) -> list[str]: :rtype: list[str] :Example: - tokens = [" ", "วันนี้ ", "เวลา ", "19.00น"] - strip_whitespace(tokens) - # ["วันนี้", "เวลา", "19.00น"] + + >>> from pythainlp.tokenize._utils import strip_whitespace + >>> tokens = [" ", "วันนี้ ", "เวลา ", "19.00น"] + >>> strip_whitespace(tokens) + ['วันนี้', 'เวลา', '19.00น'] """ segments = [token.strip(" ") for token in segments if token.strip(" ")] diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 845b3d7b2..d1483f9db 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Generic functions of tokenizers""" +"""Generic tokenizer functions for word, sentence, paragraph, and subword.""" from __future__ import annotations @@ -43,12 +43,10 @@ def word_detokenize( :return: the Thai text :rtype: Union[list[list[str]], str] :Example: - :: - from pythainlp.tokenize import word_detokenize - - print(word_detokenize(["เรา", "เล่น"])) - # output: เราเล่น + >>> from pythainlp.tokenize import word_detokenize + >>> word_detokenize(["เรา", "เล่น"]) + 'เราเล่น' """ list_all: list[list[str]] = [] @@ -117,15 +115,18 @@ def word_tokenize( :param str text: text to be tokenized :param str engine: name of the tokenizer to be used - :param pythainlp.util.Trie custom_dict: dictionary trie (some engine may not support) - :param bool keep_whitespace: True to keep whitespace, a common mark - for end of phrase in Thai. - Otherwise, whitespace is omitted. - :param bool join_broken_num: True to rejoin formatted numeric that could be wrongly separated. - Otherwise, formatted numeric could be wrongly separated. + :param pythainlp.util.Trie custom_dict: dictionary trie + (some engines may not support this) + :param bool keep_whitespace: True to keep whitespace, a common + marker for end of phrase in Thai. + Otherwise, whitespace is omitted. + :param bool join_broken_num: True to rejoin formatted numerics + that could be wrongly separated (e.g., time, IP addresses). + Otherwise, formatted numerics could be wrongly separated. :return: list of words :rtype: list[str] + **Options for engine** * *attacut* - wrapper for `AttaCut `_., @@ -145,9 +146,10 @@ def word_tokenize( * *newmm* (default) - "new multi-cut", dictionary-based, maximum matching, constrained by Thai Character Cluster (TCC) boundaries - with improved TCC rules that are used in newmm. - * *newmm-safe* - newmm, with a mechanism to avoid long - processing time for text with continuously ambiguous breaking points + with improved TCC rules. + * *newmm-safe* - newmm with a mechanism to avoid long + processing time for text with continuously ambiguous + breaking points * *nlpo3* - wrapper for a word tokenizer in `nlpO3 `_., adaptation of newmm in Rust (2.5x faster) @@ -176,64 +178,44 @@ def word_tokenize( before starting threads and only read from it during tokenization. :Example: - Tokenize text with different tokenizers:: - - from pythainlp.tokenize import word_tokenize - - text = "โอเคบ่พวกเรารักภาษาบ้านเกิด" - - word_tokenize(text, engine="newmm") - # output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด'] - - word_tokenize(text, engine='attacut') - # output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด'] - - Tokenize text with whitespace omitted:: - - text = "วรรณกรรม ภาพวาด และการแสดงงิ้ว " - - word_tokenize(text, engine="newmm") - # output: - # ['วรรณกรรม', ' ', 'ภาพวาด', ' ', 'และ', 'การแสดง', 'งิ้ว', ' '] - - word_tokenize(text, engine="newmm", keep_whitespace=False) - # output: ['วรรณกรรม', 'ภาพวาด', 'และ', 'การแสดง', 'งิ้ว'] - - Join broken formatted numeric (e.g. time, decimals, IP addresses):: - - text = "เงิน1,234บาท19:32น 127.0.0.1" - - word_tokenize(text, engine="attacut", join_broken_num=False) - # output: - # ['เงิน', '1', ',', '234', 'บาท', '19', ':', '32น', ' ', - # '127', '.', '0', '.', '0', '.', '1'] - - word_tokenize(text, engine="attacut", join_broken_num=True) - # output: - # ['เงิน', '1,234', 'บาท', '19:32น', ' ', '127.0.0.1'] - - Tokenize with default and custom dictionaries:: - - from pythainlp.corpus.common import thai_words - from pythainlp.tokenize import dict_trie - - text = 'ชินโซ อาเบะ เกิด 21 กันยายน' - - word_tokenize(text, engine="newmm") - # output: - # ['ชิน', 'โซ', ' ', 'อา', 'เบะ', ' ', - # 'เกิด', ' ', '21', ' ', 'กันยายน'] - - custom_dict_japanese_name = set(thai_words()) - custom_dict_japanese_name.add('ชินโซ') - custom_dict_japanese_name.add('อาเบะ') - - trie = dict_trie(dict_source=custom_dict_japanese_name) - - word_tokenize(text, engine="newmm", custom_dict=trie) - # output: - # ['ชินโซ', ' ', 'อาเบะ', ' ', - # 'เกิด', ' ', '21', ' ', 'กันยายน'] + Tokenize text with different tokenizers: + + >>> from pythainlp.tokenize import word_tokenize + >>> text = "โอเคบ่พวกเรารักภาษาบ้านเกิด" + >>> word_tokenize(text, engine="newmm") + ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด'] + >>> word_tokenize(text, engine='attacut') # doctest: +SKIP + ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด'] + + Tokenize text with whitespace omitted: + + >>> text = "วรรณกรรม ภาพวาด และการแสดงงิ้ว " + >>> word_tokenize(text, engine="newmm") + ['วรรณกรรม', ' ', 'ภาพวาด', ' ', 'และ', 'การแสดง', 'งิ้ว', ' '] + >>> word_tokenize(text, engine="newmm", keep_whitespace=False) + ['วรรณกรรม', 'ภาพวาด', 'และ', 'การแสดง', 'งิ้ว'] + + Join broken formatted numeric (e.g. time, decimals, IP addresses): + + >>> text = "เงิน1,234บาท19:32น 127.0.0.1" + >>> word_tokenize(text, engine="attacut", join_broken_num=False) # doctest: +SKIP + ['เงิน', '1', ',', '234', 'บาท', '19', ':', '32น', ' ', '127', '.', '0', '.', '0', '.', '1'] + >>> word_tokenize(text, engine="attacut", join_broken_num=True) # doctest: +SKIP + ['เงิน', '1,234', 'บาท', '19:32น', ' ', '127.0.0.1'] + + Tokenize with default and custom dictionaries: + + >>> from pythainlp.corpus.common import thai_words # doctest: +SKIP + >>> from pythainlp.tokenize import dict_trie # doctest: +SKIP + >>> text = 'ชินโซ อาเบะ เกิด 21 กันยายน' + >>> word_tokenize(text, engine="newmm") + ['ชิน', 'โซ', ' ', 'อา', 'เบะ', ' ', 'เกิด', ' ', '21', ' ', 'กันยายน'] + >>> custom_dict_japanese_name = set(thai_words()) # doctest: +SKIP + >>> custom_dict_japanese_name.add('ชินโซ') # doctest: +SKIP + >>> custom_dict_japanese_name.add('อาเบะ') # doctest: +SKIP + >>> trie = dict_trie(dict_source=custom_dict_japanese_name) # doctest: +SKIP + >>> word_tokenize(text, engine="newmm", custom_dict=trie) # doctest: +SKIP + ['ชินโซ', ' ', 'อาเบะ', ' ', 'เกิด', ' ', '21', ' ', 'กันยายน'] """ if not text or not isinstance(text, str): return [] @@ -354,15 +336,12 @@ def indices_words(words: list[str]) -> list[tuple[int, int]]: :rtype: list[tuple[int, int]] :Example: - :: - - from pythainlp.tokenize import indices_words - - indices_words(["สวัสดี", "ครับ"]) - # output: [(0, 5), (6, 9)] - indices_words(["hello", "world"]) - # output: [(0, 4), (5, 9)] + >>> from pythainlp.tokenize.core import indices_words + >>> indices_words(["สวัสดี", "ครับ"]) + [(0, 5), (6, 9)] + >>> indices_words(["hello", "world"]) + [(0, 4), (5, 9)] """ indices = [] start_index = 0 @@ -388,14 +367,12 @@ def map_indices_to_words( :rtype: list[list[str]] :Example: - :: - from pythainlp.tokenize import map_indices_to_words - - indices = [(0, 5), (6, 9)] - sentences = ["สวัสดีครับ"] - map_indices_to_words(indices, sentences) - # output: [['สวัสดี', 'ครับ']] + >>> from pythainlp.tokenize.core import map_indices_to_words + >>> indices = [(0, 5), (6, 9)] + >>> sentences = ["สวัสดีครับ"] + >>> map_indices_to_words(indices, sentences) + [['สวัสดี', 'ครับ']] """ result = [] c = deque(index_list) @@ -424,68 +401,50 @@ def sent_tokenize( ) -> Union[list[str], list[list[str]]]: """Sentence tokenizer. - Tokenizes running text into "sentences". Supports both string and list of strings. + Tokenizes running text into sentences. + Supports both string and list of strings as input. - :param text: the text (string) or list of words (list of strings) to be tokenized - :param str engine: choose among *'crfcut'*, *'whitespace'*, \ - *'whitespace+newline'* - :return: list of split sentences + :param text: text string or list of word tokens to be tokenized + :type text: Union[str, list[str]] + :param str engine: choose among *'crfcut'*, *'whitespace'*, + *'whitespace+newline'* + :return: list of sentences :rtype: Union[list[str], list[list[str]]] + **Options for engine** * *crfcut* - (default) split by CRF trained on TED dataset - * *thaisum* - The implementation of sentence segmenter from \ + * *thaisum* - sentence segmenter from Nakhun Chumpolsathien, 2020 - * *tltk* - split by `TLTK `_., - * *wtp* - split by `wtpsplitaxe `_., \ - It supports many sizes of models. You can use ``wtp`` to use mini model, \ - ``wtp-tiny`` to use ``wtp-bert-tiny`` model (default), \ - ``wtp-mini`` to use ``wtp-bert-mini`` model, \ - ``wtp-base`` to use ``wtp-canine-s-1l`` model, \ - and ``wtp-large`` to use ``wtp-canine-s-12l`` model. - * *whitespace+newline* - split by whitespace and newline. - * *whitespace* - split by whitespace, specifically with \ - :class:`regex` pattern ``r" +"`` + * *tltk* - split by `TLTK `_ + * *wtp* - split by + `wtpsplitaxe `_. + Supports many model sizes: + ``wtp`` uses mini model (default), + ``wtp-tiny`` uses ``wtp-bert-tiny``, + ``wtp-mini`` uses ``wtp-bert-mini``, + ``wtp-base`` uses ``wtp-canine-s-1l``, + ``wtp-large`` uses ``wtp-canine-s-12l``. + * *whitespace+newline* - split by whitespace and newline + * *whitespace* - split by whitespace, + using :class:`regex` pattern ``r" +"`` :Example: - Split the text based on *whitespace*:: - - from pythainlp.tokenize import sent_tokenize - - sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม" - sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\ - และได้รับมอบหมายให้ประจำในระดับภูมิภาค" - - sent_tokenize(sentence_1, engine="whitespace") - # output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม'] - - sent_tokenize(sentence_2, engine="whitespace") - # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ', - # '\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค'] - - Split the text based on *whitespace* and *newline*:: + Split the text based on *whitespace*: - sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม" - sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\ - และได้รับมอบหมายให้ประจำในระดับภูมิภาค" + >>> from pythainlp.tokenize import sent_tokenize + >>> sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม" + >>> sent_tokenize(sentence_1, engine="whitespace") + ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม'] - sent_tokenize(sentence_1, engine="whitespace+newline") - # output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม'] - sent_tokenize(sentence_2, engine="whitespace+newline") - # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ', - '\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค'] + Split the text based on *whitespace* and *newline*: - Split the text using CRF trained on TED dataset:: + >>> sent_tokenize(sentence_1, engine="whitespace+newline") + ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม'] - sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม" - sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\ - และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค" + Split the text using CRF trained on TED dataset: - sent_tokenize(sentence_1, engine="crfcut") - # output: ['ฉันไปประชุมเมื่อวันที่ 11 มีนาคม'] - - sent_tokenize(sentence_2, engine="crfcut") - # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ ', - 'และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค'] + >>> sent_tokenize(sentence_1, engine="crfcut") # doctest: +SKIP + ['ฉันไปประชุมเมื่อวันที่ 11 มีนาคม'] """ if not text or not isinstance(text, (str, list)): return [] @@ -590,36 +549,30 @@ def paragraph_tokenize( :param str text: text to be tokenized :param str engine: the name of paragraph tokenizer :return: list of paragraphs - :rtype: list[List[str]] + :rtype: list[list[str]] + **Options for engine** - * *wtp* - split by `wtpsplitaxe `_., \ - It supports many sizes of models. You can use ``wtp`` to use mini model, \ - ``wtp-tiny`` to use ``wtp-bert-tiny`` model (default), \ - ``wtp-mini`` to use ``wtp-bert-mini`` model, \ - ``wtp-base`` to use ``wtp-canine-s-1l`` model, \ - and ``wtp-large`` to use ``wtp-canine-s-12l`` model. + * *wtp* - split by + `wtpsplitaxe `_. + Supports many model sizes: + ``wtp`` uses mini model (default), + ``wtp-tiny`` uses ``wtp-bert-tiny``, + ``wtp-mini`` uses ``wtp-bert-mini``, + ``wtp-base`` uses ``wtp-canine-s-1l``, + ``wtp-large`` uses ``wtp-canine-s-12l``. :Example: - Split the text based on *wtp*:: - - from pythainlp.tokenize import paragraph_tokenize + Split the text based on *wtp*: - sent = ( - "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต" - +" มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" - +" จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" - ) - - paragraph_tokenize(sent) - # output: [ - # ['(1) '], - # [ - # 'บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต ', - # 'มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด ', - # 'จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ', - # 'ณ ที่นี้' - # ]] + >>> from pythainlp.tokenize import paragraph_tokenize # doctest: +SKIP + >>> sent = ( # doctest: +SKIP + ... "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต" + ... + " มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด" + ... + " จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ณ ที่นี้" + ... ) + >>> paragraph_tokenize(sent) # doctest: +SKIP + [['(1) '], ['บทความนี้ผู้เขียนสังเคราะห์ขึ้นมาจากผลงานวิจัยที่เคยทำมาในอดีต ', 'มิได้ทำการศึกษาค้นคว้าใหม่อย่างกว้างขวางแต่อย่างใด ', 'จึงใคร่ขออภัยในความบกพร่องทั้งปวงมา ', 'ณ ที่นี้']] """ if engine.startswith("wtp"): if "-" not in engine: @@ -670,58 +623,47 @@ def subword_tokenize( :param bool keep_whitespace: keep whitespace :return: list of subwords :rtype: list[str] + **Options for engine** * *dict* - newmm word tokenizer with a syllable dictionary * *etcc* - Enhanced Thai Character Cluster (Inrut et al. 2001) - * *han_solo* - CRF syllable segmenter for Thai that can work in the \ - Thai social media domain. See `PyThaiNLP/Han-solo \ - `_. - * *ssg* - CRF syllable segmenter for Thai. See `ponrawee/ssg \ - `_. - * *tcc* (default) - Thai Character Cluster (Theeramunkong et al. 2000) - * *tcc_p* - Thai Character Cluster + improved rules that are used in newmm - * *tltk* - syllable tokenizer from tltk. See `tltk \ - `_. + * *han_solo* - CRF syllable segmenter for Thai that can work + in the Thai social media domain. See + `PyThaiNLP/Han-solo `_. + * *ssg* - CRF syllable segmenter for Thai. See + `ponrawee/ssg `_. + * *tcc* (default) - Thai Character Cluster + (Theeramunkong et al. 2000) + * *tcc_p* - Thai Character Cluster with improved rules + used in newmm + * *tltk* - syllable tokenizer from tltk. See + `tltk `_. * *wangchanberta* - SentencePiece from wangchanberta model :Example: - Tokenize text into subwords based on *tcc*:: - - from pythainlp.tokenize import subword_tokenize + Tokenize text into subwords based on *tcc*: - text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง" - text_2 = "ความแปลกแยกและพัฒนาการ" + >>> from pythainlp.tokenize import subword_tokenize + >>> text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง" + >>> text_2 = "ความแปลกแยกและพัฒนาการ" + >>> subword_tokenize(text_1, engine='tcc') + ['ยุ', 'ค', 'เริ่ม', 'แร', 'ก', 'ข', 'อ', 'ง', ' ', 'รา', 'ช', 'วงศ์', 'ห', 'มิ', 'ง'] + >>> subword_tokenize(text_2, engine='tcc') + ['ค', 'วา', 'ม', 'แป', 'ล', 'ก', 'แย', 'ก', 'และ', 'พั', 'ฒ', 'นา', 'กา', 'ร'] - subword_tokenize(text_1, engine='tcc') - # output: ['ยุ', 'ค', 'เริ่ม', 'แร', 'ก', - # 'ข', 'อ', 'ง', ' ', 'รา', 'ช', 'ว', 'ง', - # 'ศ', '์', 'ห', 'มิ', 'ง'] + Tokenize text into subwords based on *etcc*: - subword_tokenize(text_2, engine='tcc') - # output: ['ค', 'วา', 'ม', 'แป', 'ล', 'ก', 'แย', 'ก', - 'และ', 'พัฒ','นา', 'กา', 'ร'] + >>> subword_tokenize(text_1, engine='etcc') + ['ยุ', 'ค', 'เริ่', 'ม', 'แร', 'ก', 'ข', 'อ', 'ง', ' ', 'รา', 'ช', 'ว', 'งศ์', 'ห', 'มิง'] + >>> subword_tokenize(text_2, engine='etcc') + ['ค', 'วา', 'ม', 'แป', 'ล', 'ก', 'แย', 'ก', 'และ', 'พัฒ', 'นา', 'กา', 'ร'] - Tokenize text into subwords based on *etcc*:: + Tokenize text into subwords based on *wangchanberta*: - text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง" - text_2 = "ความแปลกแยกและพัฒนาการ" - - subword_tokenize(text_1, engine='etcc') - # output: ['ยุคเริ่มแรกของ ราชวงศ์หมิง'] - - subword_tokenize(text_2, engine='etcc') - # output: ['ความแปลกแยกและ', 'พัฒ', 'นาการ'] - - Tokenize text into subwords based on *wangchanberta*:: - - text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง" - text_2 = "ความแปลกแยกและพัฒนาการ" - - subword_tokenize(text_1, engine='wangchanberta') - # output: ['▁', 'ยุค', 'เริ่มแรก', 'ของ', '▁', 'ราชวงศ์', 'หมิง'] - - subword_tokenize(text_2, engine='wangchanberta') - # output: ['▁ความ', 'แปลก', 'แยก', 'และ', 'พัฒนาการ'] + >>> subword_tokenize(text_1, engine='wangchanberta') # doctest: +SKIP + ['▁', 'ยุค', 'เริ่มแรก', 'ของ', '▁', 'ราชวงศ์', 'หมิง'] + >>> subword_tokenize(text_2, engine='wangchanberta') # doctest: +SKIP + ['▁ความ', 'แปลก', 'แยก', 'และ', 'พัฒนาการ'] """ if not text or not isinstance(text, str): return [] @@ -791,28 +733,26 @@ def syllable_tokenize( :param str text: text to be tokenized :param str engine: the name of syllable tokenizer :param bool keep_whitespace: keep whitespace - :return: list of subwords + :return: list of syllables :rtype: list[str] + **Options for engine** * *dict* - newmm word tokenizer with a syllable dictionary - * *han_solo* - CRF syllable segmenter for Thai that can work in the \ - Thai social media domain. See `PyThaiNLP/Han-solo \ - `_. - * *ssg* - CRF syllable segmenter for Thai. See `ponrawee/ssg \ - `_. - * *tltk* - syllable tokenizer from tltk. See `tltk \ - `_. + * *han_solo* - CRF syllable segmenter for Thai that can work + in the Thai social media domain. See + `PyThaiNLP/Han-solo `_. + * *ssg* - CRF syllable segmenter for Thai. See + `ponrawee/ssg `_. + * *tltk* - syllable tokenizer from tltk. See + `tltk `_. :Example: - :: - - from pythainlp.tokenize import syllable_tokenize - syllable_tokenize("สวัสดีครับ", engine="dict") - # output: ['สวัส', 'ดี', 'ครับ'] - - syllable_tokenize("ประเทศไทย", engine="dict") - # output: ['ประ', 'เทศ', 'ไทย'] + >>> from pythainlp.tokenize import syllable_tokenize + >>> syllable_tokenize("สวัสดีครับ", engine="dict") + ['สวัส', 'ดี', 'ครับ'] + >>> syllable_tokenize("ประเทศไทย", engine="dict") + ['ประ', 'เทศ', 'ไทย'] """ if engine not in ["dict", "han_solo", "ssg", "tltk"]: raise ValueError( @@ -834,13 +774,12 @@ def display_cell_tokenize(text: str) -> list[str]: :rtype: list[str] :Example: - Tokenize Thai text into display cells:: - - from pythainlp.tokenize import display_cell_tokenize + Tokenize Thai text into display cells: - text = "แม่น้ำอยู่ที่ไหน" - display_cell_tokenize(text) - # output: ['แ', 'ม่', 'น้ํ', 'า', 'อ', 'ยู่', 'ที่', 'ไ', 'ห', 'น'] + >>> from pythainlp.tokenize import display_cell_tokenize + >>> text = "แม่น้ำอยู่ที่ไหน" + >>> display_cell_tokenize(text) + ['แ', 'ม่', 'น้ํ', 'า', 'อ', 'ยู่', 'ที่', 'ไ', 'ห', 'น'] """ if not text or not isinstance(text, str): return [] @@ -866,67 +805,49 @@ def display_cell_tokenize(text: str) -> list[str]: class Tokenizer: """Tokenizer class for a custom tokenizer. - This class allows users to pre-define custom dictionary along with - tokenizer and encapsulate them into one single object. - It is an wrapper for both functions, that are - :func:`pythainlp.tokenize.word_tokenize`, - and :func:`pythainlp.util.dict_trie` + This class allows users to pre-define a custom dictionary along with + a tokenizer and encapsulate them into one single object. + It is a wrapper for both :func:`pythainlp.tokenize.word_tokenize` + and :func:`pythainlp.util.dict_trie`. :Example: - Tokenizer object instantiated with :class:`pythainlp.util.Trie`:: + Tokenizer object instantiated with :class:`pythainlp.util.Trie`: - from pythainlp.tokenize import Tokenizer - from pythainlp.corpus.common import thai_words - from pythainlp.util import dict_trie + >>> from pythainlp.tokenize import Tokenizer # doctest: +SKIP + >>> from pythainlp.corpus.common import thai_words # doctest: +SKIP + >>> from pythainlp.util import dict_trie # doctest: +SKIP + >>> custom_words_list = set(thai_words()) # doctest: +SKIP + >>> custom_words_list.add('อะเฟเซีย') # doctest: +SKIP + >>> custom_words_list.add('Aphasia') # doctest: +SKIP + >>> trie = dict_trie(dict_source=custom_words_list) # doctest: +SKIP + >>> text = "อะเฟเซีย (Aphasia*) เป็นอาการผิดปกติของการพูด" # doctest: +SKIP + >>> _tokenizer = Tokenizer(custom_dict=trie, engine='newmm') # doctest: +SKIP + >>> _tokenizer.word_tokenize(text) # doctest: +SKIP + ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิดปกติ', 'ของ', 'การ', 'พูด'] - custom_words_list = set(thai_words()) - custom_words_list.add('อะเฟเซีย') - custom_words_list.add('Aphasia') - trie = dict_trie(dict_source=custom_words_list) + Tokenizer object instantiated with a list of words: - text = "อะเฟเซีย (Aphasia*) เป็นอาการผิดปกติของการพูด" - _tokenizer = Tokenizer(custom_dict=trie, engine='newmm') - _tokenizer.word_tokenize(text) - # output: ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', - 'ผิดปกติ', 'ของ', 'การ', 'พูด'] - - Tokenizer object instantiated with a list of words:: - - text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด" - _tokenizer = Tokenizer(custom_dict=list(thai_words()), engine='newmm') - _tokenizer.word_tokenize(text) - # output: - # ['อะ', 'เฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', - # 'ผิดปกติ', 'ของ', 'การ', 'พูด'] + >>> text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด" # doctest: +SKIP + >>> _tokenizer = Tokenizer(custom_dict=list(thai_words()), engine='newmm') # doctest: +SKIP + >>> _tokenizer.word_tokenize(text) # doctest: +SKIP + ['อะ', 'เฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิดปกติ', 'ของ', 'การ', 'พูด'] Tokenizer object instantiated with a file path containing a list of words separated with *newline* and explicitly setting a new tokenizer - after initiation:: - - PATH_TO_CUSTOM_DICTIONARY = './custom_dictionary.txtt' - - # write a file - with open(PATH_TO_CUSTOM_DICTIONARY, 'w', encoding='utf-8') as f: - f.write('อะเฟเซีย\\nAphasia\\nผิด\\nปกติ') - - text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด" - - # initiate an object from file with `attacut` as tokenizer - _tokenizer = Tokenizer(custom_dict=PATH_TO_CUSTOM_DICTIONARY, \\ - engine='attacut') - - _tokenizer.word_tokenize(text) - # output: - # ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิด', - # 'ปกติ', 'ของ', 'การ', 'พูด'] - - # change tokenizer to `newmm` - _tokenizer.set_tokenizer_engine(engine='newmm') - _tokenizer.word_tokenize(text) - # output: - # ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็นอาการ', 'ผิด', - # 'ปกติ', 'ของการพูด'] + after initiation: + + >>> PATH_TO_CUSTOM_DICTIONARY = './custom_dictionary.txt' # doctest: +SKIP + >>> with open(PATH_TO_CUSTOM_DICTIONARY, 'w', encoding='utf-8') as f: # doctest: +SKIP + ... f.write('อะเฟเซีย\\nAphasia\\nผิด\\nปกติ') + >>> text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด" # doctest: +SKIP + >>> _tokenizer = Tokenizer( # doctest: +SKIP + ... custom_dict=PATH_TO_CUSTOM_DICTIONARY, engine='attacut') + >>> _tokenizer.word_tokenize(text) # doctest: +SKIP + ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิด', 'ปกติ', 'ของ', 'การ', 'พูด'] + >>> _tokenizer.set_tokenizer_engine(engine='newmm') # doctest: +SKIP + >>> _tokenizer.word_tokenize(text) # doctest: +SKIP + ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็นอาการ', 'ผิด', 'ปกติ', 'ของการพูด'] """ def __init__( @@ -938,13 +859,14 @@ def __init__( ) -> None: """Initialize tokenizer object. - :param str custom_dict: a file path, a list of vocaburaies* to be + :param custom_dict: a file path, a list of vocabularies to be used to create a trie, or an instantiated :class:`pythainlp.util.Trie` object. - :param str engine: choose between different options of tokenizer engines - (i.e. *newmm*, *mm*, *longest*, *deepcut*) - :param bool keep_whitespace: True to keep whitespace, a common mark - for end of phrase in Thai + :type custom_dict: Union[Trie, Iterable[str], str, None] + :param str engine: tokenizer engine + (i.e. *newmm*, *mm*, *longest*, *deepcut*) + :param bool keep_whitespace: True to keep whitespace, a common + marker for end of phrase in Thai """ self.__trie_dict: Trie = Trie([]) if custom_dict: @@ -968,13 +890,11 @@ def word_tokenize(self, text: str) -> list[str]: :rtype: list[str] :Example: - :: - - from pythainlp.tokenize import Tokenizer - tokenizer = Tokenizer() - tokenizer.word_tokenize("สวัสดีครับ") - # output: ['สวัสดี', 'ครับ'] + >>> from pythainlp.tokenize import Tokenizer + >>> tokenizer = Tokenizer() + >>> tokenizer.word_tokenize("สวัสดีครับ") + ['สวัสดี', 'ครับ'] """ return word_tokenize( text, @@ -991,13 +911,11 @@ def set_tokenize_engine(self, engine: str) -> None: (i.e. *newmm*, *mm*, *longest*, *deepcut*) :Example: - :: - - from pythainlp.tokenize import Tokenizer - tokenizer = Tokenizer() - tokenizer.set_tokenize_engine("newmm") - tokenizer.word_tokenize("สวัสดีครับ") - # output: ['สวัสดี', 'ครับ'] + >>> from pythainlp.tokenize import Tokenizer + >>> tokenizer = Tokenizer() + >>> tokenizer.set_tokenize_engine("newmm") + >>> tokenizer.word_tokenize("สวัสดีครับ") + ['สวัสดี', 'ครับ'] """ self.__engine = engine diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 627d00d06..07d29777b 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -63,8 +63,8 @@ def segment(text: str) -> list[str]: and Information Technology (ISCIT), pp. 37-40. 2001. :param str text: text to be tokenized into character clusters - :return: list of clusters, tokenized from the text - :return: List[str] + :return: list of clusters tokenized from the text + :rtype: list[str] """ if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index 3ea12d599..fb9440f6b 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Dictionary-based longest-matching Thai word segmentation. Implementation is based -on the codes from Patorn Utenpattanun. +"""Dictionary-based longest-matching Thai word segmentation. +Implementation based on code from Patorn Utenpattanun. :See Also: * `GitHub Repository \ diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py index cbda6131b..490409b02 100644 --- a/pythainlp/tokenize/tcc.py +++ b/pythainlp/tokenize/tcc.py @@ -88,7 +88,7 @@ def tcc_pos(text: str) -> set[int]: """TCC positions :param str text: text to be tokenized into character clusters - :return: list of the ending position of subwords + :return: set of the ending positions of character clusters :rtype: set[int] """ if not text or not isinstance(text, str): diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index a805f2a89..d974dcd06 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -128,15 +128,11 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: :rtype: str :Example: - :: - from pythainlp.tools.misspell import misspell - - sentence = "ภาษาไทยปรากฏครั้งแรกในพุทธศักราช 1826" - - misspell(sent, ratio=0.1) - # output: - ภาษาไทยปรากฏครั้งแรกในกุทธศักราช 1727 + >>> from pythainlp.tools.misspell import misspell # doctest: +SKIP + >>> sentence = "ภาษาไทยปรากฏครั้งแรกในพุทธศักราช 1826" # doctest: +SKIP + >>> misspell(sentence, ratio=0.1) # doctest: +SKIP + 'ภาษาไทยปรากฏครั้งแรกในกุทธศักราช 1727' """ num_misspells = math.floor(len(sentence) * ratio) positions = random.sample(range(len(sentence)), k=num_misspells) diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index b92afa876..5deceee87 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -66,16 +66,15 @@ def is_read_only_mode() -> bool: ``PYTHAINLP_READ_MODE`` are set at the same time. :Example: - :: - import os - from pythainlp import is_read_only_mode - - os.environ["PYTHAINLP_READ_ONLY"] = "1" - print(is_read_only_mode()) # True - - os.environ["PYTHAINLP_READ_ONLY"] = "0" - print(is_read_only_mode()) # False + >>> import os # doctest: +SKIP + >>> from pythainlp import is_read_only_mode # doctest: +SKIP + >>> os.environ["PYTHAINLP_READ_ONLY"] = "1" # doctest: +SKIP + >>> is_read_only_mode() # doctest: +SKIP + True + >>> os.environ["PYTHAINLP_READ_ONLY"] = "0" # doctest: +SKIP + >>> is_read_only_mode() # doctest: +SKIP + False """ import warnings @@ -144,16 +143,15 @@ def is_offline_mode() -> bool: :rtype: bool :Example: - :: - - import os - from pythainlp import is_offline_mode - - os.environ["PYTHAINLP_OFFLINE"] = "1" - print(is_offline_mode()) # True - os.environ["PYTHAINLP_OFFLINE"] = "0" - print(is_offline_mode()) # False + >>> import os # doctest: +SKIP + >>> from pythainlp import is_offline_mode # doctest: +SKIP + >>> os.environ["PYTHAINLP_OFFLINE"] = "1" # doctest: +SKIP + >>> is_offline_mode() # doctest: +SKIP + True + >>> os.environ["PYTHAINLP_OFFLINE"] = "0" # doctest: +SKIP + >>> is_offline_mode() # doctest: +SKIP + False """ val = os.getenv("PYTHAINLP_OFFLINE", "") return val.strip().lower() not in ("", "0", "false", "no", "off") @@ -198,12 +196,10 @@ def get_full_data_path(path: str) -> str: PyThaiNLP data directory (path traversal attempt). :Example: - :: - from pythainlp.tools import get_full_data_path - - get_full_data_path("ttc_freq.txt") - # output: '/root/pythainlp-data/ttc_freq.txt' + >>> from pythainlp.tools import get_full_data_path # doctest: +SKIP + >>> get_full_data_path("ttc_freq.txt") # doctest: +SKIP + '/root/pythainlp-data/ttc_freq.txt' """ return safe_path_join(get_pythainlp_data_path(), path) @@ -232,12 +228,10 @@ def get_pythainlp_data_path() -> str: :rtype: str :Example: - :: - - from pythainlp.tools import get_pythainlp_data_path - get_pythainlp_data_path() - # output: '/root/pythainlp-data' + >>> from pythainlp.tools import get_pythainlp_data_path # doctest: +SKIP + >>> get_pythainlp_data_path() # doctest: +SKIP + '/root/pythainlp-data' """ import warnings @@ -277,12 +271,10 @@ def get_pythainlp_path() -> str: :rtype: str :Example: - :: - - from pythainlp.tools import get_pythainlp_path - get_pythainlp_path() - # output: '/usr/local/lib/python3.6/dist-packages/pythainlp' + >>> from pythainlp.tools import get_pythainlp_path # doctest: +SKIP + >>> get_pythainlp_path() # doctest: +SKIP + '/usr/local/lib/python3/dist-packages/pythainlp' """ package_path = files("pythainlp") # For compatibility, convert to string path if possible diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index 56f289a9b..1cfaf20ba 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -126,7 +126,7 @@ def __init__( :param str engine: machine translation engine :param bool use_gpu: load model using GPU (Default is False) - **Options for engine* + **Options for engine** * *default* - The default engine for each language. * *small100* - A multilingual machine translation model (covering 100 languages) @@ -141,19 +141,19 @@ def __init__( :Example: - Translate text from Thai to English:: + Translate text from Thai to English: - from pythainlp.translate import Translate + >>> from pythainlp.translate import Translate # doctest: +SKIP - th2en = Translate("th", "en") + >>> th2en = Translate("th", "en") # doctest: +SKIP - th2en.translate("ฉันรักแมว") - # output: I love cat. + >>> th2en.translate("ฉันรักแมว") # doctest: +SKIP + I love cat. - Translate text with excluded words:: + Translate text with excluded words: - th2en.translate("ฉันรักแมว", exclude_words=["แมว"]) - # output: I love แมว. + >>> th2en.translate("ฉันรักแมว", exclude_words=["แมว"]) # doctest: +SKIP + I love แมว. """ self.model: Union[ Small100Translator, @@ -227,24 +227,24 @@ def word_translate( :param str src: src language :param str target: target language :param str engine: Word translate engine (the default engine is word2word) - :return: return list word translate or None - :rtype: Union[List[str], None] + :return: list of translated words or None + :rtype: list[str] or None :Example: - Translate word from Thai to English:: + Translate word from Thai to English: - from pythainlp.translate import word_translate + >>> from pythainlp.translate import word_translate # doctest: +SKIP - print(word_translate("แมว", "th", "en")) - # output: ['cat', 'cats', 'kitty', 'kitten', 'Cat'] + >>> print(word_translate("แมว", "th", "en")) # doctest: +SKIP + ['cat', 'cats', 'kitty', 'kitten', 'Cat'] - Translate word from English to Thai:: + Translate word from English to Thai: - from pythainlp.translate import word_translate + >>> from pythainlp.translate import word_translate # doctest: +SKIP - print(word_translate("cat", "en", "th")) - # output: ['แมว', 'แมวป่า', 'ข่วน', 'เลี้ยง', 'อาหาร'] + >>> print(word_translate("cat", "en", "th")) # doctest: +SKIP + ['แมว', 'แมวป่า', 'ข่วน', 'เลี้ยง', 'อาหาร'] """ if engine == "word2word": diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index 3b9e4f5b9..1912e8193 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -104,19 +104,19 @@ def translate( :Example: - Translate text from English to Thai:: + Translate text from English to Thai: - from pythainlp.translate import EnThTranslator + >>> from pythainlp.translate import EnThTranslator # doctest: +SKIP - enth = EnThTranslator() + >>> enth = EnThTranslator() # doctest: +SKIP - enth.translate("I love cat.") - # output: ฉันรักแมว + >>> enth.translate("I love cat.") # doctest: +SKIP + ฉันรักแมว - Translate text from English to Thai with excluded words:: + Translate text from English to Thai with excluded words: - enth.translate("I love cat.", exclude_words=["cat"]) - # output: ฉันรัก cat + >>> enth.translate("I love cat.", exclude_words=["cat"]) # doctest: +SKIP + ฉันรัก cat """ from pythainlp.translate.core import ( @@ -190,19 +190,19 @@ def translate( :Example: - Translate text from Thai to English:: + Translate text from Thai to English: - from pythainlp.translate import ThEnTranslator + >>> from pythainlp.translate import ThEnTranslator # doctest: +SKIP - then = ThEnTranslator() + >>> then = ThEnTranslator() # doctest: +SKIP - then.translate("ฉันรักแมว") - # output: I love cat. + >>> then.translate("ฉันรักแมว") # doctest: +SKIP + I love cat. - Translate text from Thai to English with excluded words:: + Translate text from Thai to English with excluded words: - then.translate("ฉันรักแมว", exclude_words=["แมว"]) - # output: I love แมว. + >>> then.translate("ฉันรักแมว", exclude_words=["แมว"]) # doctest: +SKIP + I love แมว. """ from pythainlp.translate.core import ( diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index a0d3d01f7..ce87a4c8b 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -58,27 +58,25 @@ def translate( :Example: - :: + >>> from pythainlp.translate.small100 import Small100Translator # doctest: +SKIP - from pythainlp.translate.small100 import Small100Translator + >>> mt = Small100Translator() # doctest: +SKIP - mt = Small100Translator() + >>> # Translate text from Thai to English + >>> mt.translate("ทดสอบระบบ", tgt_lang="en") # doctest: +SKIP + 'Testing system' - # Translate text from Thai to English - mt.translate("ทดสอบระบบ", tgt_lang="en") - # output: 'Testing system' + >>> # Translate text from Thai to Chinese + >>> mt.translate("ทดสอบระบบ", tgt_lang="zh") # doctest: +SKIP + '系统测试' - # Translate text from Thai to Chinese - mt.translate("ทดสอบระบบ", tgt_lang="zh") - # output: '系统测试' + >>> # Translate text from Thai to French + >>> mt.translate("ทดสอบระบบ", tgt_lang="fr") # doctest: +SKIP + 'Test du système' - # Translate text from Thai to French - mt.translate("ทดสอบระบบ", tgt_lang="fr") - # output: 'Test du système' - - # Translate text from Thai to English with excluded words - mt.translate("ทดสอบระบบ", tgt_lang="en", exclude_words=["ระบบ"]) - # output: 'Testing ระบบ' + >>> # Translate text from Thai to English with excluded words + >>> mt.translate("ทดสอบระบบ", tgt_lang="en", exclude_words=["ระบบ"]) # doctest: +SKIP + 'Testing ระบบ' """ from pythainlp.translate.core import ( diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index cdd990462..156b35709 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -68,19 +68,19 @@ def translate( :Example: - Translate text from Thai to French:: + Translate text from Thai to French: - from pythainlp.translate.th_fr import ThFrTranslator + >>> from pythainlp.translate.th_fr import ThFrTranslator # doctest: +SKIP - thfr = ThFrTranslator() + >>> thfr = ThFrTranslator() # doctest: +SKIP - thfr.translate("ทดสอบระบบ") - # output: "Test du système." + >>> thfr.translate("ทดสอบระบบ") # doctest: +SKIP + "Test du système." - Translate text from Thai to French with excluded words:: + Translate text from Thai to French with excluded words: - thfr.translate("ทดสอบระบบ", exclude_words=["ระบบ"]) - # output: "Test du ระบบ." + >>> thfr.translate("ทดสอบระบบ", exclude_words=["ระบบ"]) # doctest: +SKIP + "Test du ระบบ." """ from pythainlp.translate.core import ( diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index bd3cfb700..54a035454 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -58,19 +58,19 @@ def translate( :Example: - Translate text from Thai to Chinese:: + Translate text from Thai to Chinese: - from pythainlp.translate import ThZhTranslator + >>> from pythainlp.translate import ThZhTranslator # doctest: +SKIP - thzh = ThZhTranslator() + >>> thzh = ThZhTranslator() # doctest: +SKIP - thzh.translate("ผมรักคุณ") - # output: 我爱你 + >>> thzh.translate("ผมรักคุณ") # doctest: +SKIP + 我爱你 - Translate text from Thai to Chinese with excluded words:: + Translate text from Thai to Chinese with excluded words: - thzh.translate("ผมรักคุณ", exclude_words=["ผม"]) - # output: ผม爱你 + >>> thzh.translate("ผมรักคุณ", exclude_words=["ผม"]) # doctest: +SKIP + ผม爱你 """ from pythainlp.translate.core import ( @@ -133,19 +133,19 @@ def translate( :Example: - Translate text from Chinese to Thai:: + Translate text from Chinese to Thai: - from pythainlp.translate import ZhThTranslator + >>> from pythainlp.translate import ZhThTranslator # doctest: +SKIP - zhth = ZhThTranslator() + >>> zhth = ZhThTranslator() # doctest: +SKIP - zhth.translate("我爱你") - # output: ผมรักคุณนะ + >>> zhth.translate("我爱你") # doctest: +SKIP + ผมรักคุณนะ - Translate text from Chinese to Thai with excluded words:: + Translate text from Chinese to Thai with excluded words: - zhth.translate("我爱你", exclude_words=["我"]) - # output: 我รักคุณนะ + >>> zhth.translate("我爱你", exclude_words=["我"]) # doctest: +SKIP + 我รักคุณนะ """ from pythainlp.translate.core import ( diff --git a/pythainlp/transliterate/core.py b/pythainlp/transliterate/core.py index 6078bf1a2..b18c89b88 100644 --- a/pythainlp/transliterate/core.py +++ b/pythainlp/transliterate/core.py @@ -23,7 +23,7 @@ def romanize( :param str text: A Thai word to be romanized. \ The input should not include whitespace because \ the function is support subwords by splitting whitespace. - :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx, + :param str engine: One of 'royin' (default), 'thai2rom', 'thai2rom_onnx', 'tltk', and 'lookup'. See more in options for engine section. :param str fallback_engine: If engine equals 'lookup', use `fallback_engine` for words that are not in the lookup dictionary. @@ -42,33 +42,24 @@ def romanize( * *lookup* - Look up on Thai-English Transliteration dictionary v1.4 compiled by Wannaphong. :Example: - :: - from pythainlp.transliterate import romanize - - romanize("สามารถ", engine="royin") - # output: 'samant' - - romanize("สามารถ", engine="thai2rom") - # output: 'samat' - - romanize("สามารถ", engine="tltk") - # output: 'samat' - - romanize("ภาพยนตร์", engine="royin") - # output: 'phapn' - - romanize("รส ดี", engine="royin") # subwords - # output: 'rot di' - - romanize("ภาพยนตร์", engine="thai2rom") - # output: 'phapphayon' - - romanize("ภาพยนตร์", engine="thai2rom_onnx") - # output: 'phapphayon' - - romanize("ก็อปปี้", engine="lookup") - # output: 'copy' + >>> from pythainlp.transliterate import romanize + >>> romanize("สามารถ", engine="royin") + 'samant' + >>> romanize("สามารถ", engine="thai2rom") # doctest: +SKIP + 'samat' + >>> romanize("สามารถ", engine="tltk") # doctest: +SKIP + 'samat' + >>> romanize("ภาพยนตร์", engine="royin") + 'phapn' + >>> romanize("รส ดี", engine="royin") # subwords + 'rot di' + >>> romanize("ภาพยนตร์", engine="thai2rom") # doctest: +SKIP + 'phapphayon' + >>> romanize("ภาพยนตร์", engine="thai2rom_onnx") # doctest: +SKIP + 'phapphayon' + >>> romanize("ก็อปปี้", engine="lookup") + 'copy' """ @@ -129,39 +120,28 @@ def transliterate( https://huggingface.co/B-K/umt5-thai-g2p-v2-0.5k :Example: - :: - - from pythainlp.transliterate import transliterate - - transliterate("สามารถ", engine="icu") - # output: 's̄āmārt̄h' - transliterate("สามารถ", engine="ipa") - # output: 'saːmaːrot' - - transliterate("สามารถ", engine="thaig2p") - # output: 's aː ˩˩˦ . m aː t̚ ˥˩' - - transliterate("สามารถ", engine="tltk_ipa") - # output: 'saː5.maːt3' - - transliterate("สามารถ", engine="tltk_g2p") - # output: 'saa4~maat2' - - transliterate("สามารถ", engine="iso_11940") - # output: 's̄āmārt̄h' - - transliterate("ภาพยนตร์", engine="icu") - # output: 'p̣hāphyntr̒' - - transliterate("ภาพยนตร์", engine="ipa") - # output: 'pʰaːpjanot' - - transliterate("ภาพยนตร์", engine="thaig2p") - # output: 'pʰ aː p̚ ˥˩ . pʰ a ˦˥ . j o n ˧' - - transliterate("ภาพยนตร์", engine="iso_11940") - # output: 'p̣hāphyntr' + >>> from pythainlp.transliterate import transliterate + >>> transliterate("สามารถ", engine="icu") # doctest: +SKIP + 's̄āmārt̄h' + >>> transliterate("สามารถ", engine="ipa") # doctest: +SKIP + 'saːmaːrot' + >>> transliterate("สามารถ", engine="thaig2p") # doctest: +SKIP + 's aː ˩˩˦ . m aː t̚ ˥˩' + >>> transliterate("สามารถ", engine="tltk_ipa") # doctest: +SKIP + 'saː5.maːt3' + >>> transliterate("สามารถ", engine="tltk_g2p") # doctest: +SKIP + 'saa4~maat2' + >>> transliterate("สามารถ", engine="iso_11940") + 's̄āmārt̄h' + >>> transliterate("ภาพยนตร์", engine="icu") # doctest: +SKIP + 'p̣hāphyntr̒' + >>> transliterate("ภาพยนตร์", engine="ipa") # doctest: +SKIP + 'pʰaːpjanot' + >>> transliterate("ภาพยนตร์", engine="thaig2p") # doctest: +SKIP + 'pʰ aː p̚ ˥˩ . pʰ a ˦˥ . j o n ˧' + >>> transliterate("ภาพยนตร์", engine="iso_11940") + 'p̣hāphyntr̒' """ if not text or not isinstance(text, str): return "" @@ -200,15 +180,12 @@ def pronunciate(word: str, engine: str = DEFAULT_PRONUNCIATE_ENGINE) -> str: * *w2p* - Thai Word-to-Phoneme :Example: - :: - - from pythainlp.transliterate import pronunciate - - pronunciate("สามารถ", engine="w2p") - # output: 'สา-มาด' - pronunciate("ภาพยนตร์", engine="w2p") - # output: 'พาบ-พะ-ยน' + >>> from pythainlp.transliterate import pronunciate + >>> pronunciate("สามารถ", engine="w2p") # doctest: +SKIP + 'สา-มาด' + >>> pronunciate("ภาพยนตร์", engine="w2p") # doctest: +SKIP + 'พาบ-พะ-ยน' """ if not word or not isinstance(word, str): return "" diff --git a/pythainlp/transliterate/iso_11940.py b/pythainlp/transliterate/iso_11940.py index 630c27584..590b747d6 100644 --- a/pythainlp/transliterate/iso_11940.py +++ b/pythainlp/transliterate/iso_11940.py @@ -129,9 +129,12 @@ def transliterate(word: str) -> str: - """Use ISO 11940 for transliteration - :param str text: Thai text to be transliterated. - :return: A string indicating how the text should be pronounced, according to ISO 11940. + """Transliterate Thai text using ISO 11940. + + :param str word: Thai text to be transliterated. + :return: A string of the text rendered in Latin characters + according to ISO 11940. + :rtype: str """ _str = "" for i in word: diff --git a/pythainlp/transliterate/lookup.py b/pythainlp/transliterate/lookup.py index 52103620e..f593891f1 100644 --- a/pythainlp/transliterate/lookup.py +++ b/pythainlp/transliterate/lookup.py @@ -25,10 +25,12 @@ def follow_rtgs(text: str) -> Optional[bool]: - """Check if the `text` follows romanization defined by Royal Society of Thailand (RTGS). - :param str text: Text to look up. Must be a self-contained word. - :return: True if text follows the definition by RTGS, False otherwise. - `None` means unverified or unknown word. + """Check if *text* follows Royal Thai General System of Transcription (RTGS). + + :param str text: text to look up; must be a self-contained word. + :return: ``True`` if *text* follows the RTGS definition, + ``False`` otherwise. + ``None`` means unverified or unknown word. :rtype: Optional[bool] """ try: @@ -42,10 +44,10 @@ def follow_rtgs(text: str) -> Optional[bool]: def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """Romanize one word. Look up first, call `fallback_func` if not found.""" + """Romanize one word; look up first, call `fallback_func` if not found.""" try: - # try to get 0-th idx of look up result, simply ignore other possible variations. - # not found means no mapping. + # Get 0-th entry; ignore other possible variations. + # IndexError means no mapping. lookup = TRANSLITERATE_DICT[text][TRANSLITERATE_EN][_TRANSLITERATE_IDX] except IndexError: return fallback_func(text) diff --git a/pythainlp/transliterate/spoonerism.py b/pythainlp/transliterate/spoonerism.py index 6b010d151..206b96504 100644 --- a/pythainlp/transliterate/spoonerism.py +++ b/pythainlp/transliterate/spoonerism.py @@ -21,15 +21,12 @@ def puan(word: str, show_pronunciation: bool = True) -> str: :rtype: str :Example: - :: - from pythainlp.transliterate import puan - - puan("นาริน") - # output: 'นิน-รา' - - puan("นาริน", False) - # output: 'นินรา' + >>> from pythainlp.transliterate import puan + >>> puan("นาริน") # doctest: +SKIP + 'นิน-รา' + >>> puan("นาริน", False) # doctest: +SKIP + 'นินรา' """ word = pronunciate(word, engine="w2p") _list_char = [] diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 14ce6ebc5..52ab6a28e 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -93,9 +93,12 @@ def _prepare_sequence_in(self, text: str) -> torch.Tensor: return tensor.to(device) def romanize(self, text: str) -> str: - """:param str text: Thai text to be romanized + """Romanize Thai text to Latin alphabet. + + :param str text: Thai text to be romanized :return: English (more or less) text that spells out how the Thai text should be pronounced. + :rtype: str """ input_tensor = self._prepare_sequence_in(text).view(1, -1) input_length = torch.Tensor([len(text) + 1]).int() diff --git a/pythainlp/transliterate/wunsen.py b/pythainlp/transliterate/wunsen.py index ddd0ec12f..af6b90082 100644 --- a/pythainlp/transliterate/wunsen.py +++ b/pythainlp/transliterate/wunsen.py @@ -83,37 +83,25 @@ def transliterate( ภาษาจีน พ.ศ. 2543) :Example: - :: - from pythainlp.transliterate.wunsen import WunsenTransliterate - - wt = WunsenTransliterate() - - wt.transliterate("ohayō", lang="jp") - # output: 'โอฮาโย' - - wt.transliterate( - "ohayou", lang="jp", jp_input="Hepburn-no diacritic" - ) - # output: 'โอฮาโย' - - wt.transliterate("ohayō", lang="jp", system="RI35") - # output: 'โอะฮะโย' - - wt.transliterate("annyeonghaseyo", lang="ko") - # output: 'อันนย็องฮาเซโย' - - wt.transliterate("xin chào", lang="vi") - # output: 'ซีน จ่าว' - - wt.transliterate("ni3 hao3", lang="zh") - # output: 'หนี เห่า' - - wt.transliterate("ni3 hao3", lang="zh", zh_sandhi=False) - # output: 'หนี่ เห่า' - - wt.transliterate("ni3 hao3", lang="zh", system="RI49") - # output: 'หนี ห่าว' + >>> from pythainlp.transliterate.wunsen import WunsenTransliterate # doctest: +SKIP + >>> wt = WunsenTransliterate() # doctest: +SKIP + >>> wt.transliterate("ohayō", lang="jp") # doctest: +SKIP + 'โอฮาโย' + >>> wt.transliterate("ohayou", lang="jp", jp_input="Hepburn-no diacritic") # doctest: +SKIP + 'โอฮาโย' + >>> wt.transliterate("ohayō", lang="jp", system="RI35") # doctest: +SKIP + 'โอะฮะโย' + >>> wt.transliterate("annyeonghaseyo", lang="ko") # doctest: +SKIP + 'อันนย็องฮาเซโย' + >>> wt.transliterate("xin chào", lang="vi") # doctest: +SKIP + 'ซีน จ่าว' + >>> wt.transliterate("ni3 hao3", lang="zh") # doctest: +SKIP + 'หนี เห่า' + >>> wt.transliterate("ni3 hao3", lang="zh", zh_sandhi=False) # doctest: +SKIP + 'หนี่ เห่า' + >>> wt.transliterate("ni3 hao3", lang="zh", system="RI49") # doctest: +SKIP + 'หนี ห่าว' """ if ( self.lang != lang diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index f83343573..ba56c755c 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -57,14 +57,11 @@ def get_thwiki_lstm() -> dict[str, str]: - """ - Get THWIKI LSTM model paths with validation. - - Returns dictionary with 'wgts_fname' and 'itos_fname' keys containing - validated file paths as strings. + """Get THWIKI LSTM model paths with validation. - :return: Dictionary with model file paths - :raises RuntimeError: If corpus files are not found + :return: dictionary with ``wgts_fname`` and ``itos_fname`` keys + :rtype: dict[str, str] + :raises FileNotFoundError: if corpus files are not found """ wgts_fname = THWIKI_LSTM["wgts_fname"] itos_fname = THWIKI_LSTM["itos_fname"] @@ -206,19 +203,17 @@ def process_thai( def document_vector( text: str, learn: "Learner", data: "DataBunch", agg: str = "mean" ) -> "NDArray[np.float32]": - """This function vectorizes Thai input text into a 400 dimension vector using - :class:`fastai` language model and data bunch. + """Vectorize a Thai sentence into a 400-dimension vector. + + Uses a :class:`fastai` language model and data bunch. + Word vectors are aggregated by mean or summation. - :meth: `document_vector` get document vector using fastai language model - and data bunch - :param str text: text to be vectorized with :class:`fastai` language model. + :param str text: text to vectorize :param learn: :class:`fastai` language model learner :param data: :class:`fastai` data bunch - :param str agg: name of aggregation methods for word embeddings - The available methods are "mean" and "sum" + :param str agg: aggregation method; ``"mean"`` or ``"sum"`` - :return: :class:`numpy.ndarray` of dtype ``numpy.float32`` containing - the document vector produced by the encoder + :return: document vector of shape ``(1, 400)`` :rtype: numpy.typing.NDArray[numpy.float32] :Example: @@ -266,40 +261,39 @@ def merge_wgts( itos_pre: list[str], itos_new: list[str], ) -> dict[str, torch.Tensor]: - """This function is to insert new vocab into an existing model named `wgts` - and update the model's weights for new vocab with the average embedding. + """Insert new vocab into an existing model and update weights. + + New vocab weights are initialised with the average embedding + when not found in the pretrained vocab. - :meth: `merge_wgts` insert pretrained weights and vocab into a new set - of weights and vocab; use average if vocab not in pretrained vocab :param int em_sz: embedding size :param wgts: torch model weights - :param list itos_pre: pretrained list of vocab - :param list itos_new: list of new vocab + :param list[str] itos_pre: pretrained list of vocab + :param list[str] itos_new: list of new vocab :return: merged torch model weights + :rtype: dict[str, torch.Tensor] :Example: - :: - - from pythainlp.ulmfit import merge_wgts - import torch - - wgts = {"0.encoder.weight": torch.randn(5, 3)} - itos_pre = ["แมว", "คน", "หนู"] - itos_new = ["ปลา", "เต่า", "นก"] - em_sz = 3 - - merge_wgts(em_sz, wgts, itos_pre, itos_new) - # output: - # {'0.encoder.weight': tensor([[0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011]]), - # '0.encoder_dp.emb.weight': tensor([[0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011]]), - # '1.decoder.weight': tensor([[0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011], - # [0.5952, 0.4453, 0.0011]])} + + >>> from pythainlp.ulmfit import merge_wgts # doctest: +SKIP + >>> import torch # doctest: +SKIP + + >>> wgts = {"0.encoder.weight": torch.randn(5, 3)} # doctest: +SKIP + >>> itos_pre = ["แมว", "คน", "หนู"] # doctest: +SKIP + >>> itos_new = ["ปลา", "เต่า", "นก"] # doctest: +SKIP + >>> em_sz = 3 # doctest: +SKIP + + >>> merge_wgts(em_sz, wgts, itos_pre, itos_new) # doctest: +SKIP + {'0.encoder.weight': tensor([[0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011]]), + '0.encoder_dp.emb.weight': tensor([[0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011]]), + '1.decoder.weight': tensor([[0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011], + [0.5952, 0.4453, 0.0011]])} """ vocab_size = len(itos_new) enc_wgts = wgts["0.encoder.weight"].numpy().astype("float32", copy=False) diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index 90654fd2c..89625b09d 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -41,27 +41,24 @@ def __init__(self, lang: str = "th") -> None: @staticmethod def tokenizer(text: str) -> list[str]: - """This function tokenizes text using *newmm* engine and the dictionary - specifically for `ulmfit` related functions - (see: `Dictionary file (.txt) \ - `_). - :meth: tokenize text using a frozen newmm engine + """Tokenize text using the newmm engine and the thai2fit dictionary. + :param str text: text to tokenize :return: tokenized text :rtype: list[str] :Example: - Using :func:`pythainlp.ulmfit.ThaiTokenizer.tokenizer` is - similar to :func:`pythainlp.tokenize.word_tokenize` - using *ulmfit* engine. + Using :func:`ThaiTokenizer.tokenizer` is similar to + :func:`pythainlp.tokenize.word_tokenize` with the + ``"ulmfit"`` engine. >>> from pythainlp.ulmfit import ThaiTokenizer >>> from pythainlp.tokenize import word_tokenize >>> >>> text = "อาภรณ์, จินตมยปัญญา ภาวนามยปัญญา" >>> ThaiTokenizer.tokenizer(text) - ['อาภรณ์', ',', ' ', 'จิน', 'ตม', 'ย', 'ปัญญา', + ['อาภรณ์', ',', ' ', 'จิน', 'ตม', 'ย', 'ปัญญา', ' ', 'ภาวนามยปัญญา'] >>> >>> word_tokenize(text, engine='ulmfit') diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index 73765f829..b06ab769b 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -18,21 +18,20 @@ def abbreviation_to_full_text( :param str text: Thai text :param int top_k: Top K - :return: Thai full text with abbreviations converted to full text and cos scores (original text - modified text). - :rtype: List[Tuple[str, Union[float, None]]] + :return: list of ``(full_text, cosine_similarity)`` tuples. + :rtype: list[tuple[str, Optional[float]]] :Example: - :: - from pythainlp.util import abbreviation_to_full_text + >>> from pythainlp.util import abbreviation_to_full_text # doctest: +SKIP - text = "รร.ของเราน่าอยู่" + >>> text = "รร.ของเราน่าอยู่" # doctest: +SKIP - abbreviation_to_full_text(text) - # output: [ - # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), - # ('โรงแรมของเราน่าอยู่', tensor(0.2438)) - # ] + >>> abbreviation_to_full_text(text) # doctest: +SKIP + [ + ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), + ('โรงแรมของเราน่าอยู่', tensor(0.2438)) + ] """ try: from khamyo import replace as _replace diff --git a/pythainlp/util/collate.py b/pythainlp/util/collate.py index c0a821ee1..d0394851a 100644 --- a/pythainlp/util/collate.py +++ b/pythainlp/util/collate.py @@ -43,15 +43,12 @@ def collate(data: Iterable[str], reverse: bool = False) -> list[str]: :rtype: list[str] :Example: - :: - from pythainlp.util import collate - - collate(['ไก่', 'เกิด', 'กาล', 'เป็ด', 'หมู', 'วัว', 'วันที่']) - # output: ['กาล', 'เกิด', 'ไก่', 'เป็ด', 'วันที่', 'วัว', 'หมู'] - - collate(['ไก่', 'เกิด', 'กาล', 'เป็ด', 'หมู', 'วัว', 'วันที่'], \\ - reverse=True) - # output: ['หมู', 'วัว', 'วันที่', 'เป็ด', 'ไก่', 'เกิด', 'กาล'] + >>> from pythainlp.util import collate + >>> collate(['ไก่', 'เกิด', 'กาล', 'เป็ด', 'หมู', 'วัว', 'วันที่']) + ['กาล', 'เกิด', 'ไก่', 'เป็ด', 'วันที่', 'วัว', 'หมู'] + >>> collate(['ไก่', 'เกิด', 'กาล', 'เป็ด', 'หมู', 'วัว', 'วันที่'], + ... reverse=True) + ['หมู', 'วัว', 'วันที่', 'เป็ด', 'ไก่', 'เกิด', 'กาล'] """ return sorted(data, key=_thkey, reverse=reverse) diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index d35d03361..f1d6d3a55 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -137,21 +137,17 @@ def convert_years(year: str, src: str = "be", target: str = "ad") -> str: you should care about the correct calendar. :Example: - :: - from pythainlp.util import convert_years - - # Convert Buddhist Era (BE) to Anno Domini (AD) - convert_years("2566", src="be", target="ad") - # output: '2023' - - # Convert AD to BE - convert_years("2023", src="ad", target="be") - # output: '2566' - - # Convert BE to Rattanakosin Era (RE) - convert_years("2566", src="be", target="re") - # output: '242' + >>> from pythainlp.util import convert_years + >>> # Convert Buddhist Era (BE) to Anno Domini (AD) + >>> convert_years("2566", src="be", target="ad") + '2023' + >>> # Convert AD to BE + >>> convert_years("2023", src="ad", target="be") + '2566' + >>> # Convert BE to Rattanakosin Era (RE) + >>> convert_years("2566", src="be", target="re") + '242' """ output_year = None if src == "be": @@ -237,21 +233,11 @@ def thai_strptime( * *%f* - Microsecond :Example: - :: - from pythainlp.util import thai_strptime + >>> from pythainlp.util import thai_strptime - thai_strptime("15 ก.ค. 2565 09:00:01","%d %B %Y %H:%M:%S") - # output: - # datetime.datetime( - # 2022, - # 7, - # 15, - # 9, - # 0, - # 1, - # tzinfo=zoneinfo.ZoneInfo(key='Asia/Bangkok') - # ) + >>> thai_strptime("15 ก.ค. 2565 09:00:01","%d %B %Y %H:%M:%S") + datetime.datetime(2022, 7, 15, 9, 0, 1, tzinfo=zoneinfo.ZoneInfo(key='Asia/Bangkok')) """ fmt = fmt.replace("%-m", "%m") fmt = fmt.replace("%-d", "%d") @@ -327,18 +313,15 @@ def now_reign_year() -> int: :rtype: int :Example: - :: - from pythainlp.util import now_reign_year - - text = "เป็นปีที่ {reign_year} ในรัชกาลปัจจุบัน"\\ - .format(reign_year=now_reign_year()) - - print(text) - # output: เป็นปีที่ 4 ในรัชการปัจจุบัน + >>> from pythainlp.util import now_reign_year # doctest: +SKIP + >>> text = "เป็นปีที่ {reign_year} ในรัชกาลปัจจุบัน"\\ # doctest: +SKIP + ... .format(reign_year=now_reign_year()) + >>> print(text) # doctest: +SKIP + เป็นปีที่ 11 ในรัชกาลปัจจุบัน """ now_ = datetime.now() - return now_.year - 2015 + return now_.year - 2015 # hard coded def reign_year_to_ad(reign_year: int, reign: int) -> int: @@ -355,17 +338,14 @@ def reign_year_to_ad(reign_year: int, reign: int) -> int: :rtype: int :Example: - :: - - from pythainlp.util import reign_year_to_ad - - print("The 4th reign year of the King Rama X is in", \\ - reign_year_to_ad(4, 10)) - # output: The 4th reign year of the King Rama X is in 2019 - print("The 1st reign year of the King Rama IX is in", \\ - reign_year_to_ad(1, 9)) - # output: The 4th reign year of the King Rama X is in 1946 + >>> from pythainlp.util import reign_year_to_ad + >>> print("The 4th reign year of the King Rama X is in", + ... reign_year_to_ad(4, 10)) + The 4th reign year of the King Rama X is in 2019 + >>> print("The 1st reign year of the King Rama IX is in", + ... reign_year_to_ad(1, 9)) + The 1st reign year of the King Rama IX is in 1946 """ ad = 0 if int(reign) == 10: diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index 0aa4ace20..572370efe 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -80,14 +80,11 @@ def thai_digit_to_arabic_digit(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import thai_digit_to_arabic_digit - - text = "เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท" - - thai_digit_to_arabic_digit(text) - # output: เป็นจำนวน 123,400.25 บาท + >>> from pythainlp.util import thai_digit_to_arabic_digit + >>> text = "เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท" + >>> thai_digit_to_arabic_digit(text) + 'เป็นจำนวน 123,400.25 บาท' """ if not isinstance(text, str): raise TypeError("The text must be str type.") @@ -107,14 +104,11 @@ def arabic_digit_to_thai_digit(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import arabic_digit_to_thai_digit - text = "เป็นจำนวน 123,400.25 บาท" - - arabic_digit_to_thai_digit(text) - # output: เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท + >>> from pythainlp.util import arabic_digit_to_thai_digit + >>> text = "เป็นจำนวน 123,400.25 บาท" + >>> arabic_digit_to_thai_digit(text) + 'เป็นจำนวน ๑๒๓,๔๐๐.๒๕ บาท' """ if not isinstance(text, str): raise TypeError("The text must be str type.") @@ -126,22 +120,21 @@ def arabic_digit_to_thai_digit(text: str) -> str: def digit_to_text(text: str) -> str: - """:param str text: Text with digits such as '1', '2', '๓', '๔' - :return: Text with digits spelled out in Thai - - :Example: - :: + """Spell out digits in Thai. - from pythainlp.util import digit_to_text - - digit_to_text("เบอร์โทร 0812345678") - # output: 'เบอร์โทร ศูนย์แปดหนึ่งสองสามสี่ห้าหกเจ็ดแปด' + :param str text: text with digits such as '1', '2', '๓', '๔' + :return: text with digits spelled out in Thai + :rtype: str - digit_to_text("123") - # output: 'หนึ่งสองสาม' + :Example: - digit_to_text("๕๖๗") - # output: 'ห้าหกเจ็ด' + >>> from pythainlp.util import digit_to_text + >>> digit_to_text("เบอร์โทร 0812345678") + 'เบอร์โทร ศูนย์แปดหนึ่งสองสามสี่ห้าหกเจ็ดแปด' + >>> digit_to_text("123") + 'หนึ่งสองสาม' + >>> digit_to_text("๕๖๗") + 'ห้าหกเจ็ด' """ if not isinstance(text, str): raise TypeError("The text must be str type.") @@ -165,24 +158,23 @@ def text_to_arabic_digit(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import text_to_arabic_digit - - text_to_arabic_digit("ศูนย์") - # output: 0 - text_to_arabic_digit("หนึ่ง") - # output: 1 - text_to_arabic_digit("แปด") - # output: 8 - text_to_arabic_digit("เก้า") - # output: 9 - - # For text that is not digit spelled out in Thai - text_to_arabic_digit("สิบ") == "" - # output: True - text_to_arabic_digit("เก้าร้อย") == "" - # output: True + + >>> from pythainlp.util import text_to_arabic_digit # doctest: +SKIP + + >>> text_to_arabic_digit("ศูนย์") # doctest: +SKIP + 0 + >>> text_to_arabic_digit("หนึ่ง") # doctest: +SKIP + 1 + >>> text_to_arabic_digit("แปด") # doctest: +SKIP + 8 + >>> text_to_arabic_digit("เก้า") # doctest: +SKIP + 9 + + >>> # For text that is not digit spelled out in Thai + >>> text_to_arabic_digit("สิบ") == "" # doctest: +SKIP + True + >>> text_to_arabic_digit("เก้าร้อย") == "" # doctest: +SKIP + True """ if not isinstance(text, str): raise TypeError("The text must be str type.") @@ -202,24 +194,23 @@ def text_to_thai_digit(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import text_to_thai_digit - - text_to_thai_digit("ศูนย์") - # output: ๐ - text_to_thai_digit("หนึ่ง") - # output: ๑ - text_to_thai_digit("แปด") - # output: ๘ - text_to_thai_digit("เก้า") - # output: ๙ - - # For text that is not Thai digit spelled out - text_to_thai_digit("สิบ") == "" - # output: True - text_to_thai_digit("เก้าร้อย") == "" - # output: True + + >>> from pythainlp.util import text_to_thai_digit # doctest: +SKIP + + >>> text_to_thai_digit("ศูนย์") # doctest: +SKIP + ๐ + >>> text_to_thai_digit("หนึ่ง") # doctest: +SKIP + ๑ + >>> text_to_thai_digit("แปด") # doctest: +SKIP + ๘ + >>> text_to_thai_digit("เก้า") # doctest: +SKIP + ๙ + + >>> # For text that is not Thai digit spelled out + >>> text_to_thai_digit("สิบ") == "" # doctest: +SKIP + True + >>> text_to_thai_digit("เก้าร้อย") == "" # doctest: +SKIP + True """ if not isinstance(text, str): raise TypeError("The text must be str type.") diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index 6dd0a6b66..19d72eefc 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -1843,19 +1843,14 @@ def emoji_to_thai( :rtype: str :Example: - :: - from pythainlp.util import emoji_to_thai - - emoji_to_thai("จะมานั่งรถเมล์เหมือนผมก็ได้นะครับ ใกล้ชิดประชาชนดี 😀") - # output: จะมานั่งรถเมล์เหมือนผมก็ได้นะครับ - ใกล้ชิดประชาชนดี :หน้ายิ้มยิงฟัน: - - emoji_to_thai("หิวข้าวอยากกินอาหารญี่ปุ่น 🍣") - # output: หิวข้าวอยากกินอาหารญี่ปุ่น :ซูชิ: - - emoji_to_thai("🇹🇭 นี่คือธงประเทศไทย") - # output: :ธง_ไทย: นี่คือธงประเทศไทย + >>> from pythainlp.util import emoji_to_thai + >>> emoji_to_thai("จะมานั่งรถเมล์เหมือนผมก็ได้นะครับ ใกล้ชิดประชาชนดี 😀") + 'จะมานั่งรถเมล์เหมือนผมก็ได้นะครับ ใกล้ชิดประชาชนดี :หน้ายิ้มยิงฟัน:' + >>> emoji_to_thai("หิวข้าวอยากกินอาหารญี่ปุ่น 🍣") + 'หิวข้าวอยากกินอาหารญี่ปุ่น :ซูชิ:' + >>> emoji_to_thai("🇹🇭 นี่คือธงประเทศไทย") + ':ธง_ไทย: นี่คือธงประเทศไทย' """ return _emoji_regex.sub( lambda match: ( diff --git a/pythainlp/util/encoding.py b/pythainlp/util/encoding.py index 0727c3f74..936d51647 100644 --- a/pythainlp/util/encoding.py +++ b/pythainlp/util/encoding.py @@ -1,4 +1,4 @@ -# -*- coding_utf-8 -*- +# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -13,12 +13,10 @@ def tis620_to_utf8(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import tis620_to_utf8 - - tis620_to_utf8("¡ÃзÃǧÍصÊÒË¡ÃÃÁ") - # output: 'กระทรวงอุตสาหกรรม' + >>> from pythainlp.util import tis620_to_utf8 + >>> tis620_to_utf8("¡ÃзÃǧÍصÊÒË¡ÃÃÁ") + 'กระทรวงอุตสาหกรรม' """ return text.encode("cp1252", "ignore").decode("tis-620") @@ -31,11 +29,9 @@ def to_idna(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import to_idna - to_idna("คนละครึ่ง.com") - # output: 'xn--42caj4e6bk1f5b1j.com' + >>> from pythainlp.util import to_idna + >>> to_idna("คนละครึ่ง.com") + 'xn--42caj4e6bk1f5b1j.com' """ return text.encode("idna").decode("utf-8") diff --git a/pythainlp/util/keyboard.py b/pythainlp/util/keyboard.py index 0a697b1c5..6f70554dc 100644 --- a/pythainlp/util/keyboard.py +++ b/pythainlp/util/keyboard.py @@ -137,12 +137,11 @@ def eng_to_thai(text: str) -> str: :Example: - Intentionally type "ธนาคารแห่งประเทศไทย", but got "Tok8kicsj'xitgmLwmp":: + Intentionally type "ธนาคารแห่งประเทศไทย", but got "Tok8kicsj'xitgmLwmp": - from pythainlp.util import eng_to_thai - - eng_to_thai("Tok8kicsj'xitgmLwmp") - # output: ธนาคารแห่งประเทศไทย + >>> from pythainlp.util import eng_to_thai + >>> eng_to_thai("Tok8kicsj'xitgmLwmp") + 'ธนาคารแห่งประเทศไทย' """ return text.translate(EN_TH_TRANSLATE_TABLE) @@ -159,12 +158,11 @@ def thai_to_eng(text: str) -> str: :Example: - Intentionally type "Bank of Thailand", but got "ฺฟืา นด ธ้ฟรสฟืก":: - - from pythainlp.util import eng_to_thai + Intentionally type "Bank of Thailand", but got "ฺฟืา นด ธ้ฟรสฟืก": - thai_to_eng("ฺฟืา นด ธ้ฟรสฟืก") - # output: 'Bank of Thailand' + >>> from pythainlp.util import thai_to_eng + >>> thai_to_eng("ฺฟืา นด ธ้ฟรสฟืก") + 'Bank of Thailand' """ return text.translate(TH_EN_TRANSLATE_TABLE) @@ -189,23 +187,23 @@ def thai_keyboard_dist(c1: str, c2: str, shift_dist: float = 0.0) -> float: :param str c1: first character :param str c2: second character - :param str shift_dist: return value if they're shifted + :param float shift_dist: return value if they are on shifted keys :return: Euclidean distance between two characters :rtype: float :Example: - from pythainlp.util import thai_keyboard_dist - thai_keyboard_dist("ด", "ะ") - # output: 1.4142135623730951 - thai_keyboard_dist("ฟ", "ฤ") - # output: 0.0 - thai_keyboard_dist("ฟ", "ห") - # output: 1.0 - thai_keyboard_dist("ฟ", "ก") - # output: 2.0 - thai_keyboard_dist("ฟ", "ฤ", 0.5) - # output: 0.5 + >>> from pythainlp.util import thai_keyboard_dist + >>> thai_keyboard_dist("ด", "ะ") + 1.4142135623730951 + >>> thai_keyboard_dist("ฟ", "ฤ") + 0.0 + >>> thai_keyboard_dist("ฟ", "ห") + 1.0 + >>> thai_keyboard_dist("ฟ", "ก") + 2.0 + >>> thai_keyboard_dist("ฟ", "ฤ", 0.5) + 0.5 """ def get_char_coord( diff --git a/pythainlp/util/keywords.py b/pythainlp/util/keywords.py index 74d5ac2dc..01fe0d2a5 100644 --- a/pythainlp/util/keywords.py +++ b/pythainlp/util/keywords.py @@ -29,42 +29,40 @@ def rank( :Example: - Include stopwords when counting word frequencies:: - - from pythainlp.util import rank - - words = ["บันทึก", "เหตุการณ์", " ", "มี", "การ", "บันทึก", \\ - "เป็น", " ", "ลายลักษณ์อักษร"] - - rank(words) - # output: - # Counter( - # { - # ' ': 2, - # 'การ': 1, - # 'บันทึก': 2, - # 'มี': 1, - # 'ลายลักษณ์อักษร': 1, - # 'เป็น': 1, - # 'เหตุการณ์': 1 - # }) - - Exclude stopwords when counting word frequencies:: - - from pythainlp.util import rank - - words = ["บันทึก", "เหตุการณ์", " ", "มี", "การ", "บันทึก", \\ - "เป็น", " ", "ลายลักษณ์อักษร"] - - rank(words) - # output: - # Counter( - # { - # ' ': 2, - # 'บันทึก': 2, - # 'ลายลักษณ์อักษร': 1, - # 'เหตุการณ์': 1 - # }) + Include stopwords when counting word frequencies: + + >>> from pythainlp.util import rank # doctest: +SKIP + + >>> words = ["บันทึก", "เหตุการณ์", " ", "มี", "การ", "บันทึก", # doctest: +SKIP + ... "เป็น", " ", "ลายลักษณ์อักษร"] + + >>> rank(words) # doctest: +SKIP + Counter( + { + ' ': 2, + 'การ': 1, + 'บันทึก': 2, + 'มี': 1, + 'ลายลักษณ์อักษร': 1, + 'เป็น': 1, + 'เหตุการณ์': 1 + }) + + Exclude stopwords when counting word frequencies: + + >>> from pythainlp.util import rank # doctest: +SKIP + + >>> words = ["บันทึก", "เหตุการณ์", " ", "มี", "การ", "บันทึก", # doctest: +SKIP + ... "เป็น", " ", "ลายลักษณ์อักษร"] + + >>> rank(words) # doctest: +SKIP + Counter( + { + ' ': 2, + 'บันทึก': 2, + 'ลายลักษณ์อักษร': 1, + 'เหตุการณ์': 1 + }) """ if not words: return None @@ -86,20 +84,19 @@ def find_keyword(word_list: list[str], min_len: int = 3) -> dict[str, int]: :rtype: dict[str, int] :Example: - :: - from pythainlp.util import find_keyword + >>> from pythainlp.util import find_keyword # doctest: +SKIP - words = ["บันทึก", "เหตุการณ์", "บันทึก", "เหตุการณ์", - " ", "มี", "การ", "บันทึก", "เป็น", " ", "ลายลักษณ์อักษร" - "และ", "การ", "บันทึก","เสียง","ใน","เหตุการณ์"] + >>> words = ["บันทึก", "เหตุการณ์", "บันทึก", "เหตุการณ์", # doctest: +SKIP + ... " ", "มี", "การ", "บันทึก", "เป็น", " ", "ลายลักษณ์อักษร" + ... "และ", "การ", "บันทึก","เสียง","ใน","เหตุการณ์"] - find_keyword(words) - # output: {'บันทึก': 4, 'เหตุการณ์': 3} + >>> find_keyword(words) # doctest: +SKIP + {'บันทึก': 4, 'เหตุการณ์': 3} - find_keyword(words, min_len=1) - # output: {' ': 2, 'บันทึก': 4, 'ลายลักษณ์อักษรและ': 1, - 'เสียง': 1, 'เหตุการณ์': 3} + >>> find_keyword(words, min_len=1) # doctest: +SKIP + {' ': 2, 'บันทึก': 4, 'ลายลักษณ์อักษรและ': 1, + 'เสียง': 1, 'เหตุการณ์': 3} """ word_counter = rank(word_list, exclude_stopwords=True) diff --git a/pythainlp/util/khuap_klam.py b/pythainlp/util/khuap_klam.py index e1fa9855c..97a754278 100644 --- a/pythainlp/util/khuap_klam.py +++ b/pythainlp/util/khuap_klam.py @@ -35,24 +35,23 @@ def check_khuap_klam(word: str) -> Optional[bool]: :rtype: Optional[bool] :Example: - :: - from pythainlp.util import check_khuap_klam + >>> from pythainlp.util import check_khuap_klam # doctest: +SKIP - # True consonant clusters (คำควบกล้ำแท้) - print(check_khuap_klam("กราบ")) # True - print(check_khuap_klam("ปลา")) # True - print(check_khuap_klam("เพราะ")) # True - print(check_khuap_klam("ตรง")) # True + >>> # True consonant clusters (คำควบกล้ำแท้) + >>> print(check_khuap_klam("กราบ")) # True # doctest: +SKIP + >>> print(check_khuap_klam("ปลา")) # True # doctest: +SKIP + >>> print(check_khuap_klam("เพราะ")) # True # doctest: +SKIP + >>> print(check_khuap_klam("ตรง")) # True # doctest: +SKIP - # False consonant clusters (คำควบกล้ำไม่แท้) - print(check_khuap_klam("จริง")) # False - print(check_khuap_klam("ทราย")) # False - print(check_khuap_klam("เศร้า")) # False + >>> # False consonant clusters (คำควบกล้ำไม่แท้) + >>> print(check_khuap_klam("จริง")) # False # doctest: +SKIP + >>> print(check_khuap_klam("ทราย")) # False # doctest: +SKIP + >>> print(check_khuap_klam("เศร้า")) # False # doctest: +SKIP - # Not a consonant cluster - print(check_khuap_klam("แม่")) # None - print(check_khuap_klam("ตา")) # None + >>> # Not a consonant cluster + >>> print(check_khuap_klam("แม่")) # None # doctest: +SKIP + >>> print(check_khuap_klam("ตา")) # None # doctest: +SKIP """ if not word: return None diff --git a/pythainlp/util/lcs.py b/pythainlp/util/lcs.py index e127b68a2..fe8abc815 100644 --- a/pythainlp/util/lcs.py +++ b/pythainlp/util/lcs.py @@ -13,12 +13,10 @@ def longest_common_subsequence(str1: str, str2: str) -> str: :rtype: str :Example: - :: - from pythainlp.util.lcs import longest_common_subsequence - - print(longest_common_subsequence("ABCBDAB", "BDCAB")) - # output: "BDAB" + >>> from pythainlp.util.lcs import longest_common_subsequence + >>> longest_common_subsequence("ABCBDAB", "BDCAB") + 'BDAB' """ m = len(str1) n = len(str2) diff --git a/pythainlp/util/morse.py b/pythainlp/util/morse.py index fa5fdaad4..a942b9be7 100644 --- a/pythainlp/util/morse.py +++ b/pythainlp/util/morse.py @@ -144,15 +144,12 @@ def morse_encode(text: str, lang: str = "th") -> str: :rtype: str :Example: - :: - from pythainlp.util.morse import morse_encode - - print(morse_encode("แมว", lang="th")) - # output: .-.- -- .-- - - print(morse_encode("cat", lang="en")) - # output: -.-. .- - + >>> from pythainlp.util.morse import morse_encode + >>> morse_encode("แมว", lang="th") + '.-.- -- .--' + >>> morse_encode("cat", lang="en") + '-.-. .- -' """ if lang == "th": # Thai return " ".join( @@ -167,25 +164,23 @@ def morse_encode(text: str, lang: str = "th") -> str: def morse_decode(morse_text: str, lang: str = "th") -> str: - """Simple Convert Morse code to text + """Convert Morse code to text. - Thai still have some wrong character problem that\ - can fix by spell corrector. + Thai decoding may produce incorrect characters + that can be fixed with a spell corrector. :param str morse_text: Morse code - :param str lang: Language Code (*th* is Thai and *en* is English) - :return: Text + :param str lang: language code (``'th'`` for Thai, ``'en'`` for English) + :return: decoded text :rtype: str :Example: - :: - - from pythainlp.util.morse import morse_decode - print(morse_decode(".-.- -- .--", lang="th")) - # output: แมว - print(morse_decode("-.-. .- -", lang="en")) - # output: CAT + >>> from pythainlp.util.morse import morse_decode + >>> morse_decode(".-.- -- .--", lang="th") + 'แมว' + >>> morse_decode("-.-. .- -", lang="en") + 'CAT' """ if lang == "th": ans = "".join( diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index 5d78d3b44..02138420f 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -85,15 +85,12 @@ def remove_dangling(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import remove_dangling - - remove_dangling("๊ก") - # output: 'ก' - - remove_dangling("คำ ่ที่สอง") - # output: 'คำ ที่สอง' + >>> from pythainlp.util import remove_dangling + >>> remove_dangling("๊ก") + 'ก' + >>> remove_dangling("คำ ่ที่สอง") + 'คำ ที่สอง' """ text = _RE_REMOVE_DANGLINGS.sub("", text) text = _RE_REMOVE_DANGLINGS_AFTER_SPACE.sub(" ", text) @@ -111,12 +108,10 @@ def remove_dup_spaces(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import remove_dup_spaces - - remove_dup_spaces("ก ข ค") - # output: 'ก ข ค' + >>> from pythainlp.util import remove_dup_spaces + >>> remove_dup_spaces("ก ข ค") + 'ก ข ค' """ while " " in text: text = text.replace(" ", " ") @@ -144,12 +139,10 @@ def remove_tonemark(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import remove_tonemark - remove_tonemark("สองพันหนึ่งร้อยสี่สิบเจ็ดล้านสี่แสนแปดหมื่นสามพันหกร้อยสี่สิบเจ็ด") - # output: สองพันหนึงรอยสีสิบเจ็ดลานสีแสนแปดหมืนสามพันหกรอยสีสิบเจ็ด + >>> from pythainlp.util import remove_tonemark + >>> remove_tonemark("สองพันหนึ่งร้อยสี่สิบเจ็ดล้านสี่แสนแปดหมื่นสามพันหกร้อยสี่สิบเจ็ด") + 'สองพันหนึงรอยสีสิบเจ็ดลานสีแสนแปดหมืนสามพันหกรอยสีสิบเจ็ด' """ for ch in tonemarks: while ch in text: @@ -173,15 +166,12 @@ def remove_zw(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import remove_zw - - remove_zw("สวัสดี\u200bครับ") - # output: 'สวัสดีครับ' - - remove_zw("ภาษา\u200cไทย") - # output: 'ภาษาไทย' + >>> from pythainlp.util import remove_zw + >>> remove_zw("สวัสดี\u200bครับ") + 'สวัสดีครับ' + >>> remove_zw("ภาษา\u200cไทย") + 'ภาษาไทย' """ for ch in _ZERO_WIDTH_CHARS: while ch in text: @@ -202,12 +192,10 @@ def remove_spaces_before_marks(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import remove_spaces_before_marks - remove_spaces_before_marks("พ ุ่มดอกไม้") - # output: 'พุ่มดอกไม้' + >>> from pythainlp.util import remove_spaces_before_marks + >>> remove_spaces_before_marks("พ ุ่มดอกไม้") + 'พุ่มดอกไม้' """ return _RE_REMOVE_SPACES_BEFORE_NONBASE.sub(r"\1\2", text) @@ -228,17 +216,12 @@ def reorder_vowels(text: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import reorder_vowels - - # Two Sara E become Sara Ae - reorder_vowels("เเปลก") - # output: 'แปลก' - - # Reorder tone marks and vowels - reorder_vowels("ก้ำ") - # output: 'กำ้' + >>> from pythainlp.util import reorder_vowels + >>> reorder_vowels("เเปลก") # two Sara E become Sara Ae + 'แปลก' + >>> reorder_vowels("ก้ำ") # reorder tone marks and vowels + 'ก้ำ' """ for pair in _REORDER_PAIRS: text = re.sub(pair[0], pair[1], text) @@ -257,15 +240,12 @@ def remove_repeat_vowels(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import remove_repeat_vowels - remove_repeat_vowels("นานาาา") - # output: 'นานา' - - remove_repeat_vowels("ดีีีี") - # output: 'ดี' + >>> from pythainlp.util import remove_repeat_vowels + >>> remove_repeat_vowels("นานาาา") + 'นานา' + >>> remove_repeat_vowels("ดีีีี") + 'ดี' """ text = reorder_vowels(text) for pair in _NOREPEAT_PAIRS: @@ -302,15 +282,12 @@ def normalize(text: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import normalize - - normalize("เเปลก") # starts with two Sara E - # output: แปลก - normalize("นานาาา") - # output: นานา + >>> from pythainlp.util import normalize + >>> normalize("เเปลก") # starts with two Sara E + 'แปลก' + >>> normalize("นานาาา") + 'นานา' """ text = remove_zw(text) text = remove_dup_spaces(text) @@ -328,16 +305,16 @@ def expand_maiyamok(sent: Union[str, list[str]]) -> list[str]: repetition. This function preprocesses Thai text by replacing Maiyamok with a word being repeated. - :param Union[str, List[str]] sent: sentence (list or string) + :param sent: sentence (list or string) + :type sent: Union[str, list[str]] :return: list of words - :rtype: List[str] + :rtype: list[str] :Example: - :: - from pythainlp.util import expand_maiyamok - expand_maiyamok("คนๆนก") - # output: ['คน', 'คน', 'นก'] + >>> from pythainlp.util import expand_maiyamok + >>> expand_maiyamok("คนๆนก") + ['คน', 'คน', 'นก'] """ if isinstance(sent, str): sent = word_tokenize(sent) @@ -383,23 +360,24 @@ def expand_maiyamok(sent: Union[str, list[str]]) -> list[str]: def maiyamok(sent: Union[str, list[str]]) -> list[str]: """Expand Maiyamok. - Deprecated. Use expand_maiyamok() instead. + .. deprecated:: 5.0.5 + Use :func:`expand_maiyamok` instead. Maiyamok (ๆ) (Unicode U+0E46) is a Thai character indicating word repetition. This function preprocesses Thai text by replacing Maiyamok with a word being repeated. - :param Union[str, List[str]] sent: sentence (list or string) + :param sent: sentence (list or string) + :type sent: Union[str, list[str]] :return: list of words - :rtype: List[str] + :rtype: list[str] :Example: - :: - from pythainlp.util import expand_maiyamok + >>> from pythainlp.util import maiyamok - expand_maiyamok("คนๆนก") - # output: ['คน', 'คน', 'นก'] + >>> maiyamok("คนๆนก") + ['คน', 'คน', 'นก'] """ warn_deprecation( "pythainlp.util.maiyamok", diff --git a/pythainlp/util/numtoword.py b/pythainlp/util/numtoword.py index 1982b09e7..96c29d26b 100644 --- a/pythainlp/util/numtoword.py +++ b/pythainlp/util/numtoword.py @@ -44,18 +44,14 @@ def bahttext(number: float) -> str: :raises TypeError: if *number* is not a numeric type :Example: - :: - from pythainlp.util import bahttext - - bahttext(1) - # output: หนึ่งบาทถ้วน - - bahttext(21) - # output: ยี่สิบเอ็ดบาทถ้วน - - bahttext(200) - # output: สองร้อยบาทถ้วน + >>> from pythainlp.util import bahttext + >>> bahttext(1) + 'หนึ่งบาทถ้วน' + >>> bahttext(21) + 'ยี่สิบเอ็ดบาทถ้วน' + >>> bahttext(200) + 'สองร้อยบาทถ้วน' """ if not isinstance(number, (int, float)): raise TypeError( @@ -92,15 +88,12 @@ def num_to_thaiword(number: Optional[int]) -> str: :rtype: str :Example: - :: - - from pythainlp.util import num_to_thaiword - - num_to_thaiword(1) - # output: หนึ่ง - num_to_thaiword(11) - # output: สิบเอ็ด + >>> from pythainlp.util import num_to_thaiword + >>> num_to_thaiword(1) + 'หนึ่ง' + >>> num_to_thaiword(11) + 'สิบเอ็ด' """ if number is None: return "" diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index 3798f9f34..504486d6e 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -96,19 +96,17 @@ def nectec_to_ipa(pronunciation: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import nectec_to_ipa - - print(nectec_to_ipa("kl-uua-j^-2")) - # output : 'kl uua j ˥˩' + >>> from pythainlp.util import nectec_to_ipa # doctest: +SKIP + >>> print(nectec_to_ipa("kl-uua-j^-2")) # doctest: +SKIP + 'kl uua j ˥˩' References ---------- - Pornpimon Palingoon, Sumonmas Thatphithakkul. Chapter 4 Speech processing \ - and Speech corpus. In: Handbook of Thai Electronic Corpus. \ - 1st ed. p. 122–56. + Pornpimon Palingoon, Sumonmas Thatphithakkul. Chapter 4 Speech + processing and Speech corpus. In: Handbook of Thai Electronic + Corpus. 1st ed. p. 122–56. """ parts = pronunciation.split("-") @@ -127,11 +125,11 @@ def nectec_to_ipa(pronunciation: str) -> str: "f": "f", "h": "h", # The conversion of j depends on its position in the syllable. - # But, unfortunately, the current implementation cannot handle both cases. - # To remove confusions without changing the behavior and breaking existing codes, - # it is suggested that the first key-value mapping of j be simply commented out, - # as it would be overridden by the second one and thus never take effect from the beginning. - # See #846 for a more detailed discussion: https://github.com/PyThaiNLP/pythainlp/issues/846 + # The current implementation cannot handle both cases. + # The first mapping ("j": "y") is commented out because it is + # overridden by the second one ("j": "i") and never takes effect. + # See issue #846: + # https://github.com/PyThaiNLP/pythainlp/issues/846 # "j":"y", "k": "k", "kʰ": "kh", @@ -211,12 +209,11 @@ def ipa_to_rtgs(ipa: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import ipa_to_rtgs + >>> from pythainlp.util import ipa_to_rtgs # doctest: +SKIP - print(ipa_to_rtgs("kluaj")) - # output : 'kluai' + >>> print(ipa_to_rtgs("kluaj")) # doctest: +SKIP + 'kluai' """ rtgs_parts = [] @@ -249,12 +246,11 @@ def remove_tone_ipa(ipa: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import remove_tone_ipa + >>> from pythainlp.util import remove_tone_ipa # doctest: +SKIP - print(remove_tone_ipa("laː˦˥.sa˨˩.maj˩˩˦")) - # output : laː.sa.maj + >>> print(remove_tone_ipa("laː˦˥.sa˨˩.maj˩˩˦")) # doctest: +SKIP + laː.sa.maj """ _list_tone = ["˩˩˦", "˥˩", "˨˩", "˦˥", "˧"] diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index b3b27ce60..4db7c131d 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -27,19 +27,18 @@ def contains_profanity( :rtype: bool :Example: - :: - from pythainlp.util import contains_profanity + >>> from pythainlp.util import contains_profanity # doctest: +SKIP - print(contains_profanity("สวัสดีครับ")) - # output: False + >>> print(contains_profanity("สวัสดีครับ")) # doctest: +SKIP + False - print(contains_profanity("คำหยาบคาย")) - # output: True if the word is in the profanity list + >>> print(contains_profanity("คำหยาบคาย")) # doctest: +SKIP + True if the word is in the profanity list - # Add custom profanity words - print(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) - # output: True + >>> # Add custom profanity words + >>> print(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) # doctest: +SKIP + True """ if not text: return False @@ -76,19 +75,18 @@ def find_profanity( :rtype: list[str] :Example: - :: - from pythainlp.util import find_profanity + >>> from pythainlp.util import find_profanity # doctest: +SKIP - print(find_profanity("สวัสดีครับ")) - # output: [] + >>> print(find_profanity("สวัสดีครับ")) # doctest: +SKIP + [] - print(find_profanity("text with profanity words")) - # output: ['profanity_word1', 'profanity_word2'] + >>> print(find_profanity("text with profanity words")) # doctest: +SKIP + ['profanity_word1', 'profanity_word2'] - # Add custom profanity words - print(find_profanity("คำใหม่", custom_words={"คำใหม่"})) - # output: ['คำใหม่'] + >>> # Add custom profanity words + >>> print(find_profanity("คำใหม่", custom_words={"คำใหม่"})) # doctest: +SKIP + ['คำใหม่'] """ if not text: return [] @@ -130,19 +128,18 @@ def censor_profanity( :rtype: str :Example: - :: - from pythainlp.util import censor_profanity + >>> from pythainlp.util import censor_profanity # doctest: +SKIP - print(censor_profanity("สวัสดีครับ")) - # output: สวัสดีครับ + >>> print(censor_profanity("สวัสดีครับ")) # doctest: +SKIP + สวัสดีครับ - print(censor_profanity("text with profanity word")) - # output: text with *** word + >>> print(censor_profanity("text with profanity word")) # doctest: +SKIP + text with *** word - # Add custom profanity words - print(censor_profanity("คำใหม่", custom_words={"คำใหม่"})) - # output: ****** + >>> # Add custom profanity words + >>> print(censor_profanity("คำใหม่", custom_words={"คำใหม่"})) # doctest: +SKIP + ****** """ if not text: return text diff --git a/pythainlp/util/pronounce.py b/pythainlp/util/pronounce.py index d3605674d..55880f4da 100644 --- a/pythainlp/util/pronounce.py +++ b/pythainlp/util/pronounce.py @@ -30,12 +30,10 @@ def rhyme(word: str) -> list[str]: :rtype: List[str] :Example: - :: - from pythainlp.util import rhyme - - print(rhyme("จีบ")) - # output: ['กลีบ', 'กีบ', 'ครีบ', ...] + >>> from pythainlp.util import rhyme # doctest: +SKIP + >>> rhyme("จีบ") # doctest: +SKIP + ['กลีบ', 'กีบ', 'ครีบ', 'คีบ', 'งีบ', ... ] """ return sorted( i @@ -93,12 +91,10 @@ def thai_consonant_to_spelling(c: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import thai_consonant_to_spelling - print(tone_to_spelling("ก")) - # output: กอ + >>> from pythainlp.util import thai_consonant_to_spelling + >>> print(thai_consonant_to_spelling("ก")) + กอ """ if len(c) == 1 and c in thai_consonants: return c + "อ" @@ -113,12 +109,10 @@ def tone_to_spelling(t: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import tone_to_spelling - print(tone_to_spelling("่")) # ไม้เอก - # output: ไม้เอก + >>> from pythainlp.util import tone_to_spelling + >>> print(tone_to_spelling("่")) # ไม้เอก + ไม้เอก """ if t == "่": return "ไม้เอก" @@ -179,15 +173,12 @@ def spelling(word: str) -> list[str]: :rtype: List[str] :Example: - :: - - from pythainlp.util import spelling - - print(spelling("เรียน")) - # output: ['รอ', 'เอีย', 'นอ', 'เรียน'] - print(spelling("เฝ้า")) - # output: ['ฝอ', 'เอา', 'เฝา', 'ไม้โท', 'เฝ้า'] + >>> from pythainlp.util import spelling + >>> spelling("เรียน") + ['รอ', 'เอีย', 'นอ', 'เรียน'] + >>> spelling("เฝ้า") + ['ฝอ', 'เอา', 'เฝา', 'ไม้โท', 'เฝ้า'] """ if not word or not isinstance(word, str): return [] diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index dd3eb4d2e..98485ab1c 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -49,29 +49,21 @@ def remove_trailing_repeat_consonants( :rtype: str :Example: - :: - from pythainlp.util import remove_trailing_repeat_consonants - from pythainlp.util import dict_trie - - # use default dictionary (pythainlp.corpus.thai_words()) - remove_trailing_repeat_consonants('เริ่ดดดดดดดด') - # output: เริ่ด - - remove_trailing_repeat_consonants('อืมมมมมมมมมมมมมมม') - # output: อืมมม - # "อืมมม" is in the default dictionary - - # use custom dictionary - custom_dict = dict_trie(["อืมมมมม"]) - remove_trailing_repeat_consonants('อืมมมมมมมมมมมมมมม', custom_dict) - # output: อืมมมมม - - # long text - remove_trailing_repeat_consonants('อืมมมมมมมมมมมมม คุณมีบุคลิกที่เริ่ดดดดด '\ - 'ฉันจะให้เกรดดีกับคุณณณ\nนี่เป็นความลับบบบบ') - # output: อืมมม คุณมีบุคลิกที่เริ่ด ฉันจะให้เกรดดีกับคุณ - # นี่เป็นความลับ + >>> from pythainlp.util import remove_trailing_repeat_consonants + >>> from pythainlp.util import dict_trie + >>> # use default dictionary (pythainlp.corpus.thai_words()) + >>> remove_trailing_repeat_consonants('เริ่ดดดดดดดด') + 'เริ่ด' + >>> # "อืมมม" is in the default dictionary + >>> remove_trailing_repeat_consonants('อืมมมมมมมมมมมมมมม') + 'อืมมม' + >>> # use custom dictionary + >>> custom_dict = dict_trie(["อืมมมมม"]) + >>> remove_trailing_repeat_consonants('อืมมมมมมมมมมมมมมม', custom_dict) + 'อืมมมมม' + >>> remove_trailing_repeat_consonants('เริ่ดดด คุณณณ ความลับบบบบ') + 'เริ่ด คุณ ความลับ' """ # use default dictionary if not given if not custom_dict: diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index 09ec59814..0b0f02f71 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -93,17 +93,15 @@ def _clean(w: str) -> str: def spell_syllable(text: str) -> list[str]: """Spell out syllables in Thai word distribution form. - :param str s: Thai syllables only - :return: List of spelled out syllables - :rtype: List[str] + :param str text: Thai syllables only + :return: list of spelled-out syllable components + :rtype: list[str] :Example: - :: - - from pythainlp.util.spell_words import spell_syllable - print(spell_syllable("แมว")) - # output: ['มอ', 'วอ', 'แอ', 'แมว'] + >>> from pythainlp.util.spell_words import spell_syllable + >>> spell_syllable("แมว") + ['มอ', 'วอ', 'แอ', 'แมว'] """ tokens = _cut().word_tokenize(_clean(text)) @@ -122,15 +120,12 @@ def spell_word(text: Optional[str]) -> list[str]: :rtype: list[str] :Example: - :: - - from pythainlp.util.spell_words import spell_word - - print(spell_word("คนดี")) - # output: ['คอ', 'นอ', 'คน', 'ดอ', 'อี', 'ดี', 'คนดี'] - print(spell_word(None)) - # output: [] + >>> from pythainlp.util.spell_words import spell_word + >>> spell_word("คนดี") + ['คอ', 'นอ', 'คน', 'ดอ', 'อี', 'ดี', 'คนดี'] + >>> spell_word(None) + [] """ if not text: return [] diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index f4244c7e3..f96e6bef4 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -153,7 +153,7 @@ def thai_strftime( ) -> str: """Convert :class:`datetime.datetime` into Thai date and time format. - The formatting directives are similar to :func:`datatime.strrftime`. + The formatting directives are similar to :func:`datetime.strftime`. This function uses Thai names and Thai Buddhist Era for these directives: * **%a** - abbreviated weekday name @@ -215,43 +215,42 @@ def thai_strftime( :rtype: str :Example: - :: - from datetime import datetime - from pythainlp.util import thai_strftime + >>> from datetime import datetime + >>> from pythainlp.util import thai_strftime - datetime_obj = datetime(year=2019, month=6, day=9, \\ - hour=5, minute=59, second=0, microsecond=0) + >>> datetime_obj = datetime(year=2019, month=6, day=9, \\ + ... hour=5, minute=59, second=0, microsecond=0) - print(datetime_obj) - # output: 2019-06-09 05:59:00 + >>> print(datetime_obj) + 2019-06-09 05:59:00 - thai_strftime(datetime_obj, "%A %d %B %Y") - # output: 'วันอาทิตย์ 09 มิถุนายน 2562' + >>> thai_strftime(datetime_obj, "%A %d %B %Y") + 'วันอาทิตย์ 09 มิถุนายน 2562' - thai_strftime(datetime_obj, "%a %-d %b %y") # no padding - # output: 'อา 9 มิ.ย. 62' + >>> thai_strftime(datetime_obj, "%a %-d %b %y") # no padding + 'อา 9 มิ.ย. 62' - thai_strftime(datetime_obj, "%a %_d %b %y") # space padding - # output: 'อา 9 มิ.ย. 62' + >>> thai_strftime(datetime_obj, "%a %_d %b %y") # space padding + 'อา 9 มิ.ย. 62' - thai_strftime(datetime_obj, "%a %0d %b %y") # zero padding - # output: 'อา 09 มิ.ย. 62' + >>> thai_strftime(datetime_obj, "%a %0d %b %y") # zero padding + 'อา 09 มิ.ย. 62' - thai_strftime(datetime_obj, "%-H นาฬิกา %-M นาที", thaidigit=True) - # output: '๕ นาฬิกา ๕๙ นาที' + >>> thai_strftime(datetime_obj, "%-H นาฬิกา %-M นาที", thaidigit=True) + '๕ นาฬิกา ๕๙ นาที' - thai_strftime(datetime_obj, "%D (%v)") - # output: '06/09/62 ( 9-มิ.ย.-2562)' + >>> thai_strftime(datetime_obj, "%D (%v)") + '06/09/62 ( 9-มิ.ย.-2562)' - thai_strftime(datetime_obj, "%c") - # output: 'อา 9 มิ.ย. 05:59:00 2562' + >>> thai_strftime(datetime_obj, "%c") + 'อา 9 มิ.ย. 05:59:00 2562' - thai_strftime(datetime_obj, "%H:%M %p") - # output: '01:40 AM' + >>> thai_strftime(datetime_obj, "%H:%M %p") + '05:59 AM' - thai_strftime(datetime_obj, "%H:%M %#p") - # output: '01:40 am' + >>> thai_strftime(datetime_obj, "%H:%M %#p") + '05:59 am' """ thaidate_parts = [] diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index 29bc338fa..82263773e 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -79,15 +79,12 @@ def sound_syllable(syllable: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import sound_syllable - - print(sound_syllable("มา")) - # output: live - - print(sound_syllable("เลข")) - # output: dead + >>> from pythainlp.util import sound_syllable + >>> sound_syllable("มา") + 'live' + >>> sound_syllable("เลข") + 'dead' """ # if len of syllable < 2 if len(syllable) < 2: @@ -184,15 +181,12 @@ def syllable_open_close_detector(syllable: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import syllable_open_close_detector - - print(syllable_open_close_detector("มาก")) - # output: close - print(syllable_open_close_detector("คะ")) - # output: open + >>> from pythainlp.util import syllable_open_close_detector + >>> syllable_open_close_detector("มาก") + 'close' + >>> syllable_open_close_detector("คะ") + 'open' """ consonants = [i for i in syllable if i in thai_consonants] @@ -215,15 +209,12 @@ def syllable_length(syllable: str) -> str: :rtype: str :Example: - :: - from pythainlp.util import syllable_length - - print(syllable_length("มาก")) - # output: long - - print(syllable_length("คะ")) - # output: short + >>> from pythainlp.util import syllable_length + >>> syllable_length("มาก") + 'long' + >>> syllable_length("คะ") + 'short' """ consonants = [i for i in syllable if i in thai_consonants] if len(consonants) <= 3 and any((c in set(short)) for c in syllable): @@ -277,15 +268,12 @@ def tone_detector(syllable: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import tone_detector - - print(tone_detector("มา")) - # output: m - print(tone_detector("ไม้")) - # output: h + >>> from pythainlp.util import tone_detector + >>> tone_detector("มา") + 'm' + >>> tone_detector("ไม้") + 'h' """ s = sound_syllable(syllable) # get consonants diff --git a/pythainlp/util/thai.py b/pythainlp/util/thai.py index 9b97de97f..77a198168 100644 --- a/pythainlp/util/thai.py +++ b/pythainlp/util/thai.py @@ -89,15 +89,12 @@ def is_thai_char(ch: str) -> bool: :rtype: bool :Example: - :: - from pythainlp.util import is_thai_char - - is_thai_char("ก") # THAI CHARACTER KO KAI - # output: True - - is_thai_char("๕") # THAI DIGIT FIVE - # output: True + >>> from pythainlp.util import is_thai_char + >>> is_thai_char("ก") # THAI CHARACTER KO KAI + True + >>> is_thai_char("๕") # THAI DIGIT FIVE + True """ ch_val = ord(ch) if _TH_FIRST_CHAR_ASCII <= ch_val <= _TH_LAST_CHAR_ASCII: @@ -137,21 +134,16 @@ def is_thai(text: str, ignore_chars: str = ".") -> bool: :rtype: bool :Example: - :: - - from pythainlp.util import is_thai - - is_thai("กาลเวลา") - # output: True - - is_thai("กาลเวลา.") - # output: True - is_thai("กาล-เวลา") - # output: False - - is_thai("กาล-เวลา +66", ignore_chars="01234567890+-.,") - # output: True + >>> from pythainlp.util import is_thai + >>> is_thai("กาลเวลา") + True + >>> is_thai("กาลเวลา.") + True + >>> is_thai("กาล-เวลา") + False + >>> is_thai("กาล-เวลา +66", ignore_chars="01234567890+-., ") + True """ if not ignore_chars: @@ -198,21 +190,16 @@ def count_thai(text: str, ignore_chars: str = _DEFAULT_IGNORE_CHARS) -> float: :rtype: float :Example: - :: - - from pythainlp.util import count_thai - - count_thai("ไทยเอ็นแอลพี 3.0") - # output: 100.0 - - count_thai("PyThaiNLP 3.0") - # output: 0.0 - count_thai("ใช้งาน PyThaiNLP 3.0") - # output: 40.0 - - count_thai("ใช้งาน PyThaiNLP 3.0", ignore_chars="") - # output: 30.0 + >>> from pythainlp.util import count_thai + >>> count_thai("ไทยเอ็นแอลพี 3.0") + 100.0 + >>> count_thai("PyThaiNLP 3.0") + 0.0 + >>> count_thai("ใช้งาน PyThaiNLP 3.0") + 40.0 + >>> count_thai("ใช้งาน PyThaiNLP 3.0", ignore_chars="") + 30.0 """ if not text or not isinstance(text, str): return 0.0 @@ -270,12 +257,10 @@ def display_thai_char(ch: str) -> str: :rtype: str :Example: - :: - - from pythainlp.util import display_thai_char - display_thai_char("้") - # output: "_้" + >>> from pythainlp.util import display_thai_char + >>> display_thai_char("้") + '_้' """ if ( ch in thai_above_vowels @@ -294,25 +279,24 @@ def thai_word_tone_detector(word: Optional[str]) -> list[tuple[str, str]]: It uses pythainlp.transliterate.pronunciate for converting word to\ pronunciation. - :param Optional[str] word: Thai word, or None - :return: List of tuples containing Thai pronunciation with tones in each syllable.\ - Tone values: l (low), m (mid), h (high), r (rising), f (falling), or empty if it cannot be detected.\ - Returns [] if word is None or empty. + :param word: Thai word, or None + :type word: str, optional + :return: list of tuples (syllable, tone) for each syllable. + Tone values: ``l`` (low), ``m`` (mid), ``h`` (high), + ``r`` (rising), ``f`` (falling), or empty string + if it cannot be detected. + Returns ``[]`` if word is None or empty. :rtype: list[tuple[str, str]] :Example: - :: - - from pythainlp.util import thai_word_tone_detector - - print(thai_word_tone_detector("คนดี")) - # output: [('คน', 'm'), ('ดี', 'm')] - - print(thai_word_tone_detector("มือถือ")) - # output: [('มือ', 'm'), ('ถือ', 'r')] - print(thai_word_tone_detector(None)) - # output: [] + >>> from pythainlp.util import thai_word_tone_detector + >>> print(thai_word_tone_detector("คนดี")) + [('คน', 'm'), ('ดี', 'm')] + >>> print(thai_word_tone_detector("มือถือ")) + [('มือ', 'm'), ('ถือ', 'r')] + >>> print(thai_word_tone_detector(None)) + [] """ if not word: return [] @@ -325,35 +309,33 @@ def thai_word_tone_detector(word: Optional[str]) -> list[tuple[str, str]]: def count_thai_chars(text: str) -> dict[str, int]: - """Count Thai characters by type + """Count Thai characters by type. - This function will give you numbers of Thai characters by type\ - (consonants, vowels, lead_vowels, follow_vowels, above_vowels,\ - below_vowels, tonemarks, signs, thai_digits, punctuations, non_thai) + Count Thai characters by type: consonants, vowels, lead_vowels, + follow_vowels, above_vowels, below_vowels, tonemarks, signs, + thai_digits, punctuations, and non_thai. - :param str text: Text - :return: Dict with numbers of Thai characters by type + :param str text: input text + :return: dict with counts of Thai characters by type :rtype: dict[str, int] :Example: - :: - - from pythainlp.util import count_thai_chars - - count_thai_chars("ทดสอบภาษาไทย") - # output: { - # 'vowels': 3, - # 'lead_vowels': 1, - # 'follow_vowels': 2, - # 'above_vowels': 0, - # 'below_vowels': 0, - # 'consonants': 9, - # 'tonemarks': 0, - # 'signs': 0, - # 'thai_digits': 0, - # 'punctuations': 0, - # 'non_thai': 0 - # } + + >>> from pythainlp.util import count_thai_chars + >>> count_thai_chars("ทดสอบภาษาไทย") # doctest: +NORMALIZE_WHITESPACE + { + 'vowels': 3, + 'lead_vowels': 1, + 'follow_vowels': 2, + 'above_vowels': 0, + 'below_vowels': 0, + 'consonants': 9, + 'tonemarks': 0, + 'signs': 0, + 'thai_digits': 0, + 'punctuations': 0, + 'non_thai': 0 + } """ _dict = { "vowels": 0, @@ -395,21 +377,20 @@ def count_thai_chars(text: str) -> dict[str, int]: def analyze_thai_text(text: str) -> dict[str, int]: - """Analyzes a string of Thai text and returns a dictionaries, - where each values represents a single classified character from the text. + """Analyze Thai text and return a character count by descriptive name. - The function processes the text character by character and maps each Thai - character to its descriptive name or itself (for consonants and digits). + Process the text character by character and map each Thai character + to its descriptive name or to itself (for consonants and digits). - :param str text: The Thai text string to be analyzed. + :param str text: Thai text string to be analyzed + :return: dict mapping character names to their count in the text :rtype: dict[str, int] - :return: A dictionaries, with each item containing - a single character and a count of 1. - Examples: + :Example: + + >>> from pythainlp.util import analyze_thai_text >>> analyze_thai_text("คนดี") {'ค': 1, 'น': 1, 'ด': 1, 'สระ อี': 1} - >>> analyze_thai_text("เล่น") {'สระ เอ': 1, 'ล': 1, 'ไม้เอก': 1, 'น': 1} diff --git a/pythainlp/util/thai_lunar_date.py b/pythainlp/util/thai_lunar_date.py index 4646fde42..630c4a7e9 100644 --- a/pythainlp/util/thai_lunar_date.py +++ b/pythainlp/util/thai_lunar_date.py @@ -333,21 +333,17 @@ def th_zodiac(year: int, output_type: int = 1) -> Union[str, int]: :rtype: Union[str, int] :Example: - :: - from pythainlp.util import th_zodiac - - # Get Thai zodiac name - th_zodiac(2024, output_type=1) - # output: 'มะโรง' - - # Get English zodiac name - th_zodiac(2024, output_type=2) - # output: 'DRAGON' - - # Get zodiac number - th_zodiac(2024, output_type=3) - # output: 5 + >>> from pythainlp.util import th_zodiac + >>> # Get Thai zodiac name + >>> th_zodiac(2024, output_type=1) + 'มะโรง' + >>> # Get English zodiac name + >>> th_zodiac(2024, output_type=2) + 'DRAGON' + >>> # Get zodiac number + >>> th_zodiac(2024, output_type=3) + 5 """ # Calculate zodiac index result = year % 12 @@ -368,16 +364,13 @@ def to_lunar_date(input_date: date) -> str: :rtype: str :Example: - :: - - from pythainlp.util import to_lunar_date - from datetime import date - - to_lunar_date(date(2024, 1, 1)) - # output: 'แรม 5 ค่ำ เดือน 1' - to_lunar_date(date(2024, 12, 31)) - # output: 'แรม 9 ค่ำ เดือน 2' + >>> from datetime import date + >>> from pythainlp.util import to_lunar_date + >>> to_lunar_date(date(2024, 1, 1)) + 'แรม 5 ค่ำ เดือน 1' + >>> to_lunar_date(date(2024, 12, 31)) + 'ขึ้น 2 ค่ำ เดือน 2' """ # Check if date is within supported range if input_date.year < 1903 or input_date.year > 2460: diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index f5f286aca..9de2bd71c 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -161,9 +161,10 @@ def time_to_thaiword( ) -> str: """Spell out time as Thai words. - :param str time_data: time input, can be a datetime.time object \ - or a datetime.datetime object \ - or a string (in H:M or H:M:S format, using 24-hour clock) + :param time_data: time input; a :class:`datetime.time` object, + a :class:`datetime.datetime` object, or a string + in ``H:M`` or ``H:M:S`` format (24-hour clock) + :type time_data: datetime.time or datetime.datetime or str :param str fmt: time output format * *24h* - 24-hour clock (default) * *6h* - 6-hour clock @@ -176,31 +177,21 @@ def time_to_thaiword( :rtype: str :Example: - :: - time_to_thaiword("8:17") - # output: - # แปดนาฬิกาสิบเจ็ดนาที - - time_to_thaiword("8:17", "6h") - # output: - # สองโมงเช้าสิบเจ็ดนาที - - time_to_thaiword("8:17", "m6h") - # output: - # แปดโมงสิบเจ็ดนาที - - time_to_thaiword("18:30", fmt="m6h") - # output: - # หกโมงครึ่ง - - time_to_thaiword(datetime.time(12, 3, 0)) - # output: - # สิบสองนาฬิกาสามนาที - - time_to_thaiword(datetime.time(12, 3, 0), precision="s") - # output: - # สิบสองนาฬิกาสามนาทีศูนย์วินาที + >>> from datetime import time + >>> from pythainlp.util import time_to_thaiword + >>> time_to_thaiword("8:17") + 'แปดนาฬิกาสิบเจ็ดนาที' + >>> time_to_thaiword("8:17", "6h") + 'สองโมงเช้าสิบเจ็ดนาที' + >>> time_to_thaiword("8:17", "m6h") + 'แปดโมงสิบเจ็ดนาที' + >>> time_to_thaiword("18:30", fmt="m6h") + 'หกโมงครึ่ง' + >>> time_to_thaiword(time(12, 3, 0)) + 'สิบสองนาฬิกาสามนาที' + >>> time_to_thaiword(time(12, 3, 0), precision="s") + 'สิบสองนาฬิกาสามนาทีศูนย์วินาที' """ _time = None @@ -244,11 +235,10 @@ def thaiword_to_time(text: str, padding: bool = True) -> str: :rtype: str :Example: - :: - thaiword_to_time("บ่ายโมงครึ่ง") - # output: - # 13:30 + >>> from pythainlp.util import thaiword_to_time + >>> thaiword_to_time("บ่ายโมงครึ่ง") + '13:30' """ keys_dict = list(_DICT_THAI_TIME.keys()) text = text.replace("กว่า", "").replace("ๆ", "").replace(" ", "") diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index ee84289e7..2c135344a 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -23,27 +23,16 @@ class Trie(Iterable[str]): :param Iterable[str] words: An iterable collection of words to initialize the Trie :Example: - :: - from pythainlp.util import Trie - - # Create a trie with Thai words - trie = Trie(["สวัสดี", "สวัส", "ดี", "ครับ"]) - - # Check if word exists - "สวัสดี" in trie - # output: True - - # Find all prefixes of a word - trie.prefixes("สวัสดีครับ") - # output: ['สวัส', 'สวัสดี'] - - # Add a new word - trie.add("สวัสดีตอนเช้า") - - # Get number of words in trie - len(trie) - # output: 5 + >>> from pythainlp.util import Trie + >>> trie = Trie(["สวัสดี", "สวัส", "ดี", "ครับ"]) + >>> "สวัสดี" in trie + True + >>> trie.prefixes("สวัสดีครับ") + ['สวัส', 'สวัสดี'] + >>> trie.add("สวัสดีตอนเช้า") + >>> len(trie) + 5 """ root: Node diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 61c42384a..82bc00570 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -84,15 +84,12 @@ def thaiword_to_num(word: str) -> int: :rtype: int :Example: - :: - from pythainlp.util import thaiword_to_num - - thaiword_to_num("ศูนย์") - # output: 0 - - thaiword_to_num("สองล้านสามแสนหกร้อยสิบสอง") - # output: 2300612 + >>> from pythainlp.util import thaiword_to_num + >>> thaiword_to_num("ศูนย์") + 0 + >>> thaiword_to_num("สองล้านสามแสนหกร้อยสิบสอง") + 2300612 """ if not isinstance(word, str): @@ -142,19 +139,17 @@ def _decimal_unit(words: list[str]) -> float: def words_to_num(words: list[str]) -> float: - """Thai Words to float + """Thai words to float. - :param str text: Thai words - :return: float of words + :param list[str] words: Thai words (a number broken into tokens) + :return: float value of the words :rtype: float :Example: - :: - - from pythainlp.util import words_to_num - words_to_num(["ห้า", "สิบ", "จุด", "เก้า", "ห้า"]) - # output: 50.95 + >>> from pythainlp.util import words_to_num + >>> words_to_num(["ห้า", "สิบ", "จุด", "เก้า", "ห้า"]) + 50.95 """ num: Union[int, float] = 0 @@ -180,15 +175,12 @@ def text_to_num(text: str) -> list[str]: :rtype: List[str] :Example: - :: - - from pythainlp.util import text_to_num - - text_to_num("เก้าร้อยแปดสิบจุดเก้าห้าบาทนี่คือจำนวนทั้งหมด") - # output: ['980.95', 'บาท', 'นี่', 'คือ', 'จำนวน', 'ทั้งหมด'] - text_to_num("สิบล้านสองหมื่นหนึ่งพันแปดร้อยแปดสิบเก้าบาท") - # output: ['10021889', 'บาท'] + >>> from pythainlp.util import text_to_num + >>> text_to_num("เก้าร้อยแปดสิบจุดเก้าห้าบาทนี่คือจำนวนทั้งหมด") + ['980.95', 'บาท', 'นี่', 'คือ', 'จำนวน', 'ทั้งหมด'] + >>> text_to_num("สิบล้านสองหมื่นหนึ่งพันแปดร้อยแปดสิบเก้าบาท") + ['10021889', 'บาท'] """ _temp = _tokenizer_thaiwords().word_tokenize(text) diff --git a/pythainlp/wangchanberta/__init__.py b/pythainlp/wangchanberta/__init__.py index e99837ef5..073442d6e 100644 --- a/pythainlp/wangchanberta/__init__.py +++ b/pythainlp/wangchanberta/__init__.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +"""Named entity recognition using WangchanBERTa.""" __all__: list[str] = [ "NamedEntityRecognition", "ThaiNameTagger", diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 37c2f2794..42303a5f3 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -43,16 +43,6 @@ class WordVector: """ def __init__(self, model_name: str = "thai2fit_wv") -> None: - """Word Vector class - - :param str model_name: model name - - **Options for model_name** - * *thai2fit_wv* (default) - word vector from thai2fit - * *ltw2v* - word vector from LTW2V: The Large Thai Word2Vec - * *ltw2v_v1.0_15_window* - word2vec from LTW2V 1.0 and 15 window - * *ltw2v_v1.0_5_window* - word2vec from LTW2V v1.0 and 5 window - """ self.model_name: str self.model: "Word2VecKeyedVectors" self.WV_DIM: int diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index d09e5980c..dcc1c45ab 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -90,22 +90,19 @@ def get_sense( for unsupervised word sense disambiguation. :Example: - :: - - from pythainlp.wsd import get_sense - print(get_sense("เขากำลังอบขนมคุกกี้","คุกกี้")) - # output: - # [('โปรแกรมคอมพิวเตอร์ใช้ในทางอินเทอร์เน็ตสำหรับเก็บข้อมูลของผู้ใช้งาน', - # 0.0974416732788086), - # ('ชื่อขนมชนิดหนึ่งจำพวกขนมเค้ก แต่ทำเป็นชิ้นเล็ก ๆ แบน ๆ แล้วอบให้กรอบ', - # 0.09319090843200684)] - - print(get_sense("เว็บนี้ต้องการคุกกี้ในการทำงาน","คุกกี้")) - # output: - # [('โปรแกรมคอมพิวเตอร์ใช้ในทางอินเทอร์เน็ตสำหรับเก็บข้อมูลของผู้ใช้งาน', - # 0.1005704402923584), - # ('ชื่อขนมชนิดหนึ่งจำพวกขนมเค้ก แต่ทำเป็นชิ้นเล็ก ๆ แบน ๆ แล้วอบให้กรอบ', - # 0.12473666667938232)] + + >>> from pythainlp.wsd import get_sense # doctest: +SKIP + >>> print(get_sense("เขากำลังอบขนมคุกกี้","คุกกี้")) # doctest: +SKIP + [('โปรแกรมคอมพิวเตอร์ใช้ในทางอินเทอร์เน็ตสำหรับเก็บข้อมูลของผู้ใช้งาน', + 0.0974416732788086), + ('ชื่อขนมชนิดหนึ่งจำพวกขนมเค้ก แต่ทำเป็นชิ้นเล็ก ๆ แบน ๆ แล้วอบให้กรอบ', + 0.09319090843200684)] + + >>> print(get_sense("เว็บนี้ต้องการคุกกี้ในการทำงาน","คุกกี้")) # doctest: +SKIP + [('โปรแกรมคอมพิวเตอร์ใช้ในทางอินเทอร์เน็ตสำหรับเก็บข้อมูลของผู้ใช้งาน', + 0.1005704402923584), + ('ชื่อขนมชนิดหนึ่งจำพวกขนมเค้ก แต่ทำเป็นชิ้นเล็ก ๆ แบน ๆ แล้วอบให้กรอบ', + 0.12473666667938232)] """ if not custom_dict: custom_dict = _mean_all