Merge pull request #1194 from PyThaiNLP/copilot/improve-pythainlp-docs

Add missing docstrings and examples to utility and tokenization functions

Author: bact

pythainlp/tokenize/core.py

@@ -329,6 +329,26 @@ def word_tokenize(
 
 
 def indices_words(words):
+    """Convert a list of words to a list of character index pairs.
+
+    This function takes a list of words and returns the start and end
+    character indices for each word in the original text.
+
+    :param list words: list of words
+    :return: list of tuples (start_index, end_index) for each word
+    :rtype: list[tuple[int, int]]
+
+    :Example:
+    ::
+
+        from pythainlp.tokenize import indices_words
+
+        indices_words(['สวัสดี', 'ครับ'])
+        # output: [(0, 5), (6, 9)]
+
+        indices_words(['hello', 'world'])
+        # output: [(0, 4), (5, 9)]
+    """
     indices = []
     start_index = 0
     for word in words:
@@ -340,6 +360,26 @@ def indices_words(words):
 
 
 def map_indices_to_words(index_list, sentences):
+    """Map character index pairs to actual words from sentences.
+
+    This function takes a list of character index pairs and a list of
+    sentences, then extracts the corresponding words from the sentences.
+
+    :param list index_list: list of tuples (start_index, end_index)
+    :param list sentences: list of sentences (strings)
+    :return: list of lists containing extracted words for each sentence
+    :rtype: list[list[str]]
+
+    :Example:
+    ::
+
+        from pythainlp.tokenize import map_indices_to_words
+
+        indices = [(0, 5), (6, 9)]
+        sentences = ['สวัสดีครับ']
+        map_indices_to_words(indices, sentences)
+        # output: [['สวัสดี', 'ครับ']]
+    """
     result = []
     c = deque(index_list)
     n_sum = 0
@@ -735,6 +775,17 @@ def syllable_tokenize(
         <https://github.com/ponrawee/ssg>`_.
         * *tltk* - syllable tokenizer from tltk. See `tltk \
         <https://pypi.org/project/tltk/>`_.
+
+    :Example:
+    ::
+
+        from pythainlp.tokenize import syllable_tokenize
+
+        syllable_tokenize("สวัสดีครับ", engine="dict")
+        # output: ['สวัส', 'ดี', 'ครับ']
+
+        syllable_tokenize("ประเทศไทย", engine="dict")
+        # output: ['ประ', 'เทศ', 'ไทย']
     """
     if engine not in ["dict", "han_solo", "ssg", "tltk"]:
         raise ValueError(
@@ -890,6 +941,15 @@ def word_tokenize(self, text: str) -> list[str]:
         :param str text: text to be tokenized
         :return: list of words, tokenized from the text
         :rtype: list[str]
+
+        :Example:
+        ::
+
+            from pythainlp.tokenize import Tokenizer
+
+            tokenizer = Tokenizer()
+            tokenizer.word_tokenize("สวัสดีครับ")
+            # output: ['สวัสดี', 'ครับ']
         """
         return word_tokenize(
             text,
@@ -904,5 +964,15 @@ def set_tokenize_engine(self, engine: str) -> None:
 
         :param str engine: choose between different options of tokenizer engines
                            (i.e. *newmm*, *mm*, *longest*, *deepcut*)
+
+        :Example:
+        ::
+
+            from pythainlp.tokenize import Tokenizer
+
+            tokenizer = Tokenizer()
+            tokenizer.set_tokenize_engine("newmm")
+            tokenizer.word_tokenize("สวัสดีครับ")
+            # output: ['สวัสดี', 'ครับ']
         """
         self.__engine = engine

pythainlp/util/date.py

@@ -139,6 +139,23 @@ def convert_years(year: str, src="be", target="ad") -> str:
     because Thailand has change the Thai calendar in 1941.
     If you are the time traveler or the historian, \
     you should care about the correct calendar.
+
+    :Example:
+    ::
+
+        from pythainlp.util import convert_years
+
+        # Convert Buddhist Era (BE) to Anno Domini (AD)
+        convert_years("2566", src="be", target="ad")
+        # output: '2023'
+
+        # Convert AD to BE
+        convert_years("2023", src="ad", target="be")
+        # output: '2566'
+
+        # Convert BE to Rattanakosin Era (RE)
+        convert_years("2566", src="be", target="re")
+        # output: '242'
     """
     output_year = None
     if src == "be":

pythainlp/util/digitconv.py

@@ -117,6 +117,20 @@ def arabic_digit_to_thai_digit(text: str) -> str:
 def digit_to_text(text: str) -> str:
     """:param str text: Text with digits such as '1', '2', '๓', '๔'
     :return: Text with digits spelled out in Thai
+
+    :Example:
+    ::
+
+        from pythainlp.util import digit_to_text
+
+        digit_to_text("เบอร์โทร 0812345678")
+        # output: 'เบอร์โทร ศูนย์แปดหนึ่งสองสามสี่ห้าหกเจ็ดแปด'
+
+        digit_to_text("123")
+        # output: 'หนึ่งสองสาม'
+
+        digit_to_text("๕๖๗")
+        # output: 'ห้าหกเจ็ด'
     """
     if not text or not isinstance(text, str):
         raise TypeError("The text must be str type.")

pythainlp/util/normalize.py

@@ -153,6 +153,17 @@ def remove_zw(text: str) -> str:
     :param str text: input text
     :return: text without zero-width characters
     :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.util import remove_zw
+
+        remove_zw("สวัสดี\u200bครับ")
+        # output: 'สวัสดีครับ'
+
+        remove_zw("ภาษา\u200cไทย")
+        # output: 'ภาษาไทย'
     """
     for ch in _ZERO_WIDTH_CHARS:
         while ch in text:
@@ -175,6 +186,19 @@ def reorder_vowels(text: str) -> str:
     :param str text: input text
     :return: text with vowels and tone marks in the standard logical order
     :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.util import reorder_vowels
+
+        # Two Sara E become Sara Ae
+        reorder_vowels("เเปลก")
+        # output: 'แปลก'
+
+        # Reorder tone marks and vowels
+        reorder_vowels("ก้ำ")
+        # output: 'กำ้'
     """
     for pair in _REORDER_PAIRS:
         text = re.sub(pair[0], pair[1], text)
@@ -191,6 +215,17 @@ def remove_repeat_vowels(text: str) -> str:
     :param str text: input text
     :return: text without repeating Thai vowels, tone marks, and signs
     :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.util import remove_repeat_vowels
+
+        remove_repeat_vowels("นานาาา")
+        # output: 'นานา'
+
+        remove_repeat_vowels("ดีีีี")
+        # output: 'ดี'
     """
     text = reorder_vowels(text)
     for pair in _NOREPEAT_PAIRS:

pythainlp/util/thai_lunar_date.py

@@ -315,6 +315,23 @@ def th_zodiac(year: int, output_type: int = 1) -> str | int:
 
     :return: The Zodiac name or number corresponding to the input year.
     :rtype: Union[str, int]
+
+    :Example:
+    ::
+
+        from pythainlp.util import th_zodiac
+
+        # Get Thai zodiac name
+        th_zodiac(2024, output_type=1)
+        # output: 'มะโรง'
+
+        # Get English zodiac name
+        th_zodiac(2024, output_type=2)
+        # output: 'DRAGON'
+
+        # Get zodiac number
+        th_zodiac(2024, output_type=3)
+        # output: 5
     """
     # Calculate zodiac index
     result = year % 12
@@ -333,6 +350,18 @@ def to_lunar_date(input_date: date) -> str:
     :param date input_date: date of the day.
     :return: Thai text lunar date
     :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.util import to_lunar_date
+        from datetime import date
+
+        to_lunar_date(date(2024, 1, 1))
+        # output: 'แรม 5 ค่ำ เดือน 1'
+
+        to_lunar_date(date(2024, 12, 31))
+        # output: 'แรม 9 ค่ำ เดือน 2'
     """
     # Check if date is within supported range
     if input_date.year < 1903 or input_date.year > 2460:

pythainlp/util/trie.py

@@ -12,6 +12,38 @@
 
 
 class Trie(Iterable[str]):
+    """Trie data structure for efficient prefix-based word search.
+
+    A Trie (prefix tree) is a tree-like data structure used to store
+    a collection of strings. It enables fast retrieval of words with
+    common prefixes, making it ideal for dictionary-based tokenization
+    and autocomplete features.
+
+    :param Iterable[str] words: An iterable collection of words to initialize the Trie
+
+    :Example:
+    ::
+
+        from pythainlp.util import Trie
+
+        # Create a trie with Thai words
+        trie = Trie(['สวัสดี', 'สวัส', 'ดี', 'ครับ'])
+
+        # Check if word exists
+        'สวัสดี' in trie
+        # output: True
+
+        # Find all prefixes of a word
+        trie.prefixes('สวัสดีครับ')
+        # output: ['สวัส', 'สวัสดี']
+
+        # Add a new word
+        trie.add('สวัสดีตอนเช้า')
+
+        # Get number of words in trie
+        len(trie)
+        # output: 5
+    """
     class Node:
         __slots__ = "end", "children"