Merge pull request #1182 from PyThaiNLP/copilot/improve-pythainlp-performance

Optimize Python performance: eliminate anti-patterns in loops, collections, string operations, data structures, and memory usage

Author: bact

.python-version

@@ -1 +1 @@
-3.12
+3.9

pythainlp/ancient/aksonhan.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+from itertools import chain
+
 from pythainlp import thai_consonants, thai_tonemarks
 from pythainlp.corpus import thai_orst_words
 from pythainlp.tokenize import Tokenizer
@@ -17,7 +19,7 @@
         _dict_aksonhan[i + i + j + i] = i + "ั" + j + i
     _dict_aksonhan[i + i] = "ั" + i
 _set_aksonhan = set(_dict_aksonhan.keys())
-_trie = Trie(list(_dict_aksonhan.keys()) + list(thai_consonants))
+_trie = Trie(chain(_dict_aksonhan.keys(), thai_consonants))
 _tokenizer = Tokenizer(custom_dict=_trie, engine="mm")
 _dict_thai = set(thai_orst_words())  # call Thai words
 

pythainlp/benchmarks/word_tokenization.py

@@ -56,13 +56,11 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict:
     :return: a one-dimension dictionary with keys combined
     :rtype: dict[str, float | str]
     """
-    items = []
-    for k1, kv2 in my_dict.items():
-        for k2, v in kv2.items():
-            new_key = f"{k1}{sep}{k2}"
-            items.append((new_key, v))
-
-    return dict(items)
+    return {
+        f"{k1}{sep}{k2}": v
+        for k1, kv2 in my_dict.items()
+        for k2, v in kv2.items()
+    }
 
 
 def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame:
@@ -259,5 +257,5 @@ def _find_words_correctly_tokenised(
     """
     ref_b = dict(zip(ref_boundaries, [1] * len(ref_boundaries)))
 
-    labels = tuple(map(lambda x: ref_b.get(x, 0), predicted_boundaries))
+    labels = tuple(ref_b.get(x, 0) for x in predicted_boundaries)
     return labels

pythainlp/cli/benchmark.py

@@ -13,7 +13,7 @@
 
 def _read_file(path):
     with open(path, encoding="utf-8") as f:
-        lines = map(lambda r: r.strip(), f.readlines())
+        lines = (r.strip() for r in f.readlines())
     return list(lines)
 
 

pythainlp/corpus/common.py

@@ -372,7 +372,7 @@ def find_synonyms(word: str) -> list[str]:
             list_synonym.extend(synonyms["synonym"][idx])
             list_synonym.append(synonyms["word"][idx])
 
-    list_synonym = sorted(list(set(list_synonym)))
+    list_synonym = sorted(set(list_synonym))
 
     if word in list_synonym:  # remove same word
         list_synonym.remove(word)

pythainlp/corpus/core.py

@@ -188,8 +188,8 @@ def get_corpus_default_db(name: str, version: str = "") -> str | None:
     with open(default_db_path, encoding="utf-8-sig") as fh:
         corpus_db = json.load(fh)
 
-    if name in list(corpus_db.keys()):
-        if version in list(corpus_db[name]["versions"].keys()):
+    if name in corpus_db:
+        if version in corpus_db[name]["versions"]:
             return path_pythainlp_corpus(
                 corpus_db[name]["versions"][version]["filename"]
             )
@@ -247,7 +247,7 @@ def get_corpus_path(
     CUSTOMIZE: dict[str, str] = {
         # "the corpus name":"path"
     }
-    if name in list(CUSTOMIZE):
+    if name in CUSTOMIZE:
         return CUSTOMIZE[name]
 
     default_path = get_corpus_default_db(name=name, version=version)

pythainlp/corpus/oscar.py

@@ -28,9 +28,8 @@ def word_freqs() -> list[tuple[str, int]]:
     path = str(path)
 
     with open(path, encoding="utf-8-sig") as f:
-        lines = list(f.readlines())
-        del lines[0]
-        for line in lines:
+        next(f)  # Skip header line
+        for line in f:
             temp = line.strip().split(",")
             if len(temp) >= 2:
                 if temp[0] != " " and '"' not in temp[0]:
@@ -51,10 +50,9 @@ def unigram_word_freqs() -> dict[str, int]:
     path = str(path)
 
     with open(path, encoding="utf-8-sig") as fh:
-        lines = list(fh.readlines())
-        del lines[0]
-        for i in lines:
-            temp = i.strip().split(",")
+        next(fh)  # Skip header line
+        for line in fh:
+            temp = line.strip().split(",")
             if temp[0] != " " and '"' not in temp[0]:
                 freqs[temp[0]] = int(temp[-1])
             elif temp[0] == " ":

pythainlp/corpus/tnc.py

@@ -29,8 +29,7 @@ def word_freqs() -> list[tuple[str, int]]:
     Credit: Korakot Chaovavanich https://www.facebook.com/groups/thainlp/posts/434330506948445
     """
     freqs: list[tuple[str, int]] = []
-    lines = list(get_corpus(_UNIGRAM_FILENAME))
-    for line in lines:
+    for line in get_corpus(_UNIGRAM_FILENAME):
         word_freq = line.split("\t")
         if len(word_freq) >= 2:
             freqs.append((word_freq[0], int(word_freq[1])))
@@ -42,9 +41,8 @@ def unigram_word_freqs() -> dict[str, int]:
     """Get unigram word frequency from Thai National Corpus (TNC)
     """
     freqs: dict[str, int] = defaultdict(int)
-    lines = list(get_corpus(_UNIGRAM_FILENAME))
-    for i in lines:
-        _temp = i.strip().split("	")
+    for line in get_corpus(_UNIGRAM_FILENAME):
+        _temp = line.strip().split("	")
         if len(_temp) >= 2:
             freqs[_temp[0]] = int(_temp[-1])
 
@@ -60,10 +58,14 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]:
         return freqs
     path = str(path)
 
-    with open(path, encoding="utf-8-sig") as fh:
-        for i in fh.readlines():
-            temp = i.strip().split("	")
-            freqs[(temp[0], temp[1])] = int(temp[-1])
+    try:
+        with open(path, encoding="utf-8-sig") as fh:
+            for line in fh:
+                temp = line.strip().split("	")
+                if len(temp) >= 3:
+                    freqs[(temp[0], temp[1])] = int(temp[-1])
+    except (IOError, OSError):
+        pass
 
     return freqs
 
@@ -77,9 +79,13 @@ def trigram_word_freqs() -> dict[tuple[str, str, str], int]:
         return freqs
     path = str(path)
 
-    with open(path, encoding="utf-8-sig") as fh:
-        for i in fh.readlines():
-            temp = i.strip().split("	")
-            freqs[(temp[0], temp[1], temp[2])] = int(temp[-1])
+    try:
+        with open(path, encoding="utf-8-sig") as fh:
+            for line in fh:
+                temp = line.strip().split("	")
+                if len(temp) >= 4:
+                    freqs[(temp[0], temp[1], temp[2])] = int(temp[-1])
+    except (IOError, OSError):
+        pass
 
     return freqs

pythainlp/corpus/ttc.py

@@ -24,8 +24,7 @@ def word_freqs() -> list[tuple[str, int]]:
     <https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/ttc_freq.txt>`_)
     """
     freqs: list[tuple[str, int]] = []
-    lines = list(get_corpus(_UNIGRAM_FILENAME))
-    for line in lines:
+    for line in get_corpus(_UNIGRAM_FILENAME):
         word_freq = line.split("\t")
         if len(word_freq) >= 2:
             freqs.append((word_freq[0], int(word_freq[1])))
@@ -38,9 +37,8 @@ def unigram_word_freqs() -> dict[str, int]:
     """
     freqs: dict[str, int] = defaultdict(int)
 
-    lines = list(get_corpus(_UNIGRAM_FILENAME))
-    for i in lines:
-        temp = i.strip().split("	")
+    for line in get_corpus(_UNIGRAM_FILENAME):
+        temp = line.strip().split("	")
         if len(temp) >= 2:
             freqs[temp[0]] = int(temp[-1])