Merge pull request #1257 from PyThaiNLP/copilot/refactor-delayed-imports

Delay external dependency imports to class initialization

Author: bact

pythainlp/augment/lm/fasttext.py

@@ -5,9 +5,6 @@
 
 import itertools
 
-from gensim.models.fasttext import FastText as FastText_gensim
-from gensim.models.keyedvectors import KeyedVectors
-
 from pythainlp.tokenize import word_tokenize
 
 
@@ -20,6 +17,9 @@ class FastTextAug:
     def __init__(self, model_path: str):
         """:param str model_path: path of model file
         """
+        from gensim.models.fasttext import FastText as FastText_gensim
+        from gensim.models.keyedvectors import KeyedVectors
+
         if model_path.endswith(".bin"):
             self.model = FastText_gensim.load_facebook_vectors(model_path)
         elif model_path.endswith(".vec"):

pythainlp/augment/lm/wangchanberta.py

@@ -3,16 +3,16 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-from transformers import (
-    CamembertTokenizer,
-    pipeline,
-)
-
 model_name = "airesearch/wangchanberta-base-att-spm-uncased"
 
 
 class Thai2transformersAug:
     def __init__(self):
+        from transformers import (
+            CamembertTokenizer,
+            pipeline,
+        )
+
         self.model_name = "airesearch/wangchanberta-base-att-spm-uncased"
         self.target_tokenizer = CamembertTokenizer
         self.tokenizer = CamembertTokenizer.from_pretrained(

pythainlp/coref/_fastcoref.py

@@ -3,21 +3,25 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-import spacy
-
 
 class FastCoref:
     def __init__(
         self,
         model_name,
-        nlp=spacy.blank("th"),
+        nlp=None,
         device: str = "cpu",
         type: str = "FCoref",
     ) -> None:
         if type == "FCoref":
             from fastcoref import FCoref as _model
         else:
             from fastcoref import LingMessCoref as _model
+
+        if nlp is None:
+            import spacy
+
+            nlp = spacy.blank("th")
+
         self.model_name = model_name
         self.nlp = nlp
         self.model = _model(self.model_name, device=device, nlp=self.nlp)

pythainlp/coref/han_coref.py

@@ -3,13 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-import spacy
-
 from pythainlp.coref._fastcoref import FastCoref
 
 
 class HanCoref(FastCoref):
-    def __init__(self, device: str = "cpu", nlp=spacy.blank("th")) -> None:
+    def __init__(self, device: str = "cpu", nlp=None) -> None:
         super().__init__(
             model_name="pythainlp/han-coref-v1.0", device=device, nlp=nlp
         )

pythainlp/parse/transformers_ud.py

@@ -12,25 +12,22 @@
 from __future__ import annotations
 
 import os
-from typing import List, Union
-
-import numpy
-import torch
-import ufal.chu_liu_edmonds
-from transformers import (
-    AutoConfig,
-    AutoModelForQuestionAnswering,
-    AutoModelForTokenClassification,
-    AutoTokenizer,
-    TokenClassificationPipeline,
-)
-from transformers.utils import cached_file
+from typing import Union
 
 
 class Parse:
     def __init__(
         self, model: str = "KoichiYasuoka/deberta-base-thai-ud-head"
     ) -> None:
+        from transformers import (
+            AutoConfig,
+            AutoModelForQuestionAnswering,
+            AutoModelForTokenClassification,
+            AutoTokenizer,
+            TokenClassificationPipeline,
+        )
+        from transformers.utils import cached_file
+
         if model is None:
             model = "KoichiYasuoka/deberta-base-thai-ud-head"
         self.tokenizer = AutoTokenizer.from_pretrained(model)
@@ -57,7 +54,11 @@ def __init__(
             model=t, tokenizer=self.tokenizer
         )
 
-    def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]:
+    def __call__(self, text: str, tag: str = "str") -> Union[list[list[str]], str]:
+        import numpy
+        import torch
+        import ufal.chu_liu_edmonds
+
         w = [
             (t["start"], t["end"], t["entity_group"])
             for t in self.deprel(text)

pythainlp/summarize/keybert.py

@@ -14,19 +14,21 @@
 
 from collections import Counter
 from collections.abc import Iterable
-from typing import Optional, Union
-
-import numpy as np
-from transformers import pipeline
+from typing import TYPE_CHECKING, Optional, Union
 
 from pythainlp.corpus import thai_stopwords
 from pythainlp.tokenize import word_tokenize
 
+if TYPE_CHECKING:
+    import numpy as np
+
 
 class KeyBERT:
     def __init__(
         self, model_name: str = "airesearch/wangchanberta-base-att-spm-uncased"
     ):
+        from transformers import pipeline
+
         self.ft_pipeline = pipeline(
             "feature-extraction",
             tokenizer=model_name,
@@ -136,8 +138,9 @@ def extract_keywords(
             return [kw for kw, _ in keywords]
 
     def embed(self, docs: Union[str, list[str]]) -> np.ndarray:
-        """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer.
-        """
+        """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer."""
+        import numpy as np
+
         embs = self.ft_pipeline(docs)
         if isinstance(docs, str) or len(docs) == 1:
             # embed doc. return shape = [1, hidden_size]
@@ -206,6 +209,8 @@ def _rank_keywords(
     keywords: list[str],
     max_keywords: int,
 ) -> list[tuple[str, float]]:
+    import numpy as np
+
     def l2_norm(v: np.ndarray) -> np.ndarray:
         vec_size = v.shape[1]
         result = np.divide(

pythainlp/summarize/mt5.py

@@ -6,8 +6,6 @@
 
 from __future__ import annotations
 
-from transformers import MT5ForConditionalGeneration, T5Tokenizer
-
 from pythainlp.summarize import CPE_KMUTT_THAI_SENTENCE_SUM
 
 
@@ -38,6 +36,8 @@ def __init__(
         :param str pretrained_mt5_model_name: Name of pretrained model.
             If empty (default), uses google/mt5-{model_size}.
         """
+        from transformers import MT5ForConditionalGeneration, T5Tokenizer
+
         model_name = ""
         if not pretrained_mt5_model_name:
             if model_size not in ["small", "base", "large", "xl", "xxl"]:

pythainlp/wangchanberta/core.py

@@ -7,19 +7,24 @@
 import warnings
 from typing import Union
 
-from transformers import (
-    CamembertTokenizer,
-    pipeline,
-)
-
 from pythainlp.tokenize import word_tokenize
 
 _model_name = "wangchanberta-base-att-spm-uncased"
-_tokenizer = CamembertTokenizer.from_pretrained(
-    f"airesearch/{_model_name}", revision="main"
-)
-if _model_name == "wangchanberta-base-att-spm-uncased":
-    _tokenizer.additional_special_tokens = ["<s>NOTUSED", "</s>NOTUSED", "<_>"]
+_tokenizer = None
+
+
+def _get_tokenizer():
+    """Get the tokenizer, initializing it if necessary."""
+    global _tokenizer
+    if _tokenizer is None:
+        from transformers import CamembertTokenizer
+
+        _tokenizer = CamembertTokenizer.from_pretrained(
+            f"airesearch/{_model_name}", revision="main"
+        )
+        if _model_name == "wangchanberta-base-att-spm-uncased":
+            _tokenizer.additional_special_tokens = ["<s>NOTUSED", "</s>NOTUSED", "<_>"]
+    return _tokenizer
 
 
 class ThaiNameTagger:
@@ -33,11 +38,13 @@ def __init__(self, dataset_name: str = "thainer", grouped_entities: bool = True)
             * *thainer* - ThaiNER dataset
         :param bool grouped_entities: grouped entities
         """
+        from transformers import pipeline
+
         self.dataset_name = dataset_name
         self.grouped_entities = grouped_entities
         self.classify_tokens = pipeline(
             task="ner",
-            tokenizer=_tokenizer,
+            tokenizer=_get_tokenizer(),
             model=f"airesearch/{_model_name}",
             revision=f"finetuned@{self.dataset_name}-ner",
             ignore_labels=[],
@@ -226,4 +233,4 @@ def segment(text: str) -> list[str]:
     if not text or not isinstance(text, str):
         return []
 
-    return _tokenizer.tokenize(text)
+    return _get_tokenizer().tokenize(text)

pythainlp/word_vector/core.py

@@ -3,13 +3,15 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-from gensim.models import KeyedVectors
-from gensim.models.keyedvectors import Word2VecKeyedVectors
-from numpy import ndarray, zeros
+from typing import TYPE_CHECKING
 
 from pythainlp.corpus import get_corpus_path
 from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize
 
+if TYPE_CHECKING:
+    from gensim.models.keyedvectors import Word2VecKeyedVectors
+    from numpy import ndarray
+
 WV_DIM = 300  # word vector dimension
 
 _MODEL_NAME = "thai2fit_wv"
@@ -48,6 +50,8 @@ def load_wordvector(self, model_name: str) -> None:
 
         :param str model_name: model name
         """
+        from gensim.models import KeyedVectors
+
         self.model_name = model_name
         self.model = KeyedVectors.load_word2vec_format(
             get_corpus_path(self.model_name),
@@ -290,6 +294,8 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray:
             0.40506999,  1.58591403,  0.63869202, -0.702155  ,  1.62977601,
             4.52269109, -0.70760502,  0.50952601, -0.914392  ,  0.70673105]])
         """
+        from numpy import zeros
+
         vec = zeros((1, self.WV_DIM))
 
         words = self.tokenize(text)