diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py
index 3621a7896..47dfea180 100644
--- a/pythainlp/augment/lm/fasttext.py
+++ b/pythainlp/augment/lm/fasttext.py
@@ -5,9 +5,6 @@
import itertools
-from gensim.models.fasttext import FastText as FastText_gensim
-from gensim.models.keyedvectors import KeyedVectors
-
from pythainlp.tokenize import word_tokenize
@@ -20,6 +17,9 @@ class FastTextAug:
def __init__(self, model_path: str):
""":param str model_path: path of model file
"""
+ from gensim.models.fasttext import FastText as FastText_gensim
+ from gensim.models.keyedvectors import KeyedVectors
+
if model_path.endswith(".bin"):
self.model = FastText_gensim.load_facebook_vectors(model_path)
elif model_path.endswith(".vec"):
diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py
index 5624e8171..bb4d611a7 100644
--- a/pythainlp/augment/lm/wangchanberta.py
+++ b/pythainlp/augment/lm/wangchanberta.py
@@ -3,16 +3,16 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
-from transformers import (
- CamembertTokenizer,
- pipeline,
-)
-
model_name = "airesearch/wangchanberta-base-att-spm-uncased"
class Thai2transformersAug:
def __init__(self):
+ from transformers import (
+ CamembertTokenizer,
+ pipeline,
+ )
+
self.model_name = "airesearch/wangchanberta-base-att-spm-uncased"
self.target_tokenizer = CamembertTokenizer
self.tokenizer = CamembertTokenizer.from_pretrained(
diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py
index 32c537b3b..2d203aa0a 100644
--- a/pythainlp/coref/_fastcoref.py
+++ b/pythainlp/coref/_fastcoref.py
@@ -3,14 +3,12 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
-import spacy
-
class FastCoref:
def __init__(
self,
model_name,
- nlp=spacy.blank("th"),
+ nlp=None,
device: str = "cpu",
type: str = "FCoref",
) -> None:
@@ -18,6 +16,12 @@ def __init__(
from fastcoref import FCoref as _model
else:
from fastcoref import LingMessCoref as _model
+
+ if nlp is None:
+ import spacy
+
+ nlp = spacy.blank("th")
+
self.model_name = model_name
self.nlp = nlp
self.model = _model(self.model_name, device=device, nlp=self.nlp)
diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py
index 4fd6d2f6e..046c34273 100644
--- a/pythainlp/coref/han_coref.py
+++ b/pythainlp/coref/han_coref.py
@@ -3,13 +3,11 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
-import spacy
-
from pythainlp.coref._fastcoref import FastCoref
class HanCoref(FastCoref):
- def __init__(self, device: str = "cpu", nlp=spacy.blank("th")) -> None:
+ def __init__(self, device: str = "cpu", nlp=None) -> None:
super().__init__(
model_name="pythainlp/han-coref-v1.0", device=device, nlp=nlp
)
diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py
index 4a01e8d83..b3f432106 100644
--- a/pythainlp/parse/transformers_ud.py
+++ b/pythainlp/parse/transformers_ud.py
@@ -12,25 +12,22 @@
from __future__ import annotations
import os
-from typing import List, Union
-
-import numpy
-import torch
-import ufal.chu_liu_edmonds
-from transformers import (
- AutoConfig,
- AutoModelForQuestionAnswering,
- AutoModelForTokenClassification,
- AutoTokenizer,
- TokenClassificationPipeline,
-)
-from transformers.utils import cached_file
+from typing import Union
class Parse:
def __init__(
self, model: str = "KoichiYasuoka/deberta-base-thai-ud-head"
) -> None:
+ from transformers import (
+ AutoConfig,
+ AutoModelForQuestionAnswering,
+ AutoModelForTokenClassification,
+ AutoTokenizer,
+ TokenClassificationPipeline,
+ )
+ from transformers.utils import cached_file
+
if model is None:
model = "KoichiYasuoka/deberta-base-thai-ud-head"
self.tokenizer = AutoTokenizer.from_pretrained(model)
@@ -57,7 +54,11 @@ def __init__(
model=t, tokenizer=self.tokenizer
)
- def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]:
+ def __call__(self, text: str, tag: str = "str") -> Union[list[list[str]], str]:
+ import numpy
+ import torch
+ import ufal.chu_liu_edmonds
+
w = [
(t["start"], t["end"], t["entity_group"])
for t in self.deprel(text)
diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py
index 5b2bfec48..2a0dbe998 100644
--- a/pythainlp/summarize/keybert.py
+++ b/pythainlp/summarize/keybert.py
@@ -14,19 +14,21 @@
from collections import Counter
from collections.abc import Iterable
-from typing import Optional, Union
-
-import numpy as np
-from transformers import pipeline
+from typing import TYPE_CHECKING, Optional, Union
from pythainlp.corpus import thai_stopwords
from pythainlp.tokenize import word_tokenize
+if TYPE_CHECKING:
+ import numpy as np
+
class KeyBERT:
def __init__(
self, model_name: str = "airesearch/wangchanberta-base-att-spm-uncased"
):
+ from transformers import pipeline
+
self.ft_pipeline = pipeline(
"feature-extraction",
tokenizer=model_name,
@@ -136,8 +138,9 @@ def extract_keywords(
return [kw for kw, _ in keywords]
def embed(self, docs: Union[str, list[str]]) -> np.ndarray:
- """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer.
- """
+ """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer."""
+ import numpy as np
+
embs = self.ft_pipeline(docs)
if isinstance(docs, str) or len(docs) == 1:
# embed doc. return shape = [1, hidden_size]
@@ -206,6 +209,8 @@ def _rank_keywords(
keywords: list[str],
max_keywords: int,
) -> list[tuple[str, float]]:
+ import numpy as np
+
def l2_norm(v: np.ndarray) -> np.ndarray:
vec_size = v.shape[1]
result = np.divide(
diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py
index bce375fea..9cf21487b 100644
--- a/pythainlp/summarize/mt5.py
+++ b/pythainlp/summarize/mt5.py
@@ -6,8 +6,6 @@
from __future__ import annotations
-from transformers import MT5ForConditionalGeneration, T5Tokenizer
-
from pythainlp.summarize import CPE_KMUTT_THAI_SENTENCE_SUM
@@ -38,6 +36,8 @@ def __init__(
:param str pretrained_mt5_model_name: Name of pretrained model.
If empty (default), uses google/mt5-{model_size}.
"""
+ from transformers import MT5ForConditionalGeneration, T5Tokenizer
+
model_name = ""
if not pretrained_mt5_model_name:
if model_size not in ["small", "base", "large", "xl", "xxl"]:
diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py
index 927cbb755..908d5a49c 100644
--- a/pythainlp/wangchanberta/core.py
+++ b/pythainlp/wangchanberta/core.py
@@ -7,19 +7,24 @@
import warnings
from typing import Union
-from transformers import (
- CamembertTokenizer,
- pipeline,
-)
-
from pythainlp.tokenize import word_tokenize
_model_name = "wangchanberta-base-att-spm-uncased"
-_tokenizer = CamembertTokenizer.from_pretrained(
- f"airesearch/{_model_name}", revision="main"
-)
-if _model_name == "wangchanberta-base-att-spm-uncased":
- _tokenizer.additional_special_tokens = ["NOTUSED", "NOTUSED", "<_>"]
+_tokenizer = None
+
+
+def _get_tokenizer():
+ """Get the tokenizer, initializing it if necessary."""
+ global _tokenizer
+ if _tokenizer is None:
+ from transformers import CamembertTokenizer
+
+ _tokenizer = CamembertTokenizer.from_pretrained(
+ f"airesearch/{_model_name}", revision="main"
+ )
+ if _model_name == "wangchanberta-base-att-spm-uncased":
+ _tokenizer.additional_special_tokens = ["NOTUSED", "NOTUSED", "<_>"]
+ return _tokenizer
class ThaiNameTagger:
@@ -33,11 +38,13 @@ def __init__(self, dataset_name: str = "thainer", grouped_entities: bool = True)
* *thainer* - ThaiNER dataset
:param bool grouped_entities: grouped entities
"""
+ from transformers import pipeline
+
self.dataset_name = dataset_name
self.grouped_entities = grouped_entities
self.classify_tokens = pipeline(
task="ner",
- tokenizer=_tokenizer,
+ tokenizer=_get_tokenizer(),
model=f"airesearch/{_model_name}",
revision=f"finetuned@{self.dataset_name}-ner",
ignore_labels=[],
@@ -226,4 +233,4 @@ def segment(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
- return _tokenizer.tokenize(text)
+ return _get_tokenizer().tokenize(text)
diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py
index 1c4c58e0d..9abdd5189 100644
--- a/pythainlp/word_vector/core.py
+++ b/pythainlp/word_vector/core.py
@@ -3,13 +3,15 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
-from gensim.models import KeyedVectors
-from gensim.models.keyedvectors import Word2VecKeyedVectors
-from numpy import ndarray, zeros
+from typing import TYPE_CHECKING
from pythainlp.corpus import get_corpus_path
from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize
+if TYPE_CHECKING:
+ from gensim.models.keyedvectors import Word2VecKeyedVectors
+ from numpy import ndarray
+
WV_DIM = 300 # word vector dimension
_MODEL_NAME = "thai2fit_wv"
@@ -48,6 +50,8 @@ def load_wordvector(self, model_name: str) -> None:
:param str model_name: model name
"""
+ from gensim.models import KeyedVectors
+
self.model_name = model_name
self.model = KeyedVectors.load_word2vec_format(
get_corpus_path(self.model_name),
@@ -290,6 +294,8 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray:
0.40506999, 1.58591403, 0.63869202, -0.702155 , 1.62977601,
4.52269109, -0.70760502, 0.50952601, -0.914392 , 0.70673105]])
"""
+ from numpy import zeros
+
vec = zeros((1, self.WV_DIM))
words = self.tokenize(text)
diff --git a/tests/data/text-misspelled-r0.05-seed1.txt b/tests/data/text-misspelled-r0.05-seed1.txt
deleted file mode 100644
index f0718da3e..000000000
--- a/tests/data/text-misspelled-r0.05-seed1.txt
+++ /dev/null
@@ -1 +0,0 @@
-ผมไม่ชอบกินผัก ดังนั้นผมจึงมะกจะเลือปทานอาหารท่่มีเนืัอสัตว์เป็นส่วนใหญ่ อย่างไรก็ตาม ผมก็รู้ว่าการทานผักมีประโยชน์ต่อสุขภาพ ดังนั้นฟมจึงพยายามทานผักบ้างในบางมื้อ ดต่ผมก็ยังคงเลือกทานผักที่ผมชอบเท่านั่น อย่างเช่น ถั่วฝักยาว หรือ ถั่วฝักยาว ฐึ่บผมคิดวรามัรก็เปฺนผักืี่อร่อยและมีประโยชา์ด้วย
\ No newline at end of file