diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index 3621a7896..47dfea180 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -5,9 +5,6 @@ import itertools -from gensim.models.fasttext import FastText as FastText_gensim -from gensim.models.keyedvectors import KeyedVectors - from pythainlp.tokenize import word_tokenize @@ -20,6 +17,9 @@ class FastTextAug: def __init__(self, model_path: str): """:param str model_path: path of model file """ + from gensim.models.fasttext import FastText as FastText_gensim + from gensim.models.keyedvectors import KeyedVectors + if model_path.endswith(".bin"): self.model = FastText_gensim.load_facebook_vectors(model_path) elif model_path.endswith(".vec"): diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index 5624e8171..bb4d611a7 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -3,16 +3,16 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from transformers import ( - CamembertTokenizer, - pipeline, -) - model_name = "airesearch/wangchanberta-base-att-spm-uncased" class Thai2transformersAug: def __init__(self): + from transformers import ( + CamembertTokenizer, + pipeline, + ) + self.model_name = "airesearch/wangchanberta-base-att-spm-uncased" self.target_tokenizer = CamembertTokenizer self.tokenizer = CamembertTokenizer.from_pretrained( diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index 32c537b3b..2d203aa0a 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -3,14 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import spacy - class FastCoref: def __init__( self, model_name, - nlp=spacy.blank("th"), + nlp=None, device: str = "cpu", type: str = "FCoref", ) -> None: @@ -18,6 +16,12 @@ def __init__( from fastcoref import FCoref as _model else: from fastcoref import LingMessCoref as _model + + if nlp is None: + import spacy + + nlp = spacy.blank("th") + self.model_name = model_name self.nlp = nlp self.model = _model(self.model_name, device=device, nlp=self.nlp) diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py index 4fd6d2f6e..046c34273 100644 --- a/pythainlp/coref/han_coref.py +++ b/pythainlp/coref/han_coref.py @@ -3,13 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import spacy - from pythainlp.coref._fastcoref import FastCoref class HanCoref(FastCoref): - def __init__(self, device: str = "cpu", nlp=spacy.blank("th")) -> None: + def __init__(self, device: str = "cpu", nlp=None) -> None: super().__init__( model_name="pythainlp/han-coref-v1.0", device=device, nlp=nlp ) diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 4a01e8d83..b3f432106 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -12,25 +12,22 @@ from __future__ import annotations import os -from typing import List, Union - -import numpy -import torch -import ufal.chu_liu_edmonds -from transformers import ( - AutoConfig, - AutoModelForQuestionAnswering, - AutoModelForTokenClassification, - AutoTokenizer, - TokenClassificationPipeline, -) -from transformers.utils import cached_file +from typing import Union class Parse: def __init__( self, model: str = "KoichiYasuoka/deberta-base-thai-ud-head" ) -> None: + from transformers import ( + AutoConfig, + AutoModelForQuestionAnswering, + AutoModelForTokenClassification, + AutoTokenizer, + TokenClassificationPipeline, + ) + from transformers.utils import cached_file + if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-head" self.tokenizer = AutoTokenizer.from_pretrained(model) @@ -57,7 +54,11 @@ def __init__( model=t, tokenizer=self.tokenizer ) - def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]: + def __call__(self, text: str, tag: str = "str") -> Union[list[list[str]], str]: + import numpy + import torch + import ufal.chu_liu_edmonds + w = [ (t["start"], t["end"], t["entity_group"]) for t in self.deprel(text) diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index 5b2bfec48..2a0dbe998 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -14,19 +14,21 @@ from collections import Counter from collections.abc import Iterable -from typing import Optional, Union - -import numpy as np -from transformers import pipeline +from typing import TYPE_CHECKING, Optional, Union from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import word_tokenize +if TYPE_CHECKING: + import numpy as np + class KeyBERT: def __init__( self, model_name: str = "airesearch/wangchanberta-base-att-spm-uncased" ): + from transformers import pipeline + self.ft_pipeline = pipeline( "feature-extraction", tokenizer=model_name, @@ -136,8 +138,9 @@ def extract_keywords( return [kw for kw, _ in keywords] def embed(self, docs: Union[str, list[str]]) -> np.ndarray: - """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer. - """ + """Create an embedding of each input in `docs` by averaging vectors from the last hidden layer.""" + import numpy as np + embs = self.ft_pipeline(docs) if isinstance(docs, str) or len(docs) == 1: # embed doc. return shape = [1, hidden_size] @@ -206,6 +209,8 @@ def _rank_keywords( keywords: list[str], max_keywords: int, ) -> list[tuple[str, float]]: + import numpy as np + def l2_norm(v: np.ndarray) -> np.ndarray: vec_size = v.shape[1] result = np.divide( diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index bce375fea..9cf21487b 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -6,8 +6,6 @@ from __future__ import annotations -from transformers import MT5ForConditionalGeneration, T5Tokenizer - from pythainlp.summarize import CPE_KMUTT_THAI_SENTENCE_SUM @@ -38,6 +36,8 @@ def __init__( :param str pretrained_mt5_model_name: Name of pretrained model. If empty (default), uses google/mt5-{model_size}. """ + from transformers import MT5ForConditionalGeneration, T5Tokenizer + model_name = "" if not pretrained_mt5_model_name: if model_size not in ["small", "base", "large", "xl", "xxl"]: diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index 927cbb755..908d5a49c 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -7,19 +7,24 @@ import warnings from typing import Union -from transformers import ( - CamembertTokenizer, - pipeline, -) - from pythainlp.tokenize import word_tokenize _model_name = "wangchanberta-base-att-spm-uncased" -_tokenizer = CamembertTokenizer.from_pretrained( - f"airesearch/{_model_name}", revision="main" -) -if _model_name == "wangchanberta-base-att-spm-uncased": - _tokenizer.additional_special_tokens = ["NOTUSED", "NOTUSED", "<_>"] +_tokenizer = None + + +def _get_tokenizer(): + """Get the tokenizer, initializing it if necessary.""" + global _tokenizer + if _tokenizer is None: + from transformers import CamembertTokenizer + + _tokenizer = CamembertTokenizer.from_pretrained( + f"airesearch/{_model_name}", revision="main" + ) + if _model_name == "wangchanberta-base-att-spm-uncased": + _tokenizer.additional_special_tokens = ["NOTUSED", "NOTUSED", "<_>"] + return _tokenizer class ThaiNameTagger: @@ -33,11 +38,13 @@ def __init__(self, dataset_name: str = "thainer", grouped_entities: bool = True) * *thainer* - ThaiNER dataset :param bool grouped_entities: grouped entities """ + from transformers import pipeline + self.dataset_name = dataset_name self.grouped_entities = grouped_entities self.classify_tokens = pipeline( task="ner", - tokenizer=_tokenizer, + tokenizer=_get_tokenizer(), model=f"airesearch/{_model_name}", revision=f"finetuned@{self.dataset_name}-ner", ignore_labels=[], @@ -226,4 +233,4 @@ def segment(text: str) -> list[str]: if not text or not isinstance(text, str): return [] - return _tokenizer.tokenize(text) + return _get_tokenizer().tokenize(text) diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 1c4c58e0d..9abdd5189 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -3,13 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from gensim.models import KeyedVectors -from gensim.models.keyedvectors import Word2VecKeyedVectors -from numpy import ndarray, zeros +from typing import TYPE_CHECKING from pythainlp.corpus import get_corpus_path from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize +if TYPE_CHECKING: + from gensim.models.keyedvectors import Word2VecKeyedVectors + from numpy import ndarray + WV_DIM = 300 # word vector dimension _MODEL_NAME = "thai2fit_wv" @@ -48,6 +50,8 @@ def load_wordvector(self, model_name: str) -> None: :param str model_name: model name """ + from gensim.models import KeyedVectors + self.model_name = model_name self.model = KeyedVectors.load_word2vec_format( get_corpus_path(self.model_name), @@ -290,6 +294,8 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: 0.40506999, 1.58591403, 0.63869202, -0.702155 , 1.62977601, 4.52269109, -0.70760502, 0.50952601, -0.914392 , 0.70673105]]) """ + from numpy import zeros + vec = zeros((1, self.WV_DIM)) words = self.tokenize(text) diff --git a/tests/data/text-misspelled-r0.05-seed1.txt b/tests/data/text-misspelled-r0.05-seed1.txt deleted file mode 100644 index f0718da3e..000000000 --- a/tests/data/text-misspelled-r0.05-seed1.txt +++ /dev/null @@ -1 +0,0 @@ -ผมไม่ชอบกินผัก ดังนั้นผมจึงมะกจะเลือปทานอาหารท่่มีเนืัอสัตว์เป็นส่วนใหญ่ อย่างไรก็ตาม ผมก็รู้ว่าการทานผักมีประโยชน์ต่อสุขภาพ ดังนั้นฟมจึงพยายามทานผักบ้างในบางมื้อ ดต่ผมก็ยังคงเลือกทานผักที่ผมชอบเท่านั่น อย่างเช่น ถั่วฝักยาว หรือ ถั่วฝักยาว ฐึ่บผมคิดวรามัรก็เปฺนผักืี่อร่อยและมีประโยชา์ด้วย \ No newline at end of file