Skip to content

Commit 964f209

Browse files
Copilotbact
andcommitted
Address code review feedback
- Fixed wordnet.py to use the tokenize parameter as intended - Fixed thai2fit and ltw2v to properly handle None corpus paths with clear error messages - Fixed tokenization_small100.py type ignore comment formatting - All mypy checks still passing (0 errors) Co-authored-by: bact <128572+bact@users.noreply.github.com>
1 parent 808d6bd commit 964f209

4 files changed

Lines changed: 15 additions & 10 deletions

File tree

pythainlp/augment/word2vec/ltw2v.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,12 @@ def tokenizer(self, text: str) -> list[str]:
2727

2828
def load_w2v(self): # insert substitute
2929
"""Load LTW2V's word2vec model"""
30-
ltw2v_wv = self.ltw2v_wv or ""
31-
self.aug = Word2VecAug(ltw2v_wv, self.tokenizer, type="binary")
30+
if self.ltw2v_wv is None:
31+
raise ValueError(
32+
"LTW2V word2vec model not found. "
33+
"Please download it first using pythainlp.corpus.download('ltw2v_wv')"
34+
)
35+
self.aug = Word2VecAug(self.ltw2v_wv, self.tokenizer, type="binary")
3236

3337
def augment(
3438
self, sentence: str, n_sent: int = 1, p: float = 0.7

pythainlp/augment/word2vec/thai2fit.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,12 @@ def tokenizer(self, text: str) -> list[str]:
2828

2929
def load_w2v(self):
3030
"""Load Thai2Fit's word2vec model"""
31-
thai2fit_wv = self.thai2fit_wv or ""
32-
self.aug = Word2VecAug(thai2fit_wv, self.tokenizer, type="binary")
31+
if self.thai2fit_wv is None:
32+
raise ValueError(
33+
"Thai2Fit word2vec model not found. "
34+
"Please download it first using pythainlp.corpus.download('thai2fit_wv')"
35+
)
36+
self.aug = Word2VecAug(self.thai2fit_wv, self.tokenizer, type="binary")
3337

3438
def augment(
3539
self, sentence: str, n_sent: int = 1, p: float = 0.7

pythainlp/augment/wordnet.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ def augment(
188188
('เรา', 'ชอบ', 'ไปยัง', 'รร.')]
189189
"""
190190
new_sentences = []
191-
self.list_words = word_tokenize(sentence)
191+
self.list_words = tokenize(sentence)
192192
self.list_synonym = []
193193
self.p_all = 1
194194
if postag:

pythainlp/translate/tokenization_small100.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,11 +202,8 @@ def __init__(
202202

203203
@property
204204
def vocab_size(self) -> int:
205-
return (
206-
len(self.encoder)
207-
+ len(self.lang_token_to_id)
208-
+ self.num_madeup_words
209-
) # type: ignore[no-any-return]
205+
# Type ignore for external library dict operations
206+
return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words # type: ignore[no-any-return]
210207

211208
@property
212209
def tgt_lang(self) -> str:

0 commit comments

Comments
 (0)