Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tests/extra/testx_translate.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
class TranslateTestCaseX(unittest.TestCase):
def test_translate(self):
# remove("scb_1m_th-en_spm")
self.assertIsNone(download_model_all())
result = download_model_all()
self.assertIsNone(result)
self.th_en_translator = ThEnTranslator()
self.assertIsNotNone(
self.th_en_translator.translate(
Expand Down
4 changes: 2 additions & 2 deletions tests/extra/testx_translate_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_prepare_text_with_exclusions_none(self):
def test_prepare_text_with_exclusions_empty_list(self):
"""Test with empty exclusion list"""
text = "Hello world"
exclude_words = []
exclude_words: list[str] = []
prepared, mapping = _prepare_text_with_exclusions(
text, exclude_words
)
Expand Down Expand Up @@ -93,7 +93,7 @@ def test_restore_excluded_words_multiple(self):
def test_restore_excluded_words_empty_map(self):
"""Test restoring with empty mapping"""
translated = "Hello world"
mapping = {}
mapping: dict[str, str] = {}
restored = _restore_excluded_words(translated, mapping)

# Text should be unchanged
Expand Down
13 changes: 8 additions & 5 deletions tests/extra/testx_ulmfit.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import pickle
import unittest
from typing import cast

import fastai
import pandas as pd
Expand All @@ -19,7 +20,7 @@
untar_data,
)

from pythainlp.tokenize import THAI2FIT_TOKENIZER
from pythainlp.tokenize import thai2fit_tokenizer
from pythainlp.ulmfit import (
THWIKI_LSTM,
ThaiTokenizer,
Expand Down Expand Up @@ -53,12 +54,14 @@ class UlmfitTestCaseX(unittest.TestCase):
def test_ThaiTokenizer(self):
self.thai = ThaiTokenizer()
self.assertIsNotNone(self.thai.tokenizer("ทดสอบการตัดคำ"))
self.assertIsNone(self.thai.add_special_cases(["แมว"]))
result = self.thai.add_special_cases(["แมว"])
self.assertIsNone(result)

def test_BaseTokenizer(self):
self.base = base_tokenizer(lang="th")
self.assertIsNotNone(self.base.tokenizer("ทดสอบ การ ตัด คำ"))
self.assertIsNone(self.base.add_special_cases(["แมว"]))
result = self.base.add_special_cases(["แมว"])
self.assertIsNone(result)

def test_load_pretrained(self):
self.assertIsNotNone(THWIKI_LSTM)
Expand Down Expand Up @@ -179,7 +182,7 @@ def test_process_thai_dense(self):
text,
pre_rules=pre_rules_th,
post_rules=post_rules_th,
tok_func=THAI2FIT_TOKENIZER.word_tokenize,
tok_func=thai2fit_tokenizer().word_tokenize,
)

# after pre_rules_th
Expand Down Expand Up @@ -225,7 +228,7 @@ def test_document_vector(self):
thwiki = THWIKI_LSTM
# Security note: pickle.load() executes arbitrary code if file is malicious.
# These corpus files come from a trusted source with MD5 verification.
with open(thwiki["itos_fname"], "rb") as f:
with open(cast(str, thwiki["itos_fname"]), "rb") as f:
thwiki_itos = pickle.load(f)
thwiki_vocab = fastai.text.transform.Vocab(thwiki_itos)
tt = Tokenizer(
Expand Down
Loading