diff --git a/pyproject.toml b/pyproject.toml index 1ad41fd39..9724eb0ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,7 +226,7 @@ full = [ "sefr_cut>=1.1", "symspellpy==6.9.0", "thai-nner==0.3", - "tltk>=1.6.8,<2", + "tltk>=1.10,<2", "torch>=1.13.1,<3", "transformers==4.57.6", "ufal.chu-liu-edmonds==1.0.3", diff --git a/tests/README.md b/tests/README.md index 137a2caaf..49e47b131 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,7 @@ supported Python versions and operating systems: The CI/CD test workflow is at . - + ## Core tests (test_*.py) - Run `unittest tests.core` @@ -65,7 +65,7 @@ The CI/CD test workflow is at A comprehensive test suite within core tests that tests edge cases important for real-world usage: -- Empty strings and various whitespace handling (spaces, tabs, unicode spaces) +- Empty strings and various whitespace handling (spaces, tabs, Unicode spaces) - Special characters from encoding issues, BOM, terminal copy/paste - Truncated/malformed Unicode and surrogate pairs - Emoji and modern Unicode sequences (ZWJ, modifiers, flags) diff --git a/tests/compact/testc_tools.py b/tests/compact/testc_tools.py index 6b04c0a1f..0683c9bb2 100644 --- a/tests/compact/testc_tools.py +++ b/tests/compact/testc_tools.py @@ -6,7 +6,11 @@ import numpy as np -from pythainlp.tools.misspell import misspell +from pythainlp.tools.misspell import ( + find_misspell_candidates, + misspell, + search_location_of_character, +) def _count_difference(st1: str, st2: str) -> int: @@ -85,3 +89,58 @@ def test_misspell_with_ratio_100_percent(self): 2, f"expect len(text)-2 misspells with ratio=1.5. (Δ={diff})", ) + + def test_search_location_of_character(self): + """Test search_location_of_character function.""" + # Test Thai characters + loc = search_location_of_character("ก") + self.assertIsNotNone(loc) + self.assertEqual(len(loc), 4) # (language_ix, is_shift, row, pos) + + # Test English characters + loc = search_location_of_character("a") + self.assertIsNotNone(loc) + self.assertEqual(len(loc), 4) + + # Test shifted characters + loc = search_location_of_character("A") + self.assertIsNotNone(loc) + + # Test numbers + loc = search_location_of_character("1") + self.assertIsNotNone(loc) + + # Test character not in keyboard + loc = search_location_of_character("€") + self.assertIsNone(loc) + + # Test empty string + # Note: Empty string returns a location because Python's "in" operator + # matches empty string at the beginning of any string + loc = search_location_of_character("") + self.assertIsNotNone(loc) + + def test_find_misspell_candidates(self): + """Test find_misspell_candidates function.""" + # Test Thai character + candidates = find_misspell_candidates("ก") + self.assertIsNotNone(candidates) + self.assertIsInstance(candidates, list) + self.assertGreater(len(candidates), 0) + + # Test English character + candidates = find_misspell_candidates("a") + self.assertIsNotNone(candidates) + self.assertIsInstance(candidates, list) + self.assertGreater(len(candidates), 0) + + # Test character not in keyboard + candidates = find_misspell_candidates("€") + self.assertIsNone(candidates) + + # Test that candidates are different from input + candidates = find_misspell_candidates("ด") + if candidates: + for candidate in candidates: + # Candidates should be strings + self.assertIsInstance(candidate, str) diff --git a/tests/core/test_ancient.py b/tests/core/test_ancient.py index 66bac54cf..78c4afe98 100644 --- a/tests/core/test_ancient.py +++ b/tests/core/test_ancient.py @@ -23,6 +23,11 @@ def test_aksonhan_to_current(self): self.assertEqual(aksonhan_to_current("บงงคบบ"), "บังคับ") self.assertEqual(aksonhan_to_current("สรรเพชญ"), "สรรเพชญ") + # Edge cases + self.assertEqual(aksonhan_to_current(""), "") # empty string + self.assertEqual(aksonhan_to_current("ก"), "ก") # single char + self.assertEqual(aksonhan_to_current("กา"), "กา") # two chars + def test_convert_currency(self): self.assertEqual( convert_currency(80, "บาท")["ตำลึง"], @@ -44,3 +49,24 @@ def test_convert_currency(self): convert_currency(1,"ชั่ง")["ชั่ง"], 1.0 ) + + # Test all supported units + result = convert_currency(1, "บาท") + self.assertIn("เบี้ย", result) + self.assertIn("อัฐ", result) + self.assertIn("ไพ", result) + self.assertIn("เฟื้อง", result) + self.assertIn("สลึง", result) + self.assertIn("ตำลึง", result) + + # Test with zero value + result = convert_currency(0, "บาท") + self.assertEqual(result["บาท"], 0.0) + + # Test with fractional value + result = convert_currency(0.5, "บาท") + self.assertEqual(result["บาท"], 0.5) + + # Test invalid unit + with self.assertRaises(NotImplementedError): + convert_currency(1, "invalid_unit") diff --git a/tests/core/test_tools.py b/tests/core/test_tools.py index 8a535446a..72c473395 100644 --- a/tests/core/test_tools.py +++ b/tests/core/test_tools.py @@ -5,12 +5,14 @@ import os import tempfile import unittest +import warnings from pythainlp.tools import ( get_full_data_path, get_pythainlp_data_path, get_pythainlp_path, ) +from pythainlp.tools.core import safe_print, warn_deprecation class ToolsTestCase(unittest.TestCase): @@ -57,3 +59,69 @@ def test_custom_data_dir(self): os.environ["PYTHAINLP_DATA_DIR"] = original_value elif "PYTHAINLP_DATA_DIR" in os.environ: del os.environ["PYTHAINLP_DATA_DIR"] + + def test_warn_deprecation(self): + """Test deprecation warning function.""" + # Test basic deprecation warning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + warn_deprecation("old_func") + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[0].category, DeprecationWarning)) + self.assertIn("old_func", str(w[0].message)) + self.assertIn("deprecated", str(w[0].message)) + + # Test with replacement function + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + warn_deprecation("old_func", replacing_func="new_func") + self.assertEqual(len(w), 1) + self.assertIn("old_func", str(w[0].message)) + self.assertIn("new_func", str(w[0].message)) + + # Test with version information + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + warn_deprecation( + "old_func", + deprecated_version="1.0", + removal_version="2.0" + ) + self.assertEqual(len(w), 1) + self.assertIn("1.0", str(w[0].message)) + self.assertIn("2.0", str(w[0].message)) + + # Test with all parameters + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + warn_deprecation( + "old_func", + replacing_func="new_func", + deprecated_version="1.0", + removal_version="2.0" + ) + self.assertEqual(len(w), 1) + message = str(w[0].message) + self.assertIn("old_func", message) + self.assertIn("new_func", message) + self.assertIn("1.0", message) + self.assertIn("2.0", message) + + def test_safe_print(self): + """Test safe_print function.""" + # Test normal printing + safe_print("Hello, World!") + safe_print("สวัสดีครับ") + + # Test with Unicode characters + safe_print("Hello 👋 World") + safe_print("ภาษาไทย") + + # Test with empty string + safe_print("") + + # Test with special characters + safe_print("\n\t") + + # Note: Testing actual UnicodeEncodeError is environment-dependent + # and would require mocking sys.stdout.encoding diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 209d0521e..bd715bda9 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -97,6 +97,14 @@ def test_collate(self): collate(["ไก่", "เป็ด", "หมู", "วัว"], reverse=True), ["หมู", "วัว", "เป็ด", "ไก่"], ) + # Edge cases: mixed Thai and numbers + self.assertEqual(collate(["ก", "1", "ข"]), ["1", "ก", "ข"]) + # Edge cases: with spaces (spaces sort before letters) + result = collate([" ก", "ก", " ก"]) + self.assertEqual(len(result), 3) + self.assertIn(" ก", result) + self.assertIn("ก", result) + self.assertIn(" ก", result) # ### pythainlp.util.numtoword @@ -720,6 +728,11 @@ def test_emoji_to_thai(self): ":ธง_ไทย: นี่คือธงประเทศไทย", ) + # Edge cases + self.assertEqual(emoji_to_thai(""), "") # empty string + self.assertEqual(emoji_to_thai("no emoji"), "no emoji") # no emoji + self.assertEqual(emoji_to_thai("ไม่มีอีโมจิ"), "ไม่มีอีโมจิ") # Thai no emoji + def test_sound_syllable(self): test = [ ("มา", "live"), @@ -863,6 +876,10 @@ def test_syllable_open_close_detector(self): def test_to_idna(self): self.assertEqual(to_idna("คนละครึ่ง.com"), "xn--42caj4e6bk1f5b1j.com") + # Additional test cases for IDNA encoding + self.assertEqual(to_idna("ไทย.com"), "xn--o3cw4h.com") + self.assertEqual(to_idna("example.com"), "example.com") # ASCII unchanged + self.assertEqual(to_idna("ภาษาไทย.th"), "xn--o3crh0a8bb0k.th") def test_thai_strptime(self): self.assertIsNotNone( @@ -923,6 +940,11 @@ def test_tis620_to_utf8(self): self.assertEqual( tis620_to_utf8("¡ÃзÃǧÍصÊÒË¡ÃÃÁ"), "กระทรวงอุตสาหกรรม" ) + # Additional test cases + self.assertEqual(tis620_to_utf8("»ÃÐà·Èä·Â"), "ประเทศไทย") + self.assertEqual(tis620_to_utf8("ÀÒÉÒä·Â"), "ภาษาไทย") + # Empty string + self.assertEqual(tis620_to_utf8(""), "") def test_remove_repeat_consonants(self): # update of pythainlp.copus.thai_words() able to break this @@ -988,11 +1010,20 @@ def test_longest_common_subsequence(self): self.assertEqual(longest_common_subsequence("ABCBDAB", "BDCAB"), "BDAB") self.assertEqual(longest_common_subsequence("AGGTAB", "GXTXAYB"), "GTAB") self.assertEqual(longest_common_subsequence("ABCDGH", "AEDFHR"), "ADH") + + # Edge cases + self.assertEqual(longest_common_subsequence("", ""), "") # empty strings + self.assertEqual(longest_common_subsequence("ABC", ""), "") # one empty + self.assertEqual(longest_common_subsequence("", "ABC"), "") # other empty + self.assertEqual(longest_common_subsequence("A", "A"), "A") # single char match + self.assertEqual(longest_common_subsequence("A", "B"), "") # single char no match + self.assertEqual(longest_common_subsequence("ABC", "ABC"), "ABC") # identical + self.assertEqual(longest_common_subsequence("ABC", "XYZ"), "") # no common chars self.assertEqual(longest_common_subsequence("ABC", "AC"), "AC") - self.assertEqual(longest_common_subsequence("ABC", "DEF"), "") - self.assertEqual(longest_common_subsequence("", "ABC"), "") - self.assertEqual(longest_common_subsequence("ABC", ""), "") - self.assertEqual(longest_common_subsequence("", ""), "") + + # Thai text + self.assertEqual(longest_common_subsequence("ไทย", "ไทย"), "ไทย") + self.assertEqual(longest_common_subsequence("ภาษาไทย", "ไทย"), "ไทย") def test_analyze_thai_text(self): self.assertEqual(