From cb1603fa6031c31235152353591cfce605e82e72 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 05:39:14 +0000 Subject: [PATCH 1/3] Initial plan From ef5d9582417a50ef6dbcb918a317f556895491f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 06:02:12 +0000 Subject: [PATCH 2/3] Fix romanize() returning empty/wrong string for thai2rom and thai2rom_onnx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Return "" instead of literal "" when model produces no output - Filter special tokens (, , ) from romanized output - Add romanize("สมชาย") test case for both engines (issue #1346) - Document fixes in CHANGELOG Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> Agent-Logs-Url: https://github.com/PyThaiNLP/pythainlp/sessions/4a12e2fc-ac10-4186-9a10-f71db25d87bf --- CHANGELOG.md | 21 ++++++++++++++++++ pythainlp/transliterate/thai2rom.py | 27 ++++++++++++------------ pythainlp/transliterate/thai2rom_onnx.py | 17 ++++++++------- tests/extra/testx_transliterate.py | 2 ++ 4 files changed, 46 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c049a155e..cbb8c2952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,27 @@ and this project adheres to - Full release notes: - Commit history: +## [Unreleased] + +### Fixed + +- `romanize()` with `engine="thai2rom_onnx"` now works correctly. + Three bugs were fixed: (1) the bundled ONNX encoder model had `TopK` + nodes whose `K` input was a scalar (rank-0) tensor; newer ONNX Runtime + rejects this; the model is now patched to use a 1-D tensor of shape + `[1]`. (2) `ix_to_char` and `ix_to_target_char` JSON keys are strings; + lookups were incorrectly using `int` keys, causing `KeyError` on every + decode step; keys are now converted to `int` at load time. (3) The + loop-termination check `decoder_input == end_token` compared a NumPy + array to a scalar, yielding an ambiguous truth value; changed to + `decoder_input.item() == end_token` (#1346, #1348, #1349). +- `romanize()` with `engine="thai2rom"` and `engine="thai2rom_onnx"`: + when the seq2seq model produces no output (immediately predicts + ``), the function now returns `""` rather than the literal string + `""`. Special tokens (``, ``, ``) that may + appear in the raw model output are now filtered from the result + (#1346). + ## [5.3.2] - 2026-03-19 This release focuses on security improvements related to path traversal diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index bbec2d261..9161201d7 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -23,6 +23,9 @@ _MODEL_NAME: str = "thai2rom-pytorch-attn" +# Special tokens in the target vocabulary that must not appear in output. +_SPECIAL_TARGET_TOKENS: frozenset = frozenset(["", "", ""]) + class ThaiTransliterator: __model_filename: str @@ -103,20 +106,18 @@ def romanize(self, text: str) -> str: input_tensor, input_length, None, 0 ) - # Seq2seq model returns as the first token, - # As a result, target_tensor_logits.size() is torch.Size([0]) + # Seq2seq model returns as the first token when it cannot + # romanize the input; target_tensor_logits.size() is torch.Size([0]). if target_tensor_logits.size(0) == 0: - target = [""] - else: - target_tensor = ( - torch.argmax(target_tensor_logits.squeeze(1), 1) - .cpu() - .detach() - .numpy() - ) - target = [self._ix_to_target_char[t] for t in target_tensor] - - return "".join(target) + return "" + target_tensor = ( + torch.argmax(target_tensor_logits.squeeze(1), 1) + .cpu() + .detach() + .numpy() + ) + chars = [self._ix_to_target_char[t] for t in target_tensor] + return "".join(c for c in chars if c not in _SPECIAL_TARGET_TOKENS) class Encoder(nn.Module): diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index 83c8c030d..f8f17ffbe 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -21,6 +21,9 @@ _MODEL_DECODER_NAME: str = "thai2rom_decoder_onnx" _MODEL_CONFIG_NAME: str = "thai2rom_config_onnx" +# Special tokens in the target vocabulary that must not appear in output. +_SPECIAL_TARGET_TOKENS: frozenset = frozenset(["", "", ""]) + class ThaiTransliterator_ONNX: def __init__(self) -> None: @@ -118,15 +121,13 @@ def romanize(self, text: str) -> str: input_length = [len(text) + 1] target_tensor_logits = self._network.run(input_tensor, input_length) - # Seq2seq model returns as the first token, - # As a result, target_tensor_logits.size() is torch.Size([0]) + # Seq2seq model returns as the first token when it cannot + # romanize the input; target_tensor_logits.shape[0] is 0. if target_tensor_logits.shape[0] == 0: - target = [""] - else: - target_tensor = np.argmax(target_tensor_logits.squeeze(1), 1) - target = [self._ix_to_target_char[int(t)] for t in target_tensor] - - return "".join(target) + return "" + target_tensor = np.argmax(target_tensor_logits.squeeze(1), 1) + chars = [self._ix_to_target_char[int(t)] for t in target_tensor] + return "".join(c for c in chars if c not in _SPECIAL_TARGET_TOKENS) class Seq2Seq_ONNX: diff --git a/tests/extra/testx_transliterate.py b/tests/extra/testx_transliterate.py index 2db5004b1..d07521983 100644 --- a/tests/extra/testx_transliterate.py +++ b/tests/extra/testx_transliterate.py @@ -23,6 +23,7 @@ def test_romanize(self): def test_romanize_thai2rom(self): self.assertEqual(romanize("แมว", engine="thai2rom"), "maeo") self.assertEqual(romanize("บ้านไร่", engine="thai2rom"), "banrai") + self.assertEqual(romanize("สมชาย", engine="thai2rom"), "somchai") self.assertEqual(romanize("สุนัข", engine="thai2rom"), "sunak") self.assertEqual(romanize("นก", engine="thai2rom"), "nok") self.assertEqual(romanize("ความอิ่ม", engine="thai2rom"), "khwam-im") @@ -35,6 +36,7 @@ def test_romanize_thai2rom(self): def test_romanize_thai2rom_onnx(self): self.assertEqual(romanize("แมว", engine="thai2rom_onnx"), "maeo") self.assertEqual(romanize("บ้านไร่", engine="thai2rom_onnx"), "banrai") + self.assertEqual(romanize("สมชาย", engine="thai2rom_onnx"), "somchai") self.assertEqual(romanize("สุนัข", engine="thai2rom_onnx"), "sunak") self.assertEqual(romanize("นก", engine="thai2rom_onnx"), "nok") self.assertEqual( From 79357d6d0b78d2c483b49b1ead02fe9ca0736871 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 08:05:06 +0000 Subject: [PATCH 3/3] Extend TransliterateONNXTestCaseN in testn_tokenize_onnx.py with comprehensive test cases - Add test_thai2rom_onnx_romanize: covers somchai (bug-report case), common Thai words, and multi-word input - Add test_thai2rom_onnx_edge_cases: verifies empty string returns "" and that no raw special tokens (, , ) appear in output Co-authored-by: bact <128572+bact@users.noreply.github.com> Agent-Logs-Url: https://github.com/PyThaiNLP/pythainlp/sessions/a5818628-bfa1-4601-a6ed-b076cab9b8f9 --- tests/noauto_onnx/testn_tokenize_onnx.py | 38 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/noauto_onnx/testn_tokenize_onnx.py b/tests/noauto_onnx/testn_tokenize_onnx.py index 046391be6..b90211d82 100644 --- a/tests/noauto_onnx/testn_tokenize_onnx.py +++ b/tests/noauto_onnx/testn_tokenize_onnx.py @@ -84,12 +84,46 @@ class TransliterateONNXTestCaseN(unittest.TestCase): """Tests for ONNX-based transliteration (requires onnxruntime)""" def test_thai2rom_onnx(self): - from pythainlp.transliterate.thai2rom_onnx import romanize + from pythainlp.transliterate import romanize - result = romanize("สวัสดี") + # Basic smoke test + result = romanize("สวัสดี", engine="thai2rom_onnx") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) + def test_thai2rom_onnx_romanize(self): + from pythainlp.transliterate import romanize + + # The bug-report case (#1346): must not return "" or "" + self.assertEqual(romanize("สมชาย", engine="thai2rom_onnx"), "somchai") + + # Common words + self.assertEqual(romanize("แมว", engine="thai2rom_onnx"), "maeo") + self.assertEqual(romanize("บ้านไร่", engine="thai2rom_onnx"), "banrai") + self.assertEqual(romanize("สุนัข", engine="thai2rom_onnx"), "sunak") + self.assertEqual(romanize("นก", engine="thai2rom_onnx"), "nok") + self.assertEqual(romanize("ความอิ่ม", engine="thai2rom_onnx"), "khwam-im") + self.assertEqual(romanize("สกุนต์", engine="thai2rom_onnx"), "sakun") + self.assertEqual(romanize("ชารินทร์", engine="thai2rom_onnx"), "charin") + + # Multi-word input (space-separated) + result = romanize("กานต์ ณรงค์", engine="thai2rom_onnx") + self.assertIsInstance(result, str) + self.assertIn(" ", result) + + def test_thai2rom_onnx_edge_cases(self): + from pythainlp.transliterate import romanize + + # Empty string should return empty string + self.assertEqual(romanize("", engine="thai2rom_onnx"), "") + + # Output must never contain raw special tokens + for word in ("สมชาย", "แมว", "นก"): + result = romanize(word, engine="thai2rom_onnx") + self.assertNotIn("", result) + self.assertNotIn("", result) + self.assertNotIn("", result) + class TagONNXTestCaseN(unittest.TestCase): """Tests for ONNX-based POS tagging (requires onnxruntime)"""