Fix romanize() returning empty string / literal <PAD> for thai2rom engines

CHANGELOG.md

@@ -17,6 +17,27 @@ and this project adheres to
 - Full release notes: <https://github.com/PyThaiNLP/pythainlp/releases>
 - Commit history: <https://github.com/PyThaiNLP/pythainlp/compare/v5.3.1...v5.3.2>
 
+## [Unreleased]
+
+### Fixed
+
+- `romanize()` with `engine="thai2rom_onnx"` now works correctly.
+  Three bugs were fixed: (1) the bundled ONNX encoder model had `TopK`
+  nodes whose `K` input was a scalar (rank-0) tensor; newer ONNX Runtime
+  rejects this; the model is now patched to use a 1-D tensor of shape
+  `[1]`. (2) `ix_to_char` and `ix_to_target_char` JSON keys are strings;
+  lookups were incorrectly using `int` keys, causing `KeyError` on every
+  decode step; keys are now converted to `int` at load time. (3) The
+  loop-termination check `decoder_input == end_token` compared a NumPy
+  array to a scalar, yielding an ambiguous truth value; changed to
+  `decoder_input.item() == end_token` (#1346, #1348, #1349).
+- `romanize()` with `engine="thai2rom"` and `engine="thai2rom_onnx"`:
+  when the seq2seq model produces no output (immediately predicts
+  `<end>`), the function now returns `""` rather than the literal string
+  `"<PAD>"`. Special tokens (`<PAD>`, `<start>`, `<end>`) that may
+  appear in the raw model output are now filtered from the result
+  (#1346).
+
 ## [5.3.2] - 2026-03-19
 
 This release focuses on security improvements related to path traversal

pythainlp/transliterate/thai2rom.py

@@ -23,6 +23,9 @@
 
 _MODEL_NAME: str = "thai2rom-pytorch-attn"
 
+# Special tokens in the target vocabulary that must not appear in output.
+_SPECIAL_TARGET_TOKENS: frozenset = frozenset(["<PAD>", "<start>", "<end>"])
+
 
 class ThaiTransliterator:
     __model_filename: str
@@ -103,20 +106,18 @@
             input_tensor, input_length, None, 0
         )
 
-        # Seq2seq model returns <END> as the first token,
-        # As a result, target_tensor_logits.size() is torch.Size([0])
+        # Seq2seq model returns <END> as the first token when it cannot
+        # romanize the input; target_tensor_logits.size() is torch.Size([0]).
         if target_tensor_logits.size(0) == 0:
-            target = ["<PAD>"]
-        else:
-            target_tensor = (
-                torch.argmax(target_tensor_logits.squeeze(1), 1)
-                .cpu()
-                .detach()
-                .numpy()
-            )
-            target = [self._ix_to_target_char[t] for t in target_tensor]
-
-        return "".join(target)
+            return ""
+        target_tensor = (
+            torch.argmax(target_tensor_logits.squeeze(1), 1)
+            .cpu()
+            .detach()
+            .numpy()
+        )
+        chars = [self._ix_to_target_char[t] for t in target_tensor]
+        return "".join(c for c in chars if c not in _SPECIAL_TARGET_TOKENS)
 
 
 class Encoder(nn.Module):

pythainlp/transliterate/thai2rom_onnx.py

@@ -21,6 +21,9 @@
 _MODEL_DECODER_NAME: str = "thai2rom_decoder_onnx"
 _MODEL_CONFIG_NAME: str = "thai2rom_config_onnx"
 
+# Special tokens in the target vocabulary that must not appear in output.
+_SPECIAL_TARGET_TOKENS: frozenset = frozenset(["<PAD>", "<start>", "<end>"])
+
 
 class ThaiTransliterator_ONNX:
     def __init__(self) -> None:
@@ -118,15 +121,13 @@
         input_length = [len(text) + 1]
         target_tensor_logits = self._network.run(input_tensor, input_length)
 
-        # Seq2seq model returns <END> as the first token,
-        # As a result, target_tensor_logits.size() is torch.Size([0])
+        # Seq2seq model returns <END> as the first token when it cannot
+        # romanize the input; target_tensor_logits.shape[0] is 0.
         if target_tensor_logits.shape[0] == 0:
-            target = ["<PAD>"]
-        else:
-            target_tensor = np.argmax(target_tensor_logits.squeeze(1), 1)
-            target = [self._ix_to_target_char[int(t)] for t in target_tensor]
-
-        return "".join(target)
+            return ""
+        target_tensor = np.argmax(target_tensor_logits.squeeze(1), 1)
+        chars = [self._ix_to_target_char[int(t)] for t in target_tensor]
+        return "".join(c for c in chars if c not in _SPECIAL_TARGET_TOKENS)
 
 
 class Seq2Seq_ONNX:

tests/extra/testx_transliterate.py

@@ -23,6 +23,7 @@ def test_romanize(self):
     def test_romanize_thai2rom(self):
         self.assertEqual(romanize("แมว", engine="thai2rom"), "maeo")
         self.assertEqual(romanize("บ้านไร่", engine="thai2rom"), "banrai")
+        self.assertEqual(romanize("สมชาย", engine="thai2rom"), "somchai")
         self.assertEqual(romanize("สุนัข", engine="thai2rom"), "sunak")
         self.assertEqual(romanize("นก", engine="thai2rom"), "nok")
         self.assertEqual(romanize("ความอิ่ม", engine="thai2rom"), "khwam-im")
@@ -35,6 +36,7 @@ def test_romanize_thai2rom(self):
     def test_romanize_thai2rom_onnx(self):
         self.assertEqual(romanize("แมว", engine="thai2rom_onnx"), "maeo")
         self.assertEqual(romanize("บ้านไร่", engine="thai2rom_onnx"), "banrai")
+        self.assertEqual(romanize("สมชาย", engine="thai2rom_onnx"), "somchai")
         self.assertEqual(romanize("สุนัข", engine="thai2rom_onnx"), "sunak")
         self.assertEqual(romanize("นก", engine="thai2rom_onnx"), "nok")
         self.assertEqual(

tests/noauto_onnx/testn_tokenize_onnx.py

@@ -84,12 +84,46 @@ class TransliterateONNXTestCaseN(unittest.TestCase):
     """Tests for ONNX-based transliteration (requires onnxruntime)"""
 
     def test_thai2rom_onnx(self):
-        from pythainlp.transliterate.thai2rom_onnx import romanize
+        from pythainlp.transliterate import romanize
 
-        result = romanize("สวัสดี")
+        # Basic smoke test
+        result = romanize("สวัสดี", engine="thai2rom_onnx")
         self.assertIsInstance(result, str)
         self.assertGreater(len(result), 0)
 
+    def test_thai2rom_onnx_romanize(self):
+        from pythainlp.transliterate import romanize
+
+        # The bug-report case (#1346): must not return "" or "<PAD>"
+        self.assertEqual(romanize("สมชาย", engine="thai2rom_onnx"), "somchai")
+
+        # Common words
+        self.assertEqual(romanize("แมว", engine="thai2rom_onnx"), "maeo")
+        self.assertEqual(romanize("บ้านไร่", engine="thai2rom_onnx"), "banrai")
+        self.assertEqual(romanize("สุนัข", engine="thai2rom_onnx"), "sunak")
+        self.assertEqual(romanize("นก", engine="thai2rom_onnx"), "nok")
+        self.assertEqual(romanize("ความอิ่ม", engine="thai2rom_onnx"), "khwam-im")
+        self.assertEqual(romanize("สกุนต์", engine="thai2rom_onnx"), "sakun")
+        self.assertEqual(romanize("ชารินทร์", engine="thai2rom_onnx"), "charin")
+
+        # Multi-word input (space-separated)
+        result = romanize("กานต์ ณรงค์", engine="thai2rom_onnx")
+        self.assertIsInstance(result, str)
+        self.assertIn(" ", result)
+
+    def test_thai2rom_onnx_edge_cases(self):
+        from pythainlp.transliterate import romanize
+
+        # Empty string should return empty string
+        self.assertEqual(romanize("", engine="thai2rom_onnx"), "")
+
+        # Output must never contain raw special tokens
+        for word in ("สมชาย", "แมว", "นก"):
+            result = romanize(word, engine="thai2rom_onnx")
+            self.assertNotIn("<PAD>", result)
+            self.assertNotIn("<start>", result)
+            self.assertNotIn("<end>", result)
+
 
 class TagONNXTestCaseN(unittest.TestCase):
     """Tests for ONNX-based POS tagging (requires onnxruntime)"""