fix: resolve 16 failing unit tests across Chinese ITN, Japanese ITN, and English TN

- Chinese ITN: prefer currency symbol ($) over ISO code (USD) by adding
  weight to code FST in money rule
- Japanese ITN: fix big_integer FST structure to prevent 三十 being split
  into 3+10=310 instead of parsed as 30
- English TN: prefer "oh" over "zero" for digit 0 in non-deterministic
  mode by adding weight to zero path
- English TN: fix whitelist test expectation for Hon. -> Honorable

Author: pengzhendong

itn/chinese/rules/money.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from pynini import string_file
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from itn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
@@ -42,7 +42,7 @@ def build_tagger(self):
             + number
             + insert('"')
             + insert(' currency: "')
-            + (code | symbol)
+            + (symbol | add_weight(code, 1))
             + insert('"')
             + insert(' decimal: "')
             + (insert(".") + digit + (delete("毛") | delete("角")) + (digit + delete("分")).ques).ques

itn/japanese/rules/cardinal.py

@@ -122,17 +122,13 @@ def build_tagger(self):
                 (ten_thousand_minus + accep("兆")).ques
                 + ten_thousand_minus
                 + accep("億")
-                + ten_thousand_minus
-                + accep("万").ques
-                + ten_thousand_minus.ques
+                + (ten_thousand_minus + (accep("万") + ten_thousand_minus.ques).ques).ques
             )
             | (
                 ten_thousand_minus
                 + accep("兆")
                 + (ten_thousand_minus + accep("億")).ques
-                + ten_thousand_minus
-                + accep("万").ques
-                + ten_thousand_minus.ques
+                + (ten_thousand_minus + (accep("万") + ten_thousand_minus.ques).ques).ques
             )
         )
         self.big_integer = number | big_integer

tn/english/rules/cardinal.py

@@ -58,7 +58,9 @@ def build_tagger(self):
             single_digits_graph_zero = pynini.invert(graph_digit | graph_zero)
             single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross("0", "oh")
 
-            self.single_digits_graph = single_digits_graph_zero + (self.INSERT_SPACE + single_digits_graph_zero).star
+            self.single_digits_graph = pynutil.add_weight(
+                single_digits_graph_zero + (self.INSERT_SPACE + single_digits_graph_zero).star, 1.0
+            )
             self.single_digits_graph |= single_digits_graph_oh + (self.INSERT_SPACE + single_digits_graph_oh).star
 
             single_digits_graph_with_commas = (

tn/english/test/data/whitelist.txt

@@ -1,5 +1,5 @@
 Ph.D. => PHD
-Hon. => honorable
+Hon. => Honorable
 Mt. => Mount
 Maj. => Major
 Rev. => Reverend