fix: resolve 16 failing unit tests (#332)

* fix: resolve 16 failing unit tests across Chinese ITN, Japanese ITN, and English TN

- Chinese ITN: prefer currency symbol ($) over ISO code (USD) by adding
  weight to code FST in money rule
- Japanese ITN: fix big_integer FST structure to prevent 三十 being split
  into 3+10=310 instead of parsed as 30
- English TN: prefer "oh" over "zero" for digit 0 in non-deterministic
  mode by adding weight to zero path
- English TN: fix whitelist test expectation for Hon. -> Honorable

* refactor: deduplicate whitelist entries in tts.tsv

Remove entries from tts.tsv that duplicate mappings in alternatives.tsv
(Hon., Mr., Mrs., Ms.) or are self-duplicated Greek letters (κ, ω, α,
ν, δ, Δ, Α, β, Β). This avoids non-deterministic FST behavior across
platforms when the same key maps to different values in unioned files.

Author: pengzhendong

itn/chinese/rules/money.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from pynini import string_file
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from itn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
@@ -42,7 +42,7 @@ def build_tagger(self):
             + number
             + insert('"')
             + insert(' currency: "')
-            + (code | symbol)
+            + (symbol | add_weight(code, 1))
             + insert('"')
             + insert(' decimal: "')
             + (insert(".") + digit + (delete("毛") | delete("角")) + (digit + delete("分")).ques).ques

itn/japanese/rules/cardinal.py

@@ -122,17 +122,13 @@ def build_tagger(self):
                 (ten_thousand_minus + accep("兆")).ques
                 + ten_thousand_minus
                 + accep("億")
-                + ten_thousand_minus
-                + accep("万").ques
-                + ten_thousand_minus.ques
+                + (ten_thousand_minus + (accep("万") + ten_thousand_minus.ques).ques).ques
             )
             | (
                 ten_thousand_minus
                 + accep("兆")
                 + (ten_thousand_minus + accep("億")).ques
-                + ten_thousand_minus
-                + accep("万").ques
-                + ten_thousand_minus.ques
+                + (ten_thousand_minus + (accep("万") + ten_thousand_minus.ques).ques).ques
             )
         )
         self.big_integer = number | big_integer

tn/english/data/whitelist/tts.tsv

@@ -1,5 +1,4 @@
 Ph.D.	PHD
-Hon.	honorable
 Mt.	Mount
 Maj.	Major
 Rev.	Reverend
@@ -15,15 +14,6 @@ HVAC	H-vac
 SPDR	spider
 ZIP	zip
 ~	approximately
-κ	kappa
-ω	omega
-α	alpha
-ν	nu
-δ	delta
-Δ	delta
-Α	alpha
-β	beta
-Β	beta
 χ	chi
 Χ	chi
 ε	epsilon
@@ -1934,16 +1924,13 @@ M. Q.	MQ
 Mra	MRA
 MRBs	MRB's
 MR&LE	MR and LE
-Mr.	mister
 M. R.	MR
 M.R.	MR
 mRNA	MRNA
 mRNAs	MRNA's
-Mrs.	misses
 MRTs	MRT's
 Mse	MSE
 M&S	M and S
-Ms.	miss
 M. S.	MS
 M.S.	MS
 MSPs	MSP's

tn/english/rules/cardinal.py

@@ -58,7 +58,9 @@ def build_tagger(self):
             single_digits_graph_zero = pynini.invert(graph_digit | graph_zero)
             single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross("0", "oh")
 
-            self.single_digits_graph = single_digits_graph_zero + (self.INSERT_SPACE + single_digits_graph_zero).star
+            self.single_digits_graph = pynutil.add_weight(
+                single_digits_graph_zero + (self.INSERT_SPACE + single_digits_graph_zero).star, 1.0
+            )
             self.single_digits_graph |= single_digits_graph_oh + (self.INSERT_SPACE + single_digits_graph_oh).star
 
             single_digits_graph_with_commas = (

tn/english/test/data/whitelist.txt

@@ -1,5 +1,5 @@
 Ph.D. => PHD
-Hon. => honorable
+Hon. => Honorable
 Mt. => Mount
 Maj. => Major
 Rev. => Reverend