refactor: code review cleanup and bug fixes

- Fix import ordering in processor.py
- Fix 亿级 bug: 100001000 => 一亿零一千 (was 一亿一千)
- Share rule instances in Chinese ITN verbalizer (build time -15%)
- Don't create Transliteration when transliterate=False in Japanese TN
- Share Whitelist instance in Chinese TN normalizer
- Support full-width comma in Measure strip_comma

Author: pengzhendong

itn/chinese/inverse_normalizer.py

@@ -52,36 +52,46 @@ def __init__(
 
     def build_tagger_and_verbalizer(self):
         cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million)
+        char = Char()
+        date = Date()
+        fraction = Fraction(cardinal=cardinal)
+        train_number = TrainNumber()
+        math = Math(cardinal=cardinal)
+        measure = Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        money = Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        time = Time()
+        license_plate = LicensePlate()
+        whitelist = Whitelist()
 
         tagger = (
-            add_weight(Date().tagger, 1.02)
-            | add_weight(Whitelist().tagger, 1.01)
-            | add_weight(Fraction(cardinal=cardinal).tagger, 1.05)
-            | add_weight(Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.05)
-            | add_weight(Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.04)
-            | add_weight(Time().tagger, 1.05)
+            add_weight(date.tagger, 1.02)
+            | add_weight(whitelist.tagger, 1.01)
+            | add_weight(fraction.tagger, 1.05)
+            | add_weight(measure.tagger, 1.05)
+            | add_weight(money.tagger, 1.04)
+            | add_weight(time.tagger, 1.05)
             | add_weight(cardinal.tagger, 1.06)
-            | add_weight(Math(cardinal=cardinal).tagger, 1.10)
-            | add_weight(LicensePlate().tagger, 1.0)
-            | add_weight(TrainNumber().tagger, 1.0)
-            | add_weight(Char().tagger, 100)
+            | add_weight(math.tagger, 1.10)
+            | add_weight(license_plate.tagger, 1.0)
+            | add_weight(train_number.tagger, 1.0)
+            | add_weight(char.tagger, 100)
         ).optimize()
 
         tagger = tagger.star
         self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]")
 
         verbalizer = (
             cardinal.verbalizer
-            | Char().verbalizer
-            | Date().verbalizer
-            | Fraction().verbalizer
-            | TrainNumber().verbalizer
-            | Math().verbalizer
-            | Measure(enable_0_to_9=self.enable_0_to_9).verbalizer
-            | Money(enable_0_to_9=self.enable_0_to_9).verbalizer
-            | Time().verbalizer
-            | LicensePlate().verbalizer
-            | Whitelist().verbalizer
+            | char.verbalizer
+            | date.verbalizer
+            | fraction.verbalizer
+            | train_number.verbalizer
+            | math.verbalizer
+            | measure.verbalizer
+            | money.verbalizer
+            | time.verbalizer
+            | license_plate.verbalizer
+            | whitelist.verbalizer
         ).optimize()
         postprocessor = PostProcessor(remove_interjections=self.remove_interjections).processor
 

tn/chinese/normalizer.py

@@ -58,7 +58,7 @@ def build_tagger_and_verbalizer(self):
         processor = PreProcessor(traditional_to_simple=self.traditional_to_simple).processor
         cardinal = Cardinal()
         date = Date()
-        whitelist = Whitelist()
+        whitelist = Whitelist(remove_erhua=self.remove_erhua)
         sport = Sport(cardinal=cardinal)
         fraction = Fraction(cardinal=cardinal)
         measure = Measure(cardinal=cardinal)
@@ -92,7 +92,7 @@ def build_tagger_and_verbalizer(self):
             | money.verbalizer
             | sport.verbalizer
             | time.verbalizer
-            | Whitelist(remove_erhua=self.remove_erhua).verbalizer
+            | whitelist.verbalizer
         ).optimize()
 
         postprocessor = PostProcessor(

tn/chinese/rules/cardinal.py

@@ -64,7 +64,16 @@ def build_tagger(self):
         hundred_million = (
             (thousand | hundred | ten | digit)
             + insert("亿")
-            + (four_nonzero + insert("万") + four_any | rmzero**4 + four_nonzero | rmzero**8)
+            + (
+                four_nonzero + insert("万") + four_any
+                | rmzero**4 + (
+                    (zero + hundred)
+                    | (rmzero + zero + tens)
+                    | (rmzero**2 + zero + digit)
+                    | (insert("零") + thousand)
+                )
+                | rmzero**8
+            )
         )
 
         number = digits | ten | hundred | thousand | ten_thousand | hundred_million

tn/chinese/rules/measure.py

@@ -36,7 +36,7 @@ def build_tagger(self):
         to = cross("-", "到") | cross("~", "到") | accep("到")
 
         number = self.cardinal.number
-        strip_comma = self.build_rule(delete(","), self.DIGIT, self.DIGIT)
+        strip_comma = self.build_rule(delete(",") | delete("，"), self.DIGIT, self.DIGIT)
         number = strip_comma @ number
         number @= self.build_rule(cross("二", "两"), "[BOS]", "[EOS]")
         # 1-11个，1个-11个

tn/japanese/normalizer.py

@@ -84,7 +84,6 @@ def build_tagger_and_verbalizer(self):
         tagger = (processor @ tagger).star
         self.tagger = tagger @ self.build_rule(delete(" "), r="[EOS]")
 
-        transliteration = Transliteration()
         verbalizer = (
             cardinal.verbalizer
             | char.verbalizer

tn/processor.py

@@ -18,16 +18,16 @@
 
 from pynini import Fst, cdrewrite, cross, difference, escape, invert, shortestpath, union
 from pynini.lib import byte, utf8
+from pynini.lib.pynutil import delete, insert
+
+from tn.token_parser import TokenParser
 
 logger = logging.getLogger("wetext")
 if not logger.handlers:
     handler = logging.StreamHandler()
     handler.setFormatter(logging.Formatter("%(asctime)s WETEXT %(levelname)s %(message)s"))
     logger.addHandler(handler)
     logger.setLevel(logging.INFO)
-from pynini.lib.pynutil import delete, insert
-
-from tn.token_parser import TokenParser
 
 
 class Processor: