refactor: share rule instances to speed up FST construction

itn/chinese/inverse_normalizer.py

@@ -49,27 +49,27 @@ def __init__(
             cache_dir = files("itn")
         self.build_fst("zh_itn", cache_dir, overwrite_cache)
 
-    def build_tagger(self):
+    def build_tagger_and_verbalizer(self):
+        cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million)
+
         tagger = (
             add_weight(Date().tagger, 1.02)
             | add_weight(Whitelist().tagger, 1.01)
-            | add_weight(Fraction().tagger, 1.05)
-            | add_weight(Measure(enable_0_to_9=self.enable_0_to_9).tagger, 1.05)
-            | add_weight(Money(enable_0_to_9=self.enable_0_to_9).tagger, 1.04)
+            | add_weight(Fraction(cardinal=cardinal).tagger, 1.05)
+            | add_weight(Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.05)
+            | add_weight(Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.04)
             | add_weight(Time().tagger, 1.05)
-            | add_weight(Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).tagger, 1.06)
-            | add_weight(Math().tagger, 1.10)
+            | add_weight(cardinal.tagger, 1.06)
+            | add_weight(Math(cardinal=cardinal).tagger, 1.10)
             | add_weight(LicensePlate().tagger, 1.0)
             | add_weight(Char().tagger, 100)
         ).optimize()
 
         tagger = tagger.star
-        # remove the last space
         self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]")
 
-    def build_verbalizer(self):
         verbalizer = (
-            Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).verbalizer
+            cardinal.verbalizer
             | Char().verbalizer
             | Date().verbalizer
             | Fraction().verbalizer

itn/chinese/rules/fraction.py

@@ -22,13 +22,14 @@
 
 class Fraction(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="fraction")
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        number = Cardinal().number
+        number = self.cardinal.number
         sign = string_file(get_abs_path("../itn/chinese/data/number/sign.tsv"))  # + -
 
         # NOTE(xcsong): default weight = 1.0,  set to -1.0 means higher priority

itn/chinese/rules/math.py

@@ -22,15 +22,16 @@
 
 class Math(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="math")
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
         operator = string_file(get_abs_path("../itn/chinese/data/math/operator.tsv"))
 
-        number = Cardinal().number
+        number = self.cardinal.number
         tagger = number + (operator + number).plus
         tagger = insert('value: "') + tagger + insert('"')
         self.tagger = self.add_tokens(tagger)

itn/chinese/rules/measure.py

@@ -22,10 +22,11 @@
 
 class Measure(Processor):
 
-    def __init__(self, exclude_one=True, enable_0_to_9=True):
+    def __init__(self, exclude_one=True, enable_0_to_9=True, cardinal=None):
         super().__init__(name="measure")
         self.exclude_one = exclude_one
         self.enable_0_to_9 = enable_0_to_9
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
@@ -43,15 +44,15 @@ def build_tagger(self):
             add_weight(units_en, -1.0)
         )
 
-        number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9
+        number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9
         # 百分之三十, 百分三十, 百分之百，百分之三十到四十, 百分之三十到百分之五十五
         percent = (
             (sign + delete("的").ques).ques
             + delete("百分")
             + delete("之").ques
             + (
-                (Cardinal().number + (to + Cardinal().number).ques)
-                | ((Cardinal().number + to).ques + cross("百", "100"))
+                (self.cardinal.number + (to + self.cardinal.number).ques)
+                | ((self.cardinal.number + to).ques + cross("百", "100"))
             )
             + insert("%")
         )

itn/chinese/rules/money.py

@@ -22,9 +22,10 @@
 
 class Money(Processor):
 
-    def __init__(self, enable_0_to_9=True):
+    def __init__(self, enable_0_to_9=True, cardinal=None):
         super().__init__(name="money")
         self.enable_0_to_9 = enable_0_to_9
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
@@ -33,7 +34,7 @@ def build_tagger(self):
         symbol = string_file(get_abs_path("../itn/chinese/data/money/symbol.tsv"))
         digit = string_file(get_abs_path("../itn/chinese/data/number/digit.tsv"))  # 1 ~ 9
 
-        number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9
+        number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9
         # 七八美元 => $7~8
         number |= digit + insert("~") + digit
         # 三千三百八十元五毛八分 => ¥3380.58

itn/japanese/inverse_normalizer.py

@@ -50,40 +50,47 @@ def __init__(
             cache_dir = files("itn")
         self.build_fst("ja_itn", cache_dir, overwrite_cache)
 
-    def build_tagger(self):
+    def build_tagger_and_verbalizer(self):
         processor = PreProcessor(full_to_half=self.full_to_half).processor
-
-        cardinal = add_weight(Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).tagger, 1.06)
-        char = add_weight(Char().tagger, 100)
-        date = add_weight(Date().tagger, 1.02)
-        fraction = add_weight(Fraction().tagger, 1.05)
-        math = add_weight(Math().tagger, 90)
-        measure = add_weight(Measure(enable_0_to_9=self.enable_0_to_9).tagger, 1.05)
-        money = add_weight(Money(enable_0_to_9=self.enable_0_to_9).tagger, 1.04)
-        ordinal = add_weight(Ordinal().tagger, 1.04)
-        time = add_weight(Time().tagger, 1.04)
-        whitelist = add_weight(Whitelist().tagger, 1.01)
+        cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million)
+        cardinal_million = Cardinal(enable_million=True)
+        char = Char()
+        date = Date(cardinal=cardinal)
+        fraction = Fraction(cardinal=cardinal_million)
+        math = Math(cardinal=cardinal)
+        measure = Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        money = Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        ordinal = Ordinal(cardinal=cardinal)
+        time = Time()
+        whitelist = Whitelist()
 
         tagger = (
-            (cardinal | char | date | fraction | math | measure | money | ordinal | time | whitelist).optimize().star
-        )
+            add_weight(cardinal.tagger, 1.06)
+            | add_weight(char.tagger, 100)
+            | add_weight(date.tagger, 1.02)
+            | add_weight(fraction.tagger, 1.05)
+            | add_weight(math.tagger, 90)
+            | add_weight(measure.tagger, 1.05)
+            | add_weight(money.tagger, 1.04)
+            | add_weight(ordinal.tagger, 1.04)
+            | add_weight(time.tagger, 1.04)
+            | add_weight(whitelist.tagger, 1.01)
+        ).optimize().star
         tagger = (processor @ tagger).star
-        # remove the last space
         self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]")
 
-    def build_verbalizer(self):
-        cardinal = Cardinal(self.convert_number, self.enable_0_to_9).verbalizer
-        char = Char().verbalizer
-        date = Date().verbalizer
-        fraction = Fraction().verbalizer
-        math = Math().verbalizer
-        measure = Measure().verbalizer
-        money = Money().verbalizer
-        ordinal = Ordinal().verbalizer
-        time = Time().verbalizer
-        whitelist = Whitelist().verbalizer
-
-        verbalizer = cardinal | char | date | fraction | math | measure | money | ordinal | time | whitelist
+        verbalizer = (
+            cardinal.verbalizer
+            | char.verbalizer
+            | date.verbalizer
+            | fraction.verbalizer
+            | math.verbalizer
+            | measure.verbalizer
+            | money.verbalizer
+            | ordinal.verbalizer
+            | time.verbalizer
+            | whitelist.verbalizer
+        )
 
-        processor = PostProcessor().processor
-        self.verbalizer = (verbalizer @ processor).star
+        postprocessor = PostProcessor().processor
+        self.verbalizer = (verbalizer @ postprocessor).star

itn/japanese/rules/date.py

@@ -22,13 +22,14 @@
 
 class Date(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="date")
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        cardinal = Cardinal().ten_thousand_minus
+        cardinal = self.cardinal.ten_thousand_minus
         day = string_file(get_abs_path("../itn/japanese/data/date/day.tsv"))
         month = string_file(get_abs_path("../itn/japanese/data/date/month.tsv"))
         to = cross("から", "〜")

itn/japanese/rules/fraction.py

@@ -22,14 +22,15 @@
 
 class Fraction(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="fraction")
+        self.cardinal = cardinal or Cardinal(enable_million=True)
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        cardinal = Cardinal(enable_million=True).number
-        decimal = Cardinal(enable_million=True).decimal
+        cardinal = self.cardinal.number
+        decimal = self.cardinal.decimal
         sign = string_file(get_abs_path("../itn/japanese/data/number/sign.tsv"))
         sign = insert('sign: "') + sign + insert('"')
 

itn/japanese/rules/math.py

@@ -22,16 +22,17 @@
 
 class Math(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="math")
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
         operator = string_file(get_abs_path("../itn/japanese/data/math/operator.tsv"))
 
-        number = Cardinal().big_integer
-        decimal = Cardinal().decimal
+        number = self.cardinal.big_integer
+        decimal = self.cardinal.decimal
         number |= decimal
         tagger = number + (operator + number).plus
         tagger = insert('value: "') + tagger + insert('"')

itn/japanese/rules/measure.py

@@ -22,18 +22,19 @@
 
 class Measure(Processor):
 
-    def __init__(self, enable_0_to_9=True):
+    def __init__(self, enable_0_to_9=True, cardinal=None):
         super().__init__(name="measure")
         self.enable_0_to_9 = enable_0_to_9
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
         unit_en = string_file(get_abs_path("../itn/japanese/data/measure/unit_en.tsv"))
         unit_ja = string_file(get_abs_path("../itn/japanese/data/measure/unit_ja.tsv"))
 
-        cardinal = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9
-        decimal = Cardinal().decimal
+        cardinal = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9
+        decimal = self.cardinal.decimal
 
         suffix = (
             insert("/")

itn/japanese/rules/money.py

@@ -22,17 +22,18 @@
 
 class Money(Processor):
 
-    def __init__(self, enable_0_to_9=True):
+    def __init__(self, enable_0_to_9=True, cardinal=None):
         super().__init__(name="money")
         self.enable_0_to_9 = enable_0_to_9
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
         symbol = string_file(get_abs_path("../itn/japanese/data/money/symbol.tsv"))
 
-        number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9
-        decimal = Cardinal().decimal
+        number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9
+        decimal = self.cardinal.decimal
         # 三千三百八十点五八円 => ¥3380.58
         tagger = insert('value: "') + (number | decimal) + insert('"') + insert(' currency: "') + symbol + insert('"')
         self.tagger = self.add_tokens(tagger)

itn/japanese/rules/ordinal.py

@@ -21,13 +21,14 @@
 
 class Ordinal(Processor):
 
-    def __init__(self):
+    def __init__(self, cardinal=None):
         super().__init__(name="ordinal")
+        self.cardinal = cardinal or Cardinal()
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        cardinal = Cardinal().number
+        cardinal = self.cardinal.number
         ordinal = (cardinal + accep("番目")) | (accep("第") + cardinal)
         tagger = insert('value: "') + ordinal + insert('"')
         self.tagger = self.add_tokens(tagger)