From 6f52cd23784770811bbb852bd9553af0d8dee812 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Tue, 9 Jun 2026 15:17:14 +0800 Subject: [PATCH] refactor: share rule instances to eliminate redundant FST construction Each rule class now accepts optional dependency params (e.g. cardinal=None) and reuses injected instances instead of creating its own. Normalizer and InverseNormalizer build all rules once in topological order via a new build_tagger_and_verbalizer() method, sharing instances across tagger and verbalizer construction. Before: Cardinal was instantiated 60 times per English Normalizer build. After: Cardinal is instantiated once. Performance improvement: - English Normalizer build: 71s -> 17s (4x faster) - Full test suite: 196s -> 88s (2.2x faster) --- itn/chinese/inverse_normalizer.py | 18 ++--- itn/chinese/rules/fraction.py | 5 +- itn/chinese/rules/math.py | 5 +- itn/chinese/rules/measure.py | 9 +-- itn/chinese/rules/money.py | 5 +- itn/japanese/inverse_normalizer.py | 67 ++++++++++-------- itn/japanese/rules/date.py | 5 +- itn/japanese/rules/fraction.py | 7 +- itn/japanese/rules/math.py | 7 +- itn/japanese/rules/measure.py | 7 +- itn/japanese/rules/money.py | 7 +- itn/japanese/rules/ordinal.py | 5 +- tn/chinese/normalizer.py | 66 ++++++++++-------- tn/chinese/rules/fraction.py | 5 +- tn/chinese/rules/math.py | 6 +- tn/chinese/rules/measure.py | 7 +- tn/chinese/rules/money.py | 5 +- tn/chinese/rules/sport.py | 5 +- tn/english/normalizer.py | 108 ++++++++++++----------------- tn/english/rules/date.py | 13 ++-- tn/english/rules/decimal.py | 12 ++-- tn/english/rules/electronic.py | 10 +-- tn/english/rules/fraction.py | 13 ++-- tn/english/rules/measure.py | 25 ++++--- tn/english/rules/money.py | 15 ++-- tn/english/rules/ordinal.py | 11 +-- tn/english/rules/range.py | 13 ++-- tn/english/rules/time.py | 10 +-- tn/english/rules/word.py | 10 +-- tn/japanese/normalizer.py | 72 +++++++++++-------- tn/japanese/rules/date.py | 5 +- tn/japanese/rules/fraction.py | 5 +- tn/japanese/rules/math.py | 5 +- tn/japanese/rules/measure.py | 5 +- tn/japanese/rules/money.py | 5 +- tn/japanese/rules/sport.py | 5 +- tn/processor.py | 7 +- 37 files changed, 295 insertions(+), 295 deletions(-) diff --git a/itn/chinese/inverse_normalizer.py b/itn/chinese/inverse_normalizer.py index 0f08b2c6..b4516fdd 100644 --- a/itn/chinese/inverse_normalizer.py +++ b/itn/chinese/inverse_normalizer.py @@ -49,27 +49,27 @@ def __init__( cache_dir = files("itn") self.build_fst("zh_itn", cache_dir, overwrite_cache) - def build_tagger(self): + def build_tagger_and_verbalizer(self): + cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million) + tagger = ( add_weight(Date().tagger, 1.02) | add_weight(Whitelist().tagger, 1.01) - | add_weight(Fraction().tagger, 1.05) - | add_weight(Measure(enable_0_to_9=self.enable_0_to_9).tagger, 1.05) - | add_weight(Money(enable_0_to_9=self.enable_0_to_9).tagger, 1.04) + | add_weight(Fraction(cardinal=cardinal).tagger, 1.05) + | add_weight(Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.05) + | add_weight(Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.04) | add_weight(Time().tagger, 1.05) - | add_weight(Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).tagger, 1.06) - | add_weight(Math().tagger, 1.10) + | add_weight(cardinal.tagger, 1.06) + | add_weight(Math(cardinal=cardinal).tagger, 1.10) | add_weight(LicensePlate().tagger, 1.0) | add_weight(Char().tagger, 100) ).optimize() tagger = tagger.star - # remove the last space self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]") - def build_verbalizer(self): verbalizer = ( - Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).verbalizer + cardinal.verbalizer | Char().verbalizer | Date().verbalizer | Fraction().verbalizer diff --git a/itn/chinese/rules/fraction.py b/itn/chinese/rules/fraction.py index 87700785..718d135e 100644 --- a/itn/chinese/rules/fraction.py +++ b/itn/chinese/rules/fraction.py @@ -22,13 +22,14 @@ class Fraction(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="fraction") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): - number = Cardinal().number + number = self.cardinal.number sign = string_file(get_abs_path("../itn/chinese/data/number/sign.tsv")) # + - # NOTE(xcsong): default weight = 1.0, set to -1.0 means higher priority diff --git a/itn/chinese/rules/math.py b/itn/chinese/rules/math.py index 14253f10..b358a478 100644 --- a/itn/chinese/rules/math.py +++ b/itn/chinese/rules/math.py @@ -22,15 +22,16 @@ class Math(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="math") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): operator = string_file(get_abs_path("../itn/chinese/data/math/operator.tsv")) - number = Cardinal().number + number = self.cardinal.number tagger = number + (operator + number).plus tagger = insert('value: "') + tagger + insert('"') self.tagger = self.add_tokens(tagger) diff --git a/itn/chinese/rules/measure.py b/itn/chinese/rules/measure.py index 23ff24f3..2077a300 100644 --- a/itn/chinese/rules/measure.py +++ b/itn/chinese/rules/measure.py @@ -22,10 +22,11 @@ class Measure(Processor): - def __init__(self, exclude_one=True, enable_0_to_9=True): + def __init__(self, exclude_one=True, enable_0_to_9=True, cardinal=None): super().__init__(name="measure") self.exclude_one = exclude_one self.enable_0_to_9 = enable_0_to_9 + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -43,15 +44,15 @@ def build_tagger(self): add_weight(units_en, -1.0) ) - number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9 + number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9 # 百分之三十, 百分三十, 百分之百，百分之三十到四十, 百分之三十到百分之五十五 percent = ( (sign + delete("的").ques).ques + delete("百分") + delete("之").ques + ( - (Cardinal().number + (to + Cardinal().number).ques) - | ((Cardinal().number + to).ques + cross("百", "100")) + (self.cardinal.number + (to + self.cardinal.number).ques) + | ((self.cardinal.number + to).ques + cross("百", "100")) ) + insert("%") ) diff --git a/itn/chinese/rules/money.py b/itn/chinese/rules/money.py index e9c21d1e..1841ae20 100644 --- a/itn/chinese/rules/money.py +++ b/itn/chinese/rules/money.py @@ -22,9 +22,10 @@ class Money(Processor): - def __init__(self, enable_0_to_9=True): + def __init__(self, enable_0_to_9=True, cardinal=None): super().__init__(name="money") self.enable_0_to_9 = enable_0_to_9 + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -33,7 +34,7 @@ def build_tagger(self): symbol = string_file(get_abs_path("../itn/chinese/data/money/symbol.tsv")) digit = string_file(get_abs_path("../itn/chinese/data/number/digit.tsv")) # 1 ~ 9 - number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9 + number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9 # 七八美元 => $7~8 number |= digit + insert("~") + digit # 三千三百八十元五毛八分 => ¥3380.58 diff --git a/itn/japanese/inverse_normalizer.py b/itn/japanese/inverse_normalizer.py index cb936969..5f5813a8 100644 --- a/itn/japanese/inverse_normalizer.py +++ b/itn/japanese/inverse_normalizer.py @@ -50,40 +50,47 @@ def __init__( cache_dir = files("itn") self.build_fst("ja_itn", cache_dir, overwrite_cache) - def build_tagger(self): + def build_tagger_and_verbalizer(self): processor = PreProcessor(full_to_half=self.full_to_half).processor - - cardinal = add_weight(Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million).tagger, 1.06) - char = add_weight(Char().tagger, 100) - date = add_weight(Date().tagger, 1.02) - fraction = add_weight(Fraction().tagger, 1.05) - math = add_weight(Math().tagger, 90) - measure = add_weight(Measure(enable_0_to_9=self.enable_0_to_9).tagger, 1.05) - money = add_weight(Money(enable_0_to_9=self.enable_0_to_9).tagger, 1.04) - ordinal = add_weight(Ordinal().tagger, 1.04) - time = add_weight(Time().tagger, 1.04) - whitelist = add_weight(Whitelist().tagger, 1.01) + cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million) + cardinal_million = Cardinal(enable_million=True) + char = Char() + date = Date(cardinal=cardinal) + fraction = Fraction(cardinal=cardinal_million) + math = Math(cardinal=cardinal) + measure = Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal) + money = Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal) + ordinal = Ordinal(cardinal=cardinal) + time = Time() + whitelist = Whitelist() tagger = ( - (cardinal | char | date | fraction | math | measure | money | ordinal | time | whitelist).optimize().star - ) + add_weight(cardinal.tagger, 1.06) + | add_weight(char.tagger, 100) + | add_weight(date.tagger, 1.02) + | add_weight(fraction.tagger, 1.05) + | add_weight(math.tagger, 90) + | add_weight(measure.tagger, 1.05) + | add_weight(money.tagger, 1.04) + | add_weight(ordinal.tagger, 1.04) + | add_weight(time.tagger, 1.04) + | add_weight(whitelist.tagger, 1.01) + ).optimize().star tagger = (processor @ tagger).star - # remove the last space self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]") - def build_verbalizer(self): - cardinal = Cardinal(self.convert_number, self.enable_0_to_9).verbalizer - char = Char().verbalizer - date = Date().verbalizer - fraction = Fraction().verbalizer - math = Math().verbalizer - measure = Measure().verbalizer - money = Money().verbalizer - ordinal = Ordinal().verbalizer - time = Time().verbalizer - whitelist = Whitelist().verbalizer - - verbalizer = cardinal | char | date | fraction | math | measure | money | ordinal | time | whitelist + verbalizer = ( + cardinal.verbalizer + | char.verbalizer + | date.verbalizer + | fraction.verbalizer + | math.verbalizer + | measure.verbalizer + | money.verbalizer + | ordinal.verbalizer + | time.verbalizer + | whitelist.verbalizer + ) - processor = PostProcessor().processor - self.verbalizer = (verbalizer @ processor).star + postprocessor = PostProcessor().processor + self.verbalizer = (verbalizer @ postprocessor).star diff --git a/itn/japanese/rules/date.py b/itn/japanese/rules/date.py index 0db0e78d..b681c4fe 100644 --- a/itn/japanese/rules/date.py +++ b/itn/japanese/rules/date.py @@ -22,13 +22,14 @@ class Date(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="date") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): - cardinal = Cardinal().ten_thousand_minus + cardinal = self.cardinal.ten_thousand_minus day = string_file(get_abs_path("../itn/japanese/data/date/day.tsv")) month = string_file(get_abs_path("../itn/japanese/data/date/month.tsv")) to = cross("から", "〜") diff --git a/itn/japanese/rules/fraction.py b/itn/japanese/rules/fraction.py index 5ffdd89c..1eb87778 100644 --- a/itn/japanese/rules/fraction.py +++ b/itn/japanese/rules/fraction.py @@ -22,14 +22,15 @@ class Fraction(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="fraction") + self.cardinal = cardinal or Cardinal(enable_million=True) self.build_tagger() self.build_verbalizer() def build_tagger(self): - cardinal = Cardinal(enable_million=True).number - decimal = Cardinal(enable_million=True).decimal + cardinal = self.cardinal.number + decimal = self.cardinal.decimal sign = string_file(get_abs_path("../itn/japanese/data/number/sign.tsv")) sign = insert('sign: "') + sign + insert('"') diff --git a/itn/japanese/rules/math.py b/itn/japanese/rules/math.py index f72d6bb2..3d3b134a 100644 --- a/itn/japanese/rules/math.py +++ b/itn/japanese/rules/math.py @@ -22,16 +22,17 @@ class Math(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="math") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): operator = string_file(get_abs_path("../itn/japanese/data/math/operator.tsv")) - number = Cardinal().big_integer - decimal = Cardinal().decimal + number = self.cardinal.big_integer + decimal = self.cardinal.decimal number |= decimal tagger = number + (operator + number).plus tagger = insert('value: "') + tagger + insert('"') diff --git a/itn/japanese/rules/measure.py b/itn/japanese/rules/measure.py index 60602ecd..9d684ed7 100644 --- a/itn/japanese/rules/measure.py +++ b/itn/japanese/rules/measure.py @@ -22,9 +22,10 @@ class Measure(Processor): - def __init__(self, enable_0_to_9=True): + def __init__(self, enable_0_to_9=True, cardinal=None): super().__init__(name="measure") self.enable_0_to_9 = enable_0_to_9 + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -32,8 +33,8 @@ def build_tagger(self): unit_en = string_file(get_abs_path("../itn/japanese/data/measure/unit_en.tsv")) unit_ja = string_file(get_abs_path("../itn/japanese/data/measure/unit_ja.tsv")) - cardinal = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9 - decimal = Cardinal().decimal + cardinal = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9 + decimal = self.cardinal.decimal suffix = ( insert("/") diff --git a/itn/japanese/rules/money.py b/itn/japanese/rules/money.py index 03366ec2..6c90dedf 100644 --- a/itn/japanese/rules/money.py +++ b/itn/japanese/rules/money.py @@ -22,17 +22,18 @@ class Money(Processor): - def __init__(self, enable_0_to_9=True): + def __init__(self, enable_0_to_9=True, cardinal=None): super().__init__(name="money") self.enable_0_to_9 = enable_0_to_9 + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): symbol = string_file(get_abs_path("../itn/japanese/data/money/symbol.tsv")) - number = Cardinal().number if self.enable_0_to_9 else Cardinal().number_exclude_0_to_9 - decimal = Cardinal().decimal + number = self.cardinal.number if self.enable_0_to_9 else self.cardinal.number_exclude_0_to_9 + decimal = self.cardinal.decimal # 三千三百八十点五八円 => ¥3380.58 tagger = insert('value: "') + (number | decimal) + insert('"') + insert(' currency: "') + symbol + insert('"') self.tagger = self.add_tokens(tagger) diff --git a/itn/japanese/rules/ordinal.py b/itn/japanese/rules/ordinal.py index bc70a867..e8cf1568 100644 --- a/itn/japanese/rules/ordinal.py +++ b/itn/japanese/rules/ordinal.py @@ -21,13 +21,14 @@ class Ordinal(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="ordinal") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): - cardinal = Cardinal().number + cardinal = self.cardinal.number ordinal = (cardinal + accep("番目")) | (accep("第") + cardinal) tagger = insert('value: "') + ordinal + insert('"') self.tagger = self.add_tokens(tagger) diff --git a/tn/chinese/normalizer.py b/tn/chinese/normalizer.py index 3cfb176d..53c66b17 100644 --- a/tn/chinese/normalizer.py +++ b/tn/chinese/normalizer.py @@ -54,43 +54,51 @@ def __init__( cache_dir = files("tn") self.build_fst("zh_tn", cache_dir, overwrite_cache) - def build_tagger(self): + def build_tagger_and_verbalizer(self): processor = PreProcessor(traditional_to_simple=self.traditional_to_simple).processor + cardinal = Cardinal() + date = Date() + whitelist = Whitelist() + sport = Sport(cardinal=cardinal) + fraction = Fraction(cardinal=cardinal) + measure = Measure(cardinal=cardinal) + money = Money(cardinal=cardinal) + time = Time() + math = Math(cardinal=cardinal) + char = Char() - date = add_weight(Date().tagger, 1.02) - whitelist = add_weight(Whitelist().tagger, 1.03) - sport = add_weight(Sport().tagger, 1.04) - fraction = add_weight(Fraction().tagger, 1.05) - measure = add_weight(Measure().tagger, 1.05) - money = add_weight(Money().tagger, 1.05) - time = add_weight(Time().tagger, 1.05) - cardinal = add_weight(Cardinal().tagger, 1.06) - math = add_weight(Math().tagger, 90) - char = add_weight(Char().tagger, 100) - - tagger = (date | whitelist | sport | fraction | measure | money | time | cardinal | math | char).optimize() + tagger = ( + add_weight(date.tagger, 1.02) + | add_weight(whitelist.tagger, 1.03) + | add_weight(sport.tagger, 1.04) + | add_weight(fraction.tagger, 1.05) + | add_weight(measure.tagger, 1.05) + | add_weight(money.tagger, 1.05) + | add_weight(time.tagger, 1.05) + | add_weight(cardinal.tagger, 1.06) + | add_weight(math.tagger, 90) + | add_weight(char.tagger, 100) + ).optimize() tagger = (processor @ tagger).star - # delete the last space self.tagger = tagger @ self.build_rule(delete(" "), r="[EOS]") - def build_verbalizer(self): - cardinal = Cardinal().verbalizer - char = Char().verbalizer - date = Date().verbalizer - fraction = Fraction().verbalizer - math = Math().verbalizer - measure = Measure().verbalizer - money = Money().verbalizer - sport = Sport().verbalizer - time = Time().verbalizer - whitelist = Whitelist(remove_erhua=self.remove_erhua).verbalizer - - verbalizer = (cardinal | char | date | fraction | math | measure | money | sport | time | whitelist).optimize() + verbalizer = ( + cardinal.verbalizer + | char.verbalizer + | date.verbalizer + | fraction.verbalizer + | math.verbalizer + | measure.verbalizer + | money.verbalizer + | sport.verbalizer + | time.verbalizer + | Whitelist(remove_erhua=self.remove_erhua).verbalizer + ).optimize() - processor = PostProcessor( + postprocessor = PostProcessor( remove_interjections=self.remove_interjections, remove_puncts=self.remove_puncts, full_to_half=self.full_to_half, tag_oov=self.tag_oov, ).processor - self.verbalizer = (verbalizer @ processor).star + self.verbalizer = (verbalizer @ postprocessor).star diff --git a/tn/chinese/rules/fraction.py b/tn/chinese/rules/fraction.py index 66ee34d2..78ce634a 100644 --- a/tn/chinese/rules/fraction.py +++ b/tn/chinese/rules/fraction.py @@ -20,14 +20,15 @@ class Fraction(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="fraction") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): rmspace = delete(" ").ques - number = Cardinal().number + number = self.cardinal.number tagger = ( insert('numerator: "') diff --git a/tn/chinese/rules/math.py b/tn/chinese/rules/math.py index 2b866017..ce358bfb 100644 --- a/tn/chinese/rules/math.py +++ b/tn/chinese/rules/math.py @@ -22,17 +22,17 @@ class Math(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="math") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): operator = string_file(get_abs_path("chinese/data/math/operator.tsv")) - # When it appears alone, it is treated as punctuation symbols = cross("~", "到") | cross(":", "比") | cross("<", "小于") | cross(">", "大于") - number = Cardinal().number + number = self.cardinal.number tagger = number + (delete(" ").ques + (operator | symbols) + delete(" ").ques + number).star tagger |= operator tagger = insert('value: "') + tagger + insert('"') diff --git a/tn/chinese/rules/measure.py b/tn/chinese/rules/measure.py index fe70d0bc..264f260d 100644 --- a/tn/chinese/rules/measure.py +++ b/tn/chinese/rules/measure.py @@ -22,8 +22,9 @@ class Measure(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="measure") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -34,7 +35,7 @@ def build_tagger(self): rmspace = delete(" ").ques to = cross("-", "到") | cross("~", "到") | accep("到") - number = Cardinal().number + number = self.cardinal.number number @= self.build_rule(cross("二", "两"), "[BOS]", "[EOS]") # 1-11个，1个-11个 prefix = number + (rmspace + units).ques + to @@ -45,7 +46,7 @@ def build_tagger(self): measure @= self.build_rule(cross("到两" + unit, "到二" + unit), r="[EOS]") # -xxxx年, -xx年 - digits = Cardinal().digits + digits = self.cardinal.digits cardinal = digits**2 | digits**4 unit = accep("年") | accep("年度") | accep("赛季") prefix = cardinal + (rmspace + unit).ques + to diff --git a/tn/chinese/rules/money.py b/tn/chinese/rules/money.py index a27a0fb6..1d36cf57 100644 --- a/tn/chinese/rules/money.py +++ b/tn/chinese/rules/money.py @@ -22,8 +22,9 @@ class Money(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="money") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -31,7 +32,7 @@ def build_tagger(self): code = string_file(get_abs_path("chinese/data/money/code.tsv")) symbol = string_file(get_abs_path("chinese/data/money/symbol.tsv")) - number = Cardinal().number + number = self.cardinal.number tagger = ( insert('currency: "') + (code | symbol) diff --git a/tn/chinese/rules/sport.py b/tn/chinese/rules/sport.py index 6fe1066f..de109b24 100644 --- a/tn/chinese/rules/sport.py +++ b/tn/chinese/rules/sport.py @@ -22,8 +22,9 @@ class Sport(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="sport") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -33,7 +34,7 @@ def build_tagger(self): rmsign = delete("/") | delete("-") | delete(":") rmspace = delete(" ").ques - number = Cardinal().number + number = self.cardinal.number score = rmspace + number + rmsign + insert("比") + number + rmspace tagger = insert('team: "') + (country | club) + insert('" score: "') + score + insert('"') self.tagger = self.add_tokens(tagger) diff --git a/tn/english/normalizer.py b/tn/english/normalizer.py index d4167a2d..12b94e78 100644 --- a/tn/english/normalizer.py +++ b/tn/english/normalizer.py @@ -41,70 +41,54 @@ def __init__(self, cache_dir=None, overwrite_cache=False): cache_dir = files("tn") self.build_fst("en_tn", cache_dir, overwrite_cache) - def build_tagger(self): - cardinal = add_weight(Cardinal().tagger, 1.0) - ordinal = add_weight(Ordinal().tagger, 1.0) - decimal = add_weight(Decimal().tagger, 1.0) - fraction = add_weight(Fraction().tagger, 1.0) - date = add_weight(Date().tagger, 0.99) - time = add_weight(Time().tagger, 1.00) - measure = add_weight(Measure().tagger, 1.00) - money = add_weight(Money().tagger, 1.00) - telephone = add_weight(Telephone().tagger, 1.00) - electronic = add_weight(Electronic().tagger, 1.00) - word = add_weight(Word().tagger, 100) - whitelist = add_weight(WhiteList().tagger, 1.00) - punct = add_weight(Punctuation().tagger, 2.00) - rang = add_weight(Range().tagger, 1.01) - # TODO(xcsong): add roman + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + ordinal = Ordinal(cardinal=cardinal) + decimal = Decimal(cardinal=cardinal) + fraction = Fraction(cardinal=cardinal, ordinal=ordinal) + punctuation = Punctuation() + date = Date(cardinal=cardinal, ordinal=ordinal) + time = Time(cardinal=cardinal) + measure = Measure(cardinal=cardinal, decimal=decimal, fraction=fraction, ordinal=ordinal) + money = Money(cardinal=cardinal, decimal=decimal) + telephone = Telephone() + electronic = Electronic(cardinal=cardinal) + word = Word(punctuation=punctuation) + whitelist = WhiteList() + rang = Range(date=date, time=time) + tagger = ( - cardinal - | ordinal - | word - | date - | decimal - | fraction - | time - | measure - | money - | telephone - | electronic - | whitelist - | rang - | punct - ).optimize() + (punct.plus | self.DELETE_SPACE) - # delete the first and last space + add_weight(cardinal.tagger, 1.0) + | add_weight(ordinal.tagger, 1.0) + | add_weight(word.tagger, 100) + | add_weight(date.tagger, 0.99) + | add_weight(decimal.tagger, 1.0) + | add_weight(fraction.tagger, 1.0) + | add_weight(time.tagger, 1.00) + | add_weight(measure.tagger, 1.00) + | add_weight(money.tagger, 1.00) + | add_weight(telephone.tagger, 1.00) + | add_weight(electronic.tagger, 1.00) + | add_weight(whitelist.tagger, 1.00) + | add_weight(rang.tagger, 1.01) + | add_weight(punctuation.tagger, 2.00) + ).optimize() + (add_weight(punctuation.tagger, 2.00).plus | self.DELETE_SPACE) self.tagger = (delete(" ").star + tagger.star) @ self.build_rule(delete(" "), r="[EOS]") - def build_verbalizer(self): - cardinal = Cardinal().verbalizer - ordinal = Ordinal().verbalizer - decimal = Decimal().verbalizer - fraction = Fraction().verbalizer - word = Word().verbalizer - date = Date().verbalizer - time = Time().verbalizer - measure = Measure().verbalizer - money = Money().verbalizer - telephone = Telephone().verbalizer - electronic = Electronic().verbalizer - whitelist = WhiteList().verbalizer - punct = Punctuation().verbalizer - rang = Range().verbalizer verbalizer = ( - cardinal - | ordinal - | word - | date - | decimal - | fraction - | time - | measure - | money - | telephone - | electronic - | whitelist - | punct - | rang - ).optimize() + (punct.plus | self.INSERT_SPACE) + cardinal.verbalizer + | ordinal.verbalizer + | word.verbalizer + | date.verbalizer + | decimal.verbalizer + | fraction.verbalizer + | time.verbalizer + | measure.verbalizer + | money.verbalizer + | telephone.verbalizer + | electronic.verbalizer + | whitelist.verbalizer + | punctuation.verbalizer + | rang.verbalizer + ).optimize() + (punctuation.verbalizer.plus | self.INSERT_SPACE) self.verbalizer = verbalizer.star @ self.build_rule(delete(" "), r="[EOS]") diff --git a/tn/english/rules/date.py b/tn/english/rules/date.py index f79237f2..edc603cd 100644 --- a/tn/english/rules/date.py +++ b/tn/english/rules/date.py @@ -168,14 +168,11 @@ def _get_financial_period_graph(): class Date(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None, ordinal=None): super().__init__("date", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) + self.ordinal = ordinal or Ordinal(deterministic, cardinal=self.cardinal) self.build_tagger() self.build_verbalizer() @@ -190,7 +187,7 @@ def build_tagger(self): 2012/01/05 -> date { year: "twenty twelve" month: "january" day: "five" } 2012 -> date { year: "twenty twelve" } """ - cardinal = Cardinal(self.deterministic) + cardinal = self.cardinal # january, January, JANUARY month_graph = pynini.string_file(get_abs_path("english/data/date/month_name.tsv")) # jan, Jan, JAN @@ -325,7 +322,7 @@ def build_verbalizer(self): date { month: "february" day: "five" year: "twenty twelve" } -> the fifth of february twenty twelve date { day: "five" month: "february" year: "twenty twelve" } -> the fifth of february twenty twelve """ - ordinal = Ordinal(self.deterministic) + ordinal = self.ordinal phrase = self.NOT_QUOTE.plus day_cardinal = pynutil.delete("day:") + self.DELETE_SPACE + pynutil.delete('"') + phrase + pynutil.delete('"') day = day_cardinal @ ordinal.suffix diff --git a/tn/english/rules/decimal.py b/tn/english/rules/decimal.py index 7415456a..00b7c528 100644 --- a/tn/english/rules/decimal.py +++ b/tn/english/rules/decimal.py @@ -62,14 +62,10 @@ def get_quantity( class Decimal(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None): super().__init__("decimal", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) self.build_tagger() self.build_verbalizer() @@ -79,7 +75,7 @@ def build_tagger(self): -12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" } 1 billion -> decimal { integer_part: "one" quantity: "billion" } """ - cardinal = Cardinal(deterministic=self.deterministic) + cardinal = self.cardinal cardinal_graph = cardinal.graph_with_and cardinal_graph_hundred_component_at_least_one_none_zero_digit = ( cardinal.graph_hundred_component_at_least_one_none_zero_digit @@ -146,7 +142,7 @@ def build_verbalizer(self): Finite state transducer for verbalizing decimal, e.g. decimal { negative: "true" integer_part: "twelve" fractional_part: "five o o six" quantity: "billion" } -> minus twelve point five o o six billion """ - cardinal = Cardinal(deterministic=self.deterministic) + cardinal = self.cardinal self.optional_sign = pynini.cross('negative: "true"', "minus ") if not self.deterministic: self.optional_sign |= pynutil.add_weight(pynini.cross('negative: "true"', "negative "), 0.1) diff --git a/tn/english/rules/electronic.py b/tn/english/rules/electronic.py index ef7dd89b..aa80d8e3 100644 --- a/tn/english/rules/electronic.py +++ b/tn/english/rules/electronic.py @@ -24,14 +24,10 @@ class Electronic(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None): super().__init__("electronic", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) self.build_tagger() self.build_verbalizer() @@ -40,7 +36,7 @@ def build_tagger(self): Finite state transducer for classifying electronic: as URLs, email addresses, etc. e.g. cdf1@abc.edu -> tokens { electronic { username: "cdf one" domain: "abc.edu" } } """ - cardinal = Cardinal(self.deterministic) + cardinal = self.cardinal if self.deterministic: numbers = self.DIGIT else: diff --git a/tn/english/rules/fraction.py b/tn/english/rules/fraction.py index afd4a9cd..5136a69b 100644 --- a/tn/english/rules/fraction.py +++ b/tn/english/rules/fraction.py @@ -25,14 +25,11 @@ class Fraction(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None, ordinal=None): super().__init__("fraction", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) + self.ordinal = ordinal or Ordinal(deterministic, cardinal=self.cardinal) self.build_tagger() self.build_verbalizer() @@ -44,7 +41,7 @@ def build_tagger(self): "23 4/5th" -> fraction { integer_part: "twenty three" numerator: "four" denominator: "five" } """ - cardinal_graph = Cardinal(self.deterministic).graph + cardinal_graph = self.cardinal.graph integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') numerator = ( pynutil.insert('numerator: "') + cardinal_graph + (pynini.cross("/", '" ') | pynini.cross(" / ", '" ')) @@ -72,7 +69,7 @@ def build_verbalizer(self): e.g. fraction { integer_part: "twenty three" numerator: "four" denominator: "five" } -> twenty three and four fifth """ - suffix = Ordinal(self.deterministic).suffix + suffix = self.ordinal.suffix integer = pynutil.delete('integer_part: "') + self.NOT_QUOTE.star + pynutil.delete('" ') denominator_one = pynini.cross('denominator: "one"', "over one") diff --git a/tn/english/rules/measure.py b/tn/english/rules/measure.py index 669eb117..a572417b 100644 --- a/tn/english/rules/measure.py +++ b/tn/english/rules/measure.py @@ -67,14 +67,13 @@ class Measure(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None, decimal=None, fraction=None, ordinal=None): super().__init__("measure", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) + self.ordinal = ordinal or Ordinal(deterministic, cardinal=self.cardinal) + self.decimal = decimal or Decimal(deterministic, cardinal=self.cardinal) + self.fraction = fraction or Fraction(deterministic, cardinal=self.cardinal, ordinal=self.ordinal) self.build_tagger() self.build_verbalizer() @@ -85,7 +84,7 @@ def build_tagger(self): 1kg -> measure { integer: "one" units: "kilogram" } .5kg -> measure { fractional_part: "five" units: "kilograms" } """ - cardinal = Cardinal(self.deterministic) + cardinal = self.cardinal cardinal_graph = cardinal.graph_with_and | self.get_range(cardinal.graph_with_and) graph_unit = pynini.string_file(get_abs_path("english/data/measure/unit.tsv")) @@ -113,7 +112,7 @@ def build_tagger(self): pynutil.insert(' units: "') + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') ) - decimal = Decimal(self.deterministic) + decimal = self.decimal subgraph_decimal = ( optional_graph_negative + decimal.final_graph_wo_negative + pynini.accep(" ").ques + unit_plural ) @@ -176,7 +175,7 @@ def build_tagger(self): + decimal.final_graph_wo_negative ) - fraction = Fraction(self.deterministic) + fraction = self.fraction subgraph_fraction = fraction.graph + pynini.accep(" ").ques + unit_plural address = self.get_address_graph(cardinal) @@ -250,7 +249,7 @@ def get_address_graph(self, cardinal: Processor): 2788 San Tomas Expy, Santa Clara, CA 95051 -> units: "address" integer: "two seven eight eight San Tomas Expressway Santa Clara California nine five zero five one" """ - ordinal = Ordinal(self.deterministic) + ordinal = self.ordinal ordinal_verbalizer = ordinal.graph_v ordinal_tagger = ordinal.graph ordinal_num = pynini.compose( @@ -311,7 +310,7 @@ def build_verbalizer(self): measure { negative: "true" integer: "twelve" units: "kilograms" } -> minus twelve kilograms measure { integer_part: "twelve" fractional_part: "five" units: "kilograms" } -> twelve point five kilograms """ - cardinal = Cardinal(self.deterministic) + cardinal = self.cardinal unit = ( pynutil.delete('units: "') + pynini.difference(self.NOT_QUOTE.plus, pynini.union("address", "math")) @@ -322,7 +321,7 @@ def build_verbalizer(self): if not self.deterministic: unit |= pynini.compose(unit, pynini.cross(pynini.union("inch", "inches"), '"')) - decimal = Decimal(self.deterministic) + decimal = self.decimal graph_decimal = decimal.numbers if not self.deterministic: @@ -339,7 +338,7 @@ def build_verbalizer(self): graph_cardinal = cardinal.numbers - fraction = Fraction(self.deterministic) + fraction = self.fraction graph_fraction = fraction.graph_v graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit diff --git a/tn/english/rules/money.py b/tn/english/rules/money.py index 048e5f8e..2707f198 100644 --- a/tn/english/rules/money.py +++ b/tn/english/rules/money.py @@ -27,14 +27,11 @@ class Money(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None, decimal=None): super().__init__("money", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) + self.decimal = decimal or Decimal(deterministic, cardinal=self.cardinal) self.build_tagger() self.build_verbalizer() @@ -50,8 +47,8 @@ def build_tagger(self): $1.2 million -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two" quantity: "million" } $1.2320 -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two three two" } """ - cardinal = Cardinal(self.deterministic) - decimal = Decimal(self.deterministic) + cardinal = self.cardinal + decimal = self.decimal cardinal_graph = cardinal.graph_with_and graph_decimal_final = decimal.final_graph_wo_negative_w_abbr @@ -97,7 +94,7 @@ def build_verbalizer(self): Finite state transducer for verbalizing money, e.g. money { integer_part: "twelve" fractional_part: "o five" currency: "dollars" } -> twelve o five dollars """ - decimal = Decimal(self.deterministic) + decimal = self.decimal keep_space = pynini.accep(" ") maj = pynutil.delete('currency_maj: "') + self.NOT_QUOTE.plus + pynutil.delete('"') diff --git a/tn/english/rules/ordinal.py b/tn/english/rules/ordinal.py index 7492bb02..fc63bec6 100644 --- a/tn/english/rules/ordinal.py +++ b/tn/english/rules/ordinal.py @@ -23,14 +23,10 @@ class Ordinal(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None): super().__init__("ordinal", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) self.build_tagger() self.build_verbalizer() @@ -39,8 +35,7 @@ def build_tagger(self): Finite state transducer for classifying ordinal, e.g. 13th -> ordinal { integer: "thirteen" } """ - cardinal = Cardinal(self.deterministic) - cardinal_graph = cardinal.graph + cardinal_graph = self.cardinal.graph cardinal_format = (self.DIGIT | pynini.accep(",")).star st_format = ( (cardinal_format + (self.DIGIT - "1")).ques diff --git a/tn/english/rules/range.py b/tn/english/rules/range.py index e3ebccd2..d6cbb6b8 100644 --- a/tn/english/rules/range.py +++ b/tn/english/rules/range.py @@ -25,14 +25,11 @@ class Range(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, date=None, time=None): super().__init__("range", ordertype="en_tn") self.deterministic = deterministic + self.date = date or Date(deterministic) + self.time = time or Time(deterministic) self.build_tagger() self.build_verbalizer() @@ -42,9 +39,9 @@ def build_tagger(self): 2-3 => range { value "two to three" } """ cardinal = Cardinal(deterministic=True).graph_with_and - time = Time(deterministic=self.deterministic) + time = self.time time = time.tagger @ time.verbalizer - date = Date(deterministic=self.deterministic) + date = self.date date = date.tagger @ date.verbalizer week = pynini.string_file(get_abs_path("english/data/date/week.tsv")) delete_space = pynutil.delete(" ").ques diff --git a/tn/english/rules/time.py b/tn/english/rules/time.py index 215e79b5..4ca22d99 100644 --- a/tn/english/rules/time.py +++ b/tn/english/rules/time.py @@ -23,14 +23,10 @@ class Time(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, cardinal=None): super().__init__("time", ordertype="en_tn") self.deterministic = deterministic + self.cardinal = cardinal or Cardinal(deterministic) self.build_tagger() self.build_verbalizer() @@ -53,7 +49,7 @@ def build_tagger(self): time_zone_graph = pynini.string_file(get_abs_path("english/data/time/zone.tsv")) # only used for < 1000 thousand -> 0 weight - cardinal = Cardinal(self.deterministic).graph + cardinal = self.cardinal.graph labels_hour = [str(x) for x in range(0, 24)] labels_minute_single = [str(x) for x in range(1, 10)] diff --git a/tn/english/rules/word.py b/tn/english/rules/word.py index 83e3a3d9..4cedf054 100644 --- a/tn/english/rules/word.py +++ b/tn/english/rules/word.py @@ -22,14 +22,10 @@ class Word(Processor): - def __init__(self, deterministic: bool = False): - """ - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + def __init__(self, deterministic: bool = False, punctuation=None): super().__init__("w", ordertype="en_tn") self.deterministic = deterministic + self.punctuation = punctuation or Punctuation(deterministic) self.build_tagger() self.build_verbalizer() @@ -38,7 +34,7 @@ def build_tagger(self): Finite state transducer for classifying word. Considers sentence boundary exceptions. e.g. sleep -> w { v: "sleep" } """ - punct = Punctuation(self.deterministic).graph + punct = self.punctuation.graph default_graph = difference(self.NOT_SPACE, punct.project("input")) symbols_to_exclude = union("$", "€", "₩", "£", "¥", "#", "%") | self.DIGIT self.char = difference(default_graph, symbols_to_exclude) diff --git a/tn/japanese/normalizer.py b/tn/japanese/normalizer.py index f0f295e3..fd141eff 100644 --- a/tn/japanese/normalizer.py +++ b/tn/japanese/normalizer.py @@ -53,42 +53,54 @@ def __init__( cache_dir = files("tn") self.build_fst("ja_tn", cache_dir, overwrite_cache) - def build_tagger(self): + def build_tagger_and_verbalizer(self): processor = PreProcessor(full_to_half=self.full_to_half).processor - cardinal = add_weight(Cardinal().tagger, 1.06) - char = add_weight(Char().tagger, 100) - date = add_weight(Date().tagger, 1.02) - fraction = add_weight(Fraction().tagger, 1.05) - math = add_weight(Math().tagger, 90) - measure = add_weight(Measure().tagger, 1.05) - money = add_weight(Money().tagger, 1.05) - sport = add_weight(Sport().tagger, 1.06) - time = add_weight(Time().tagger, 1.05) - whitelist = add_weight(Whitelist().tagger, 1.03) - tagger = (cardinal | char | date | fraction | math | measure | money | sport | time | whitelist).optimize() + cardinal = Cardinal() + char = Char() + date = Date(cardinal=cardinal) + fraction = Fraction(cardinal=cardinal) + math = Math(cardinal=cardinal) + measure = Measure(cardinal=cardinal) + money = Money(cardinal=cardinal) + sport = Sport(cardinal=cardinal) + time = Time() + whitelist = Whitelist() + + tagger = ( + add_weight(cardinal.tagger, 1.06) + | add_weight(char.tagger, 100) + | add_weight(date.tagger, 1.02) + | add_weight(fraction.tagger, 1.05) + | add_weight(math.tagger, 90) + | add_weight(measure.tagger, 1.05) + | add_weight(money.tagger, 1.05) + | add_weight(sport.tagger, 1.06) + | add_weight(time.tagger, 1.05) + | add_weight(whitelist.tagger, 1.03) + ).optimize() if self.transliterate: - transliteration = add_weight(Transliteration().tagger, 1.04) - tagger = (tagger | transliteration).optimize() + transliteration = Transliteration() + tagger = (tagger | add_weight(transliteration.tagger, 1.04)).optimize() tagger = (processor @ tagger).star self.tagger = tagger @ self.build_rule(delete(" "), r="[EOS]") - def build_verbalizer(self): - cardinal = Cardinal().verbalizer - char = Char().verbalizer - date = Date().verbalizer - fraction = Fraction().verbalizer - math = Math().verbalizer - measure = Measure().verbalizer - money = Money().verbalizer - sport = Sport().verbalizer - time = Time().verbalizer - transliteration = Transliteration().verbalizer - whitelist = Whitelist().verbalizer - verbalizer = (cardinal | char | date | fraction | math | measure | money | sport | time | whitelist).optimize() + transliteration = Transliteration() + verbalizer = ( + cardinal.verbalizer + | char.verbalizer + | date.verbalizer + | fraction.verbalizer + | math.verbalizer + | measure.verbalizer + | money.verbalizer + | sport.verbalizer + | time.verbalizer + | whitelist.verbalizer + ).optimize() if self.transliterate: - verbalizer = (verbalizer | transliteration).optimize() + verbalizer = (verbalizer | transliteration.verbalizer).optimize() - processor = PostProcessor( + postprocessor = PostProcessor( remove_interjections=self.remove_interjections, remove_puncts=self.remove_puncts, tag_oov=self.tag_oov ).processor - self.verbalizer = (verbalizer @ processor).star + self.verbalizer = (verbalizer @ postprocessor).star diff --git a/tn/japanese/rules/date.py b/tn/japanese/rules/date.py index b09556c6..ab9e0848 100644 --- a/tn/japanese/rules/date.py +++ b/tn/japanese/rules/date.py @@ -22,13 +22,14 @@ class Date(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="date") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): - yyyy = Cardinal().thousand + yyyy = self.cardinal.thousand m = string_file(get_abs_path("japanese/data/date/m.tsv")) mm = string_file(get_abs_path("japanese/data/date/mm.tsv")) d = string_file(get_abs_path("japanese/data/date/d.tsv")) diff --git a/tn/japanese/rules/fraction.py b/tn/japanese/rules/fraction.py index 85418b7f..51c55efc 100644 --- a/tn/japanese/rules/fraction.py +++ b/tn/japanese/rules/fraction.py @@ -20,14 +20,15 @@ class Fraction(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="fraction") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): rmspace = delete(" ").ques - number = Cardinal().number + number = self.cardinal.number tagger = ( insert('numerator: "') diff --git a/tn/japanese/rules/math.py b/tn/japanese/rules/math.py index cd640bd8..5d0fc59d 100644 --- a/tn/japanese/rules/math.py +++ b/tn/japanese/rules/math.py @@ -22,15 +22,16 @@ class Math(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="math") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() def build_tagger(self): operator = string_file(get_abs_path("japanese/data/math/operator.tsv")) - number = Cardinal().number + number = self.cardinal.number operator = number + (delete(" ").ques + operator + delete(" ").ques + number).star tagger = insert('value: "') + operator + insert('"') self.tagger = self.add_tokens(tagger) diff --git a/tn/japanese/rules/measure.py b/tn/japanese/rules/measure.py index e45a4b14..96d1b820 100644 --- a/tn/japanese/rules/measure.py +++ b/tn/japanese/rules/measure.py @@ -22,8 +22,9 @@ class Measure(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="measure") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -36,7 +37,7 @@ def build_tagger(self): rmspace = delete(" ").ques to = cross("-", "から") | cross("~", "から") | accep("から") - number = Cardinal().number + number = self.cardinal.number # 1-11月，1月-11月 prefix = number + (rmspace + units).ques + to measure = prefix.ques + number + rmspace + units diff --git a/tn/japanese/rules/money.py b/tn/japanese/rules/money.py index 3a6be8d2..1c30fea3 100644 --- a/tn/japanese/rules/money.py +++ b/tn/japanese/rules/money.py @@ -22,8 +22,9 @@ class Money(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="money") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -31,7 +32,7 @@ def build_tagger(self): code = string_file(get_abs_path("japanese/data/money/code.tsv")) symbol = string_file(get_abs_path("japanese/data/money/symbol.tsv")) - number = Cardinal().number + number = self.cardinal.number tagger = ( insert('currency: "') + (code | symbol) diff --git a/tn/japanese/rules/sport.py b/tn/japanese/rules/sport.py index 362b8385..04042e7d 100644 --- a/tn/japanese/rules/sport.py +++ b/tn/japanese/rules/sport.py @@ -22,8 +22,9 @@ class Sport(Processor): - def __init__(self): + def __init__(self, cardinal=None): super().__init__(name="sport") + self.cardinal = cardinal or Cardinal() self.build_tagger() self.build_verbalizer() @@ -33,7 +34,7 @@ def build_tagger(self): rmsign = delete("/") | delete("-") | delete(":") rmspace = delete(" ").ques - number = Cardinal().positive_integer + number = self.cardinal.positive_integer score = rmspace + number + rmsign + insert("対") + number + rmspace only_score = rmspace + number + cross(":", "対") + number + rmspace tagger = (insert('team: "') + (country | club) + insert('" score: "') + score + insert('"')) | ( diff --git a/tn/processor.py b/tn/processor.py index 735268ea..4fe5a1fc 100644 --- a/tn/processor.py +++ b/tn/processor.py @@ -93,8 +93,11 @@ def build_fst(self, prefix, cache_dir, overwrite_cache): self.verbalizer = Fst.read(verbalizer_path).optimize() else: logger.info("building fst for {} ...".format(self.name)) - self.build_tagger() - self.build_verbalizer() + if hasattr(self, 'build_tagger_and_verbalizer'): + self.build_tagger_and_verbalizer() + else: + self.build_tagger() + self.build_verbalizer() self.tagger.optimize().write(tagger_path) self.verbalizer.optimize().write(verbalizer_path) logger.info("done")