feat: add Serial rule for English TN (#259) (#354)

Add Serial rule to handle mixed alphanumeric sequences like model
numbers, product codes, and compound words with numbers:
  B2A23C => B two A twenty three C
  covid-19 => covid-nineteen
  MIG-25 => MIG-twenty five
  f++ => f plus plus
  7-eleven => seven-eleven

Author: pengzhendong

tn/english/normalizer.py

@@ -26,6 +26,7 @@
 from tn.english.rules.ordinal import Ordinal
 from tn.english.rules.punctuation import Punctuation
 from tn.english.rules.range import Range
+from tn.english.rules.serial import Serial
 from tn.english.rules.telephone import Telephone
 from tn.english.rules.time import Time
 from tn.english.rules.whitelist import WhiteList
@@ -53,6 +54,7 @@ def build_tagger_and_verbalizer(self):
         money = Money(cardinal=cardinal, decimal=decimal)
         telephone = Telephone()
         electronic = Electronic(cardinal=cardinal)
+        serial = Serial(cardinal=cardinal, ordinal=ordinal)
         word = Word(punctuation=punctuation)
         whitelist = WhiteList()
         rang = Range(date=date, time=time)
@@ -69,6 +71,7 @@ def build_tagger_and_verbalizer(self):
             | add_weight(money.tagger, 1.00)
             | add_weight(telephone.tagger, 1.00)
             | add_weight(electronic.tagger, 1.00)
+            | add_weight(serial.tagger, 1.01)
             | add_weight(whitelist.tagger, 1.00)
             | add_weight(rang.tagger, 1.01)
             | add_weight(punctuation.tagger, 2.00)
@@ -87,6 +90,7 @@ def build_tagger_and_verbalizer(self):
             | money.verbalizer
             | telephone.verbalizer
             | electronic.verbalizer
+            | serial.verbalizer
             | whitelist.verbalizer
             | punctuation.verbalizer
             | rang.verbalizer

tn/english/rules/serial.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2024 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib import pynutil
+
+from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
+from tn.processor import Processor
+from tn.utils import get_abs_path, load_labels
+
+
+class Serial(Processor):
+
+    def __init__(self, deterministic: bool = False, cardinal=None, ordinal=None):
+        super().__init__("serial", ordertype="en_tn")
+        self.deterministic = deterministic
+        self.cardinal = cardinal or Cardinal(deterministic)
+        self.ordinal = ordinal or Ordinal(deterministic, cardinal=self.cardinal)
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        cardinal = self.cardinal
+
+        num_graph = pynini.compose(self.DIGIT ** (6, ...), cardinal.single_digits_graph).optimize()
+        num_graph |= pynini.compose(self.DIGIT ** (1, 5), cardinal.graph).optimize()
+        num_graph |= pynini.compose(
+            pynini.accep("0") + pynini.closure(self.DIGIT), cardinal.single_digits_graph
+        ).optimize()
+
+        symbols_graph = pynini.string_file(get_abs_path("english/data/whitelist/symbol.tsv")).optimize()
+        symbols_graph |= pynini.cross("#", "hash")
+        num_graph |= symbols_graph
+
+        symbols = [x[0] for x in load_labels(get_abs_path("english/data/whitelist/symbol.tsv"))]
+        symbols = pynini.union(*symbols)
+        digit_symbol = self.DIGIT | symbols
+
+        graph_with_space = pynini.compose(
+            pynini.cdrewrite(pynutil.insert(" "), self.ALPHA | symbols, digit_symbol, self.VSIGMA),
+            pynini.cdrewrite(pynutil.insert(" "), digit_symbol, self.ALPHA | symbols, self.VSIGMA),
+        )
+
+        delimiter = pynini.accep("-") | pynini.accep("/") | pynini.accep(" ")
+
+        alphas = pynini.closure(self.ALPHA, 1)
+        letter_num = alphas + delimiter + num_graph
+        num_letter = pynini.closure(num_graph + delimiter, 1) + alphas
+        next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph))
+
+        serial_graph = letter_num + next_alpha_or_num
+        serial_graph |= num_letter + next_alpha_or_num
+        serial_graph |= (
+            num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph)
+        )
+
+        serial_graph = pynini.compose(
+            pynini.difference(self.VSIGMA, pynini.project(self.ordinal.graph, "input")), serial_graph
+        ).optimize()
+
+        serial_graph |= (
+            pynini.closure(self.NOT_SPACE, 1)
+            + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize()
+        )
+
+        serial_graph = (
+            pynini.closure((serial_graph | num_graph | alphas) + delimiter)
+            + serial_graph
+            + pynini.closure(delimiter + (serial_graph | num_graph | alphas))
+        )
+
+        serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize()
+        serial_graph = pynini.compose(pynini.closure(self.NOT_SPACE, 2), serial_graph).optimize()
+
+        serial_graph = pynini.compose(
+            pynini.difference(
+                self.VSIGMA, pynini.closure(self.ALPHA, 1) + pynini.accep("/") + pynini.closure(self.ALPHA, 1)
+            ),
+            serial_graph,
+        )
+
+        self.graph = serial_graph.optimize()
+        graph = pynutil.insert('name: "') + self.graph + pynutil.insert('"')
+        self.tagger = self.add_tokens(graph)
+
+    def build_verbalizer(self):
+        graph = (
+            pynutil.delete("name:")
+            + self.DELETE_SPACE
+            + pynutil.delete('"')
+            + self.NOT_QUOTE.plus
+            + pynutil.delete('"')
+        )
+        self.verbalizer = self.delete_tokens(graph)