feat: add English ITN core rules (cardinal, ordinal, decimal)

pengzhendong · pengzhendong · commit 567d61bd71b2 · 2026-06-09T21:02:58.000+08:00
Add core English inverse text normalization rules:
- Cardinal: supports up to sextillion, eleven hundred pattern,
  minus/negative, and/deletion
- Ordinal: first=&gt;1st, second=&gt;2nd, thirteenth=&gt;13th, etc.
- Decimal: twelve point five =&gt; 12.5, point o five =&gt; .05
- Char: fallback for unmatched characters

NeMo cardinal test: 27/29 passed (missing: zero preservation,
Indian numeric format)
diff --git a/itn/english/rules/cardinal.py b/itn/english/rules/cardinal.py
@@ -0,0 +1,127 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini import closure, cross, string_file, union
+from pynini.lib.pynutil import delete, insert
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Cardinal(Processor):
+
+    def __init__(self):
+        super().__init__(name="cardinal", ordertype="itn")
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
+        digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
+        teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
+        ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
+        ds = delete(" ")
+
+        # 1~9, 10~19, 20~99
+        one_digit = digit
+        two_digit = teen | (ties + (ds + digit | insert("0")))
+        up_to_99 = one_digit | two_digit
+
+        # one hundred, one hundred twenty three, one hundred one
+        hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + one_digit | insert("00"))
+        # eleven hundred => 1100, twenty one hundred => 2100
+        hundreds_as_thousand = (teen | ties + ds + digit) + ds + delete("hundred") + (
+            ds + two_digit | ds + insert("0") + one_digit | insert("00")
+        )
+
+        # 1~999
+        up_to_999 = up_to_99 | hundreds
+        # 1~999 with zero-padding to 3 digits
+        up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + one_digit
+
+        def _with_mag(name):
+            return up_to_999 + ds + delete(name)
+
+        def _with_mag_padded(name):
+            return up_to_999_padded + ds + delete(name)
+
+        graph = zero | up_to_999 | hundreds_as_thousand
+        graph |= _with_mag("thousand") + (ds + up_to_999_padded | insert("000"))
+        graph |= (
+            _with_mag("million")
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+        graph |= (
+            _with_mag("billion")
+            + (ds + _with_mag_padded("million") | insert("000"))
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+        graph |= (
+            _with_mag("trillion")
+            + (ds + _with_mag_padded("billion") | insert("000"))
+            + (ds + _with_mag_padded("million") | insert("000"))
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+        graph |= (
+            _with_mag("quadrillion")
+            + (ds + _with_mag_padded("trillion") | insert("000"))
+            + (ds + _with_mag_padded("billion") | insert("000"))
+            + (ds + _with_mag_padded("million") | insert("000"))
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+        graph |= (
+            _with_mag("quintillion")
+            + (ds + _with_mag_padded("quadrillion") | insert("000"))
+            + (ds + _with_mag_padded("trillion") | insert("000"))
+            + (ds + _with_mag_padded("billion") | insert("000"))
+            + (ds + _with_mag_padded("million") | insert("000"))
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+        graph |= (
+            _with_mag("sextillion")
+            + (ds + _with_mag_padded("quintillion") | insert("000"))
+            + (ds + _with_mag_padded("quadrillion") | insert("000"))
+            + (ds + _with_mag_padded("trillion") | insert("000"))
+            + (ds + _with_mag_padded("billion") | insert("000"))
+            + (ds + _with_mag_padded("million") | insert("000"))
+            + (ds + _with_mag_padded("thousand") | insert("000"))
+            + (ds + up_to_999_padded | insert("000"))
+        )
+
+        # strip leading zeros
+        graph = graph @ union(delete(closure("0")) + (self.DIGIT - "0") + closure(self.DIGIT), "0")
+        # delete optional "and"
+        delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
+        graph = (delete_and @ graph).optimize()
+
+        self.graph = graph
+
+        minus = delete("minus") | delete("negative")
+        optional_minus = closure(insert('negative: "-" ') + minus + ds, 0, 1)
+        final_graph = optional_minus + insert('integer: "') + graph + insert('"')
+        self.tagger = self.add_tokens(final_graph)
+
+    def build_verbalizer(self):
+        optional_sign = closure(
+            delete('negative:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE + delete('"') + self.DELETE_SPACE,
+            0, 1,
+        )
+        integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
+        self.numbers = integer
+        self.verbalizer = self.delete_tokens(optional_sign + integer)
diff --git a/itn/english/rules/char.py b/itn/english/rules/char.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini.lib.pynutil import insert
+
+from tn.processor import Processor
+
+
+class Char(Processor):
+
+    def __init__(self):
+        super().__init__(name="char", ordertype="itn")
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        tagger = insert('value: "') + self.CHAR + insert('"')
+        self.tagger = self.add_tokens(tagger)
diff --git a/itn/english/rules/decimal.py b/itn/english/rules/decimal.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini import closure, cross, string_file
+from pynini.lib.pynutil import delete, insert
+
+from itn.english.rules.cardinal import Cardinal
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Decimal(Processor):
+
+    def __init__(self, cardinal=None):
+        super().__init__(name="decimal", ordertype="itn")
+        self.cardinal = cardinal or Cardinal()
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
+        zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
+        ds = delete(" ")
+
+        # fractional part: digit by digit, "o" => 0
+        frac_digit = digit | zero | cross("o", "0")
+        frac_graph = closure(frac_digit + ds) + frac_digit
+
+        optional_negative = closure(
+            insert('negative: "true" ') + delete("minus") + ds, 0, 1
+        )
+        integer_part = insert('integer_part: "') + self.cardinal.graph + insert('"')
+        frac_part = insert('fractional_part: "') + frac_graph + insert('"')
+        point = delete("point")
+
+        graph = optional_negative + closure(integer_part + ds, 0, 1) + point + ds + frac_part
+        self.tagger = self.add_tokens(graph)
+
+    def build_verbalizer(self):
+        optional_sign = closure(cross('negative: "true"', "-") + self.DELETE_SPACE, 0, 1)
+        integer = delete('integer_part:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
+        optional_integer = closure(integer + self.DELETE_SPACE, 0, 1)
+        fractional = (
+            insert(".") + delete('fractional_part:') + self.DELETE_SPACE
+            + delete('"') + self.NOT_QUOTE.plus + delete('"')
+        )
+        optional_fractional = closure(fractional + self.DELETE_SPACE, 0, 1)
+        graph = optional_sign + optional_integer + optional_fractional
+        self.numbers = graph
+        self.verbalizer = self.delete_tokens(graph)
diff --git a/itn/english/rules/inverse_normalizer.py b/itn/english/rules/inverse_normalizer.py
@@ -0,0 +1,56 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from importlib_resources import files
+from pynini.lib.pynutil import add_weight, delete
+
+from itn.english.rules.cardinal import Cardinal
+from itn.english.rules.char import Char
+from itn.english.rules.decimal import Decimal
+from itn.english.rules.ordinal import Ordinal
+from tn.processor import Processor
+
+
+class InverseNormalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name="en_inverse_normalizer", ordertype="itn")
+        if cache_dir is None:
+            cache_dir = files("itn")
+        self.build_fst("en_itn", cache_dir, overwrite_cache)
+
+    def build_tagger_and_verbalizer(self):
+        cardinal = Cardinal()
+        ordinal = Ordinal(cardinal=cardinal)
+        decimal = Decimal(cardinal=cardinal)
+        char = Char()
+
+        tagger = (
+            add_weight(ordinal.tagger, 1.0)
+            | add_weight(decimal.tagger, 1.01)
+            | add_weight(cardinal.tagger, 1.02)
+            | add_weight(char.tagger, 100)
+        ).optimize()
+
+        tagger = tagger.star
+        self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]")
+
+        verbalizer = (
+            cardinal.verbalizer
+            | ordinal.verbalizer
+            | decimal.verbalizer
+            | char.verbalizer
+        ).optimize()
+
+        self.verbalizer = verbalizer.star
diff --git a/itn/english/rules/ordinal.py b/itn/english/rules/ordinal.py
@@ -0,0 +1,52 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini import closure, cross, string_file
+from pynini.lib.pynutil import delete, insert
+
+from itn.english.rules.cardinal import Cardinal
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Ordinal(Processor):
+
+    def __init__(self, cardinal=None):
+        super().__init__(name="ordinal", ordertype="itn")
+        self.cardinal = cardinal or Cardinal()
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        graph_digit = string_file(get_abs_path("../itn/english/data/ordinals/digit.tsv"))
+        graph_teen = string_file(get_abs_path("../itn/english/data/ordinals/teen.tsv"))
+        # first => one => 1, twelfth => twelve => 12, twentieth => twenty => 2(0)
+        suffix = graph_digit | graph_teen | cross("tieth", "ty") | cross("th", "")
+        graph = closure(self.VCHAR) + suffix
+        self.graph = pynini.compose(graph, self.cardinal.graph)
+
+        final_graph = insert('integer: "') + self.graph + insert('"')
+        self.tagger = self.add_tokens(final_graph)
+
+    def build_verbalizer(self):
+        integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
+        # 1 => 1st, 2 => 2nd, 3 => 3rd, 11 => 11th, 12 => 12th, 13 => 13th, rest => Xth
+        ordinal_suffix = (
+            cross("11", "11th") | cross("12", "12th") | cross("13", "13th")
+            | cross("1", "1st") | cross("2", "2nd") | cross("3", "3rd")
+            | insert("th", weight=0.01)
+        )
+        graph = integer @ pynini.cdrewrite(ordinal_suffix, "", "[EOS]", self.VSIGMA)
+        self.verbalizer = self.delete_tokens(graph)
