diff --git a/itn/english/rules/inverse_normalizer.py b/itn/english/inverse_normalizer.py similarity index 100% rename from itn/english/rules/inverse_normalizer.py rename to itn/english/inverse_normalizer.py diff --git a/itn/english/rules/date.py b/itn/english/rules/date.py new file mode 100644 index 0000000..42dc46d --- /dev/null +++ b/itn/english/rules/date.py @@ -0,0 +1,165 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini import closure, cross, string_file +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.english.rules.cardinal import Cardinal +from itn.english.rules.ordinal import Ordinal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Date(Processor): + + def __init__(self, cardinal=None, ordinal=None): + super().__init__(name="date", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.ordinal = ordinal or Ordinal(cardinal=self.cardinal) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + ds = delete(" ") + month_names = string_file(get_abs_path("../itn/english/data/months.tsv")) + month = insert('month: "') + month_names + insert('"') + + # Day: accept ordinal words ("fifth", "twenty first") or cardinal + # words ("thirty") -- both resolve to a number via the cardinal graph. + # Restrict to 1-31 range via composition with DIGIT{1,2}. + day_graph = self.ordinal.graph | self.cardinal.graph + day_graph = pynini.compose(day_graph, self.DIGIT + closure(self.DIGIT, 0, 1)) + day = insert('day: "') + day_graph + insert('"') + + # Year graph: handles common spoken year forms + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv")) + ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv")) + + # Two-digit: 10-99 + two_digit = teen | (ties + (ds + digit | insert("0"))) + + # "oh five" / "o five" => 05 + oh_digit = (cross("oh", "0") | cross("o", "0")) + ds + digit + + # Year as two groups of two digits: "twenty twelve" => 2012 + year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen) + + # Year as "X thousand Y": "two thousand twelve" => 2012 + # Need zero-padded variants so "two thousand three" => 2003 + hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00")) + up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit + year_thousands = ( + digit + + ds + + delete("thousand") + + (ds + up_to_999_padded | insert("000")) + ) + + # Year as hundreds: "nineteen oh five" => 1905 + year_hundreds = (teen | two_digit) + ds + oh_digit + + year_graph = year_two_parts | year_thousands | year_hundreds + + # Delete optional "and" within year + delete_and = self.build_rule(delete("and "), " ", self.ALPHA) + year_graph = (delete_and @ year_graph).optimize() + + year = insert('year: "') + year_graph + insert('"') + + # Marker to preserve field order through TokenParser + po = insert(' preserve_order: "true"') + + # Format: month day year => "july twenty fifth two thousand twelve" + graph_mdy = month + ds + insert(" ") + day + ds + insert(" ") + year + po + # Format: month day (no year) => "january first" + graph_md = month + ds + insert(" ") + day + po + # Format: month year (no day) => "july two thousand twelve" + graph_my = month + ds + insert(" ") + add_weight(year, -0.1) + po + # Format: "the day of month year" => "the twenty fifth of july twenty twelve" + graph_dmy = ( + delete("the") + + ds + + day + + ds + + delete("of") + + ds + + insert(" ") + + month + + ds + + insert(" ") + + year + + po + ) + # Format: "the day of month" (no year) => "the fifteenth of january" + graph_dm = ( + delete("the") + + ds + + day + + ds + + delete("of") + + ds + + insert(" ") + + month + + po + ) + # Year only => "twenty twelve", "two thousand three" + graph_y = add_weight(year, 0.01) + po + + final_graph = graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + month = ( + delete("month:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + day = ( + delete("day:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + year = ( + delete("year:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + delete_po = ( + delete("preserve_order:") + + self.DELETE_SPACE + + delete('"') + + delete("true") + + delete('"') + ) + + optional_day = closure(self.DELETE_SPACE + insert(" ") + day, 0, 1) + optional_year = closure(self.DELETE_SPACE + insert(" ") + year, 0, 1) + + # month (day) (year) + graph_mdy = month + optional_day + optional_year + # day month (year) + graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year + # year only + graph_y = year + + graph = (graph_mdy | graph_dmy | graph_y) + self.DELETE_SPACE + delete_po + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/rules/electronic.py b/itn/english/rules/electronic.py new file mode 100644 index 0000000..46ca328 --- /dev/null +++ b/itn/english/rules/electronic.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import closure, cross, invert, string_file +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Electronic(Processor): + + def __init__(self): + super().__init__(name="electronic", ordertype="itn") + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + ds = delete(" ") + + # Single characters: digits and letters + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv")) + alpha_or_digit = self.ALPHA | digit | zero + + # Symbols from TSV (symbol\tname): invert to get name -> symbol + symbols = invert( + string_file(get_abs_path("../itn/english/data/electronic/symbols.tsv")) + ) + + # A "token" is either a single char (letter/digit/symbol) or a + # multi-letter word kept verbatim (e.g. "gmail", "nvidia"). + # Multi-letter words have lower priority so spelled-out letters are preferred. + word = add_weight(closure(self.ALPHA, 2), 0.01) + token = alpha_or_digit | symbols | word + + # A component is one or more tokens separated by spaces + component = token + closure(ds + token) + + username = insert('username: "') + component + insert('"') + + # Domain: component(s) separated by "dot" => "." + dot = cross("dot", ".") + domain_content = component + closure(ds + dot + ds + component) + domain = insert('domain: "') + domain_content + insert('"') + + # Email: username at domain + graph_email = ( + username + + ds + + delete("at") + + ds + + insert(" ") + + domain + ) + + # URL protocol: "h t t p colon slash slash" or "h t t p s colon slash slash" + http = cross("h t t p", "http") + https = cross("h t t p s", "https") + colon_slash_slash = cross(" colon slash slash ", "://") + protocol_start = (http | https) + colon_slash_slash + + # www prefix + www = cross("w w w", "www") + + # URL: [protocol] [www.] domain + url_content = ( + closure(protocol_start, 0, 1) + + closure(www + ds + dot + ds, 0, 1) + + domain_content + ) + graph_url = insert('protocol: "') + url_content + insert('"') + + final_graph = graph_email | graph_url + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + username = ( + delete("username:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + domain = ( + delete("domain:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + protocol = ( + delete("protocol:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + + # Email: username@domain + graph_email = username + self.DELETE_SPACE + insert("@") + domain + # URL: just output the protocol content directly + graph_url = protocol + + graph = graph_email | graph_url + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/rules/measure.py b/itn/english/rules/measure.py new file mode 100644 index 0000000..909c799 --- /dev/null +++ b/itn/english/rules/measure.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini import closure, cross, invert, string_file +from pynini.lib.pynutil import delete, insert + +from itn.english.rules.cardinal import Cardinal +from itn.english.rules.decimal import Decimal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Measure(Processor): + + def __init__(self, cardinal=None, decimal=None): + super().__init__(name="measure", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.decimal = decimal or Decimal(cardinal=self.cardinal) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + ds = delete(" ") + + # Load measurements: symbol\tname, invert to get name -> symbol + units_graph = invert( + string_file(get_abs_path("../itn/english/data/measurements.tsv")) + ) + + # Handle plurals: strip trailing "s" to match singular form + # e.g. "meters" -> "meter" -> "m", "kilograms" -> "kilogram" -> "kg" + depluralize = pynini.cdrewrite( + cross("s", ""), "", "[EOS]", self.VSIGMA + ) + # Handle irregular plurals: "feet" -> "foot" + irregular = pynini.string_map([("feet", "foot")]) + unit_singular = units_graph + unit_plural = (depluralize | irregular) @ units_graph + + unit = unit_singular | unit_plural + + # Handle "per" units: "per hour" -> "/h" + per_unit = insert("/") + delete("per") + ds + unit_singular + full_unit = unit + closure(ds + per_unit, 0, 1) | per_unit + + # Cardinal value + cardinal_value = self.cardinal.graph + + # Decimal value (reuse decimal's internal graph for the number) + decimal_digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + decimal_zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv")) + frac_digit = decimal_digit | decimal_zero | cross("o", "0") + frac_graph = closure(frac_digit + ds) + frac_digit + decimal_value = cardinal_value + ds + delete("point") + ds + insert(".") + frac_graph + + # Optional minus/negative prefix + minus = delete("minus") | delete("negative") + optional_sign = closure(insert("-") + minus + ds, 0, 1) + + # "point X" with no integer part + point_only = delete("point") + ds + insert(".") + frac_graph + + number = optional_sign + (decimal_value | cardinal_value | point_only) + + value = insert('value: "') + number + insert('"') + units = insert('units: "') + full_unit + insert('"') + + final_graph = value + ds + insert(" ") + units + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + value = ( + delete("value:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + units = ( + delete("units:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + graph = value + self.DELETE_SPACE + insert(" ") + units + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/rules/telephone.py b/itn/english/rules/telephone.py new file mode 100644 index 0000000..0ce5f46 --- /dev/null +++ b/itn/english/rules/telephone.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import closure, cross, string_file +from pynini.lib.pynutil import delete, insert + +from itn.english.rules.cardinal import Cardinal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Telephone(Processor): + + def __init__(self, cardinal=None): + super().__init__(name="telephone", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + ds = delete(" ") + + # Single digit: spoken word -> digit character + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv")) + single_digit = digit | zero | cross("o", "0") | cross("oh", "0") + + # 10 digits formatted as XXX-XXX-XXXX + ten_digits = ( + single_digit + ds + single_digit + ds + single_digit + + insert("-") + + ds + single_digit + ds + single_digit + ds + single_digit + + insert("-") + + ds + single_digit + ds + single_digit + ds + single_digit + ds + single_digit + ) + + # Optional country code: "plus X" or just digits before the main number + country_code_digits = ( + closure(single_digit + ds, 0, 2) + single_digit + ) + country_code = ( + closure(cross("plus ", "+"), 0, 1) + country_code_digits + ) + optional_country_code = closure( + country_code + insert(" ") + ds, 0, 1 + ) + + graph = optional_country_code + ten_digits + final_graph = insert('value: "') + graph + insert('"') + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + value = ( + delete("value:") + + self.DELETE_SPACE + + delete('"') + + self.NOT_QUOTE.plus + + delete('"') + ) + self.verbalizer = self.delete_tokens(value) diff --git a/itn/english/rules/time.py b/itn/english/rules/time.py new file mode 100644 index 0000000..dbdea08 --- /dev/null +++ b/itn/english/rules/time.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import closure, cross, string_file +from pynini.lib.pynutil import delete, insert + +from itn.english.rules.cardinal import Cardinal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Time(Processor): + + def __init__(self, cardinal=None): + super().__init__(name="time", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv")) + ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv")) + time_suffix = string_file(get_abs_path("../itn/english/data/time/time_suffix.tsv")) + time_zone = string_file(get_abs_path("../itn/english/data/time/time_zone.tsv")) + ds = delete(" ") + + hour = teen | (insert("0") + digit) + minute = teen | (ties + (ds + digit | insert("0"))) | insert("0") + digit + + # two thirty => 02:30 + graph = insert('hour: "') + hour + insert('" ') + ds + insert('minute: "') + minute + insert('"') + # eight oclock => 08:00 + oclock = cross("o'clock", "") | cross("oclock", "") + graph |= insert('hour: "') + hour + insert('" minute: "00"') + ds + oclock + + suffix = ds + insert(' noon: "') + time_suffix + insert('"') + zone = ds + insert(' zone: "') + time_zone + insert('"') + graph += suffix.ques + zone.ques + + self.tagger = self.add_tokens(graph) + + def build_verbalizer(self): + hours = delete('hour: "') + self.NOT_QUOTE.plus + delete('"') + minutes = delete(' minute: "') + self.NOT_QUOTE.plus + delete('"') + suffix = delete(' noon: "') + self.NOT_QUOTE.plus + delete('"') + zone = delete(' zone: "') + self.NOT_QUOTE.plus + delete('"') + graph = hours + insert(":") + self.DELETE_SPACE + minutes + graph += closure(insert(" ") + self.DELETE_SPACE + suffix, 0, 1) + graph += closure(insert(" ") + self.DELETE_SPACE + zone, 0, 1) + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/rules/whitelist.py b/itn/english/rules/whitelist.py new file mode 100644 index 0000000..eaab762 --- /dev/null +++ b/itn/english/rules/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import invert, string_file +from pynini.lib.pynutil import insert + +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Whitelist(Processor): + + def __init__(self): + super().__init__(name="whitelist", ordertype="itn") + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + whitelist = invert(string_file(get_abs_path("../itn/english/data/whitelist.tsv"))) + tagger = insert('value: "') + whitelist + insert('"') + self.tagger = self.add_tokens(tagger) diff --git a/itn/english/test/data/en_date.txt b/itn/english/test/data/en_date.txt new file mode 100644 index 0000000..b0aacec --- /dev/null +++ b/itn/english/test/data/en_date.txt @@ -0,0 +1,17 @@ +july twenty fifth two thousand twelve => july 25 2012 +the twenty fifth of july twenty twelve => 25 july 2012 +the twenty fifth of july two thousand twelve => 25 july 2012 +the twenty second of july twenty twelve => 22 july 2012 +the fifteenth of january => 15 january +january first => january 1 +july twenty second two thousand eight => july 22 2008 +june thirty => june 30 +july twenty fifth twenty twelve => july 25 2012 +twenty twelve => 2012 +july two thousand twelve => july 2012 +october nineteen oh five => october 1905 +february twenty fifth twenty sixteen => february 25 2016 +november twenty fourth twenty fourteen => november 24 2014 +two thousand and three => 2003 +two thousand and twenty => 2020 +nineteen seventy six => 1976 diff --git a/itn/english/test/data/en_electronic.txt b/itn/english/test/data/en_electronic.txt new file mode 100644 index 0000000..a296c65 --- /dev/null +++ b/itn/english/test/data/en_electronic.txt @@ -0,0 +1,5 @@ +a at gmail dot com => a@gmail.com +c d f at a b c dot e d u => cdf@abc.edu +a b c at a b c dot com => abc@abc.com +a b c at g mail dot a b c => abc@gmail.abc +a dot b c at nvidia dot com => a.bc@nvidia.com diff --git a/itn/english/test/data/en_measure.txt b/itn/english/test/data/en_measure.txt new file mode 100644 index 0000000..9b3aa65 --- /dev/null +++ b/itn/english/test/data/en_measure.txt @@ -0,0 +1,14 @@ +two hundred meters => 200 m +two hundred kilometers per hour => 200 km/h +minus sixty six kilograms => -66 kg +three hours => 3 h +one milli volt => 1 mv +ninety grams => 90 g +eight kilograms => 8 kg +eighteen feet => 18 ft +eighteen ounces => 18 oz +eight hundred kilowatts => 800 kW +eight hundred horsepower => 800 hp +fifty six point three per square kilometer => 56.3 /km² +twelve point five meters => 12.5 m +point two meters => .2 m diff --git a/itn/english/test/data/en_telephone.txt b/itn/english/test/data/en_telephone.txt new file mode 100644 index 0000000..f1ef0e2 --- /dev/null +++ b/itn/english/test/data/en_telephone.txt @@ -0,0 +1,3 @@ +one two three one two three five six seven eight => 123-123-5678 +one two three four five six seven eight nine zero => 123-456-7890 +plus one one two three one two three five six seven eight => +1 123-123-5678 diff --git a/itn/english/test/normalizer_test.py b/itn/english/test/normalizer_test.py index 4fb0756..507ead3 100644 --- a/itn/english/test/normalizer_test.py +++ b/itn/english/test/normalizer_test.py @@ -16,7 +16,7 @@ import pytest -from itn.english.rules.inverse_normalizer import InverseNormalizer +from itn.english.inverse_normalizer import InverseNormalizer from itn.english.test.utils import parse_test_case