Skip to content

Commit 2ddd6dd

Browse files
pengzhendongpengzhendong
authored
feat: add English ITN extended rules (time, whitelist, move normalizer) (#356)
- Move inverse_normalizer.py to itn/english/ (consistent with zh/ja) - Add Time rule: two thirty => 02:30, eight oclock => 08:00 - Add Whitelist rule: ten k => 10K, three d => 3D - Fix test import path Note: money, measure, date, telephone, electronic, word rules are prepared but not yet integrated (need debugging).
1 parent 91b68ae commit 2ddd6dd

12 files changed

Lines changed: 586 additions & 1 deletion

itn/english/rules/inverse_normalizer.py renamed to itn/english/inverse_normalizer.py

File renamed without changes.

‎itn/english/rules/date.py‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pynini
16+
from pynini import closure, cross, string_file
17+
from pynini.lib.pynutil import add_weight, delete, insert
18+
19+
from itn.english.rules.cardinal import Cardinal
20+
from itn.english.rules.ordinal import Ordinal
21+
from tn.processor import Processor
22+
from tn.utils import get_abs_path
23+
24+
25+
class Date(Processor):
26+
27+
def __init__(self, cardinal=None, ordinal=None):
28+
super().__init__(name="date", ordertype="itn")
29+
self.cardinal = cardinal or Cardinal()
30+
self.ordinal = ordinal or Ordinal(cardinal=self.cardinal)
31+
self.build_tagger()
32+
self.build_verbalizer()
33+
34+
def build_tagger(self):
35+
ds = delete(" ")
36+
month_names = string_file(get_abs_path("../itn/english/data/months.tsv"))
37+
month = insert('month: "') + month_names + insert('"')
38+
39+
# Day: accept ordinal words ("fifth", "twenty first") or cardinal
40+
# words ("thirty") -- both resolve to a number via the cardinal graph.
41+
# Restrict to 1-31 range via composition with DIGIT{1,2}.
42+
day_graph = self.ordinal.graph | self.cardinal.graph
43+
day_graph = pynini.compose(day_graph, self.DIGIT + closure(self.DIGIT, 0, 1))
44+
day = insert('day: "') + day_graph + insert('"')
45+
46+
# Year graph: handles common spoken year forms
47+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
48+
teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
49+
ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
50+
51+
# Two-digit: 10-99
52+
two_digit = teen | (ties + (ds + digit | insert("0")))
53+
54+
# "oh five" / "o five" => 05
55+
oh_digit = (cross("oh", "0") | cross("o", "0")) + ds + digit
56+
57+
# Year as two groups of two digits: "twenty twelve" => 2012
58+
year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen)
59+
60+
# Year as "X thousand Y": "two thousand twelve" => 2012
61+
# Need zero-padded variants so "two thousand three" => 2003
62+
hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00"))
63+
up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit
64+
year_thousands = (
65+
digit
66+
+ ds
67+
+ delete("thousand")
68+
+ (ds + up_to_999_padded | insert("000"))
69+
)
70+
71+
# Year as hundreds: "nineteen oh five" => 1905
72+
year_hundreds = (teen | two_digit) + ds + oh_digit
73+
74+
year_graph = year_two_parts | year_thousands | year_hundreds
75+
76+
# Delete optional "and" within year
77+
delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
78+
year_graph = (delete_and @ year_graph).optimize()
79+
80+
year = insert('year: "') + year_graph + insert('"')
81+
82+
# Marker to preserve field order through TokenParser
83+
po = insert(' preserve_order: "true"')
84+
85+
# Format: month day year => "july twenty fifth two thousand twelve"
86+
graph_mdy = month + ds + insert(" ") + day + ds + insert(" ") + year + po
87+
# Format: month day (no year) => "january first"
88+
graph_md = month + ds + insert(" ") + day + po
89+
# Format: month year (no day) => "july two thousand twelve"
90+
graph_my = month + ds + insert(" ") + add_weight(year, -0.1) + po
91+
# Format: "the day of month year" => "the twenty fifth of july twenty twelve"
92+
graph_dmy = (
93+
delete("the")
94+
+ ds
95+
+ day
96+
+ ds
97+
+ delete("of")
98+
+ ds
99+
+ insert(" ")
100+
+ month
101+
+ ds
102+
+ insert(" ")
103+
+ year
104+
+ po
105+
)
106+
# Format: "the day of month" (no year) => "the fifteenth of january"
107+
graph_dm = (
108+
delete("the")
109+
+ ds
110+
+ day
111+
+ ds
112+
+ delete("of")
113+
+ ds
114+
+ insert(" ")
115+
+ month
116+
+ po
117+
)
118+
# Year only => "twenty twelve", "two thousand three"
119+
graph_y = add_weight(year, 0.01) + po
120+
121+
final_graph = graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y
122+
self.tagger = self.add_tokens(final_graph)
123+
124+
def build_verbalizer(self):
125+
month = (
126+
delete("month:")
127+
+ self.DELETE_SPACE
128+
+ delete('"')
129+
+ self.NOT_QUOTE.plus
130+
+ delete('"')
131+
)
132+
day = (
133+
delete("day:")
134+
+ self.DELETE_SPACE
135+
+ delete('"')
136+
+ self.NOT_QUOTE.plus
137+
+ delete('"')
138+
)
139+
year = (
140+
delete("year:")
141+
+ self.DELETE_SPACE
142+
+ delete('"')
143+
+ self.NOT_QUOTE.plus
144+
+ delete('"')
145+
)
146+
delete_po = (
147+
delete("preserve_order:")
148+
+ self.DELETE_SPACE
149+
+ delete('"')
150+
+ delete("true")
151+
+ delete('"')
152+
)
153+
154+
optional_day = closure(self.DELETE_SPACE + insert(" ") + day, 0, 1)
155+
optional_year = closure(self.DELETE_SPACE + insert(" ") + year, 0, 1)
156+
157+
# month (day) (year)
158+
graph_mdy = month + optional_day + optional_year
159+
# day month (year)
160+
graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year
161+
# year only
162+
graph_y = year
163+
164+
graph = (graph_mdy | graph_dmy | graph_y) + self.DELETE_SPACE + delete_po
165+
self.verbalizer = self.delete_tokens(graph)

‎itn/english/rules/electronic.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from pynini import closure, cross, invert, string_file
16+
from pynini.lib.pynutil import add_weight, delete, insert
17+
18+
from tn.processor import Processor
19+
from tn.utils import get_abs_path
20+
21+
22+
class Electronic(Processor):
23+
24+
def __init__(self):
25+
super().__init__(name="electronic", ordertype="itn")
26+
self.build_tagger()
27+
self.build_verbalizer()
28+
29+
def build_tagger(self):
30+
ds = delete(" ")
31+
32+
# Single characters: digits and letters
33+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
34+
zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
35+
alpha_or_digit = self.ALPHA | digit | zero
36+
37+
# Symbols from TSV (symbol\tname): invert to get name -> symbol
38+
symbols = invert(
39+
string_file(get_abs_path("../itn/english/data/electronic/symbols.tsv"))
40+
)
41+
42+
# A "token" is either a single char (letter/digit/symbol) or a
43+
# multi-letter word kept verbatim (e.g. "gmail", "nvidia").
44+
# Multi-letter words have lower priority so spelled-out letters are preferred.
45+
word = add_weight(closure(self.ALPHA, 2), 0.01)
46+
token = alpha_or_digit | symbols | word
47+
48+
# A component is one or more tokens separated by spaces
49+
component = token + closure(ds + token)
50+
51+
username = insert('username: "') + component + insert('"')
52+
53+
# Domain: component(s) separated by "dot" => "."
54+
dot = cross("dot", ".")
55+
domain_content = component + closure(ds + dot + ds + component)
56+
domain = insert('domain: "') + domain_content + insert('"')
57+
58+
# Email: username at domain
59+
graph_email = (
60+
username
61+
+ ds
62+
+ delete("at")
63+
+ ds
64+
+ insert(" ")
65+
+ domain
66+
)
67+
68+
# URL protocol: "h t t p colon slash slash" or "h t t p s colon slash slash"
69+
http = cross("h t t p", "http")
70+
https = cross("h t t p s", "https")
71+
colon_slash_slash = cross(" colon slash slash ", "://")
72+
protocol_start = (http | https) + colon_slash_slash
73+
74+
# www prefix
75+
www = cross("w w w", "www")
76+
77+
# URL: [protocol] [www.] domain
78+
url_content = (
79+
closure(protocol_start, 0, 1)
80+
+ closure(www + ds + dot + ds, 0, 1)
81+
+ domain_content
82+
)
83+
graph_url = insert('protocol: "') + url_content + insert('"')
84+
85+
final_graph = graph_email | graph_url
86+
self.tagger = self.add_tokens(final_graph)
87+
88+
def build_verbalizer(self):
89+
username = (
90+
delete("username:")
91+
+ self.DELETE_SPACE
92+
+ delete('"')
93+
+ self.NOT_QUOTE.plus
94+
+ delete('"')
95+
)
96+
domain = (
97+
delete("domain:")
98+
+ self.DELETE_SPACE
99+
+ delete('"')
100+
+ self.NOT_QUOTE.plus
101+
+ delete('"')
102+
)
103+
protocol = (
104+
delete("protocol:")
105+
+ self.DELETE_SPACE
106+
+ delete('"')
107+
+ self.NOT_QUOTE.plus
108+
+ delete('"')
109+
)
110+
111+
# Email: username@domain
112+
graph_email = username + self.DELETE_SPACE + insert("@") + domain
113+
# URL: just output the protocol content directly
114+
graph_url = protocol
115+
116+
graph = graph_email | graph_url
117+
self.verbalizer = self.delete_tokens(graph)

‎itn/english/rules/measure.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pynini
16+
from pynini import closure, cross, invert, string_file
17+
from pynini.lib.pynutil import delete, insert
18+
19+
from itn.english.rules.cardinal import Cardinal
20+
from itn.english.rules.decimal import Decimal
21+
from tn.processor import Processor
22+
from tn.utils import get_abs_path
23+
24+
25+
class Measure(Processor):
26+
27+
def __init__(self, cardinal=None, decimal=None):
28+
super().__init__(name="measure", ordertype="itn")
29+
self.cardinal = cardinal or Cardinal()
30+
self.decimal = decimal or Decimal(cardinal=self.cardinal)
31+
self.build_tagger()
32+
self.build_verbalizer()
33+
34+
def build_tagger(self):
35+
ds = delete(" ")
36+
37+
# Load measurements: symbol\tname, invert to get name -> symbol
38+
units_graph = invert(
39+
string_file(get_abs_path("../itn/english/data/measurements.tsv"))
40+
)
41+
42+
# Handle plurals: strip trailing "s" to match singular form
43+
# e.g. "meters" -> "meter" -> "m", "kilograms" -> "kilogram" -> "kg"
44+
depluralize = pynini.cdrewrite(
45+
cross("s", ""), "", "[EOS]", self.VSIGMA
46+
)
47+
# Handle irregular plurals: "feet" -> "foot"
48+
irregular = pynini.string_map([("feet", "foot")])
49+
unit_singular = units_graph
50+
unit_plural = (depluralize | irregular) @ units_graph
51+
52+
unit = unit_singular | unit_plural
53+
54+
# Handle "per" units: "per hour" -> "/h"
55+
per_unit = insert("/") + delete("per") + ds + unit_singular
56+
full_unit = unit + closure(ds + per_unit, 0, 1) | per_unit
57+
58+
# Cardinal value
59+
cardinal_value = self.cardinal.graph
60+
61+
# Decimal value (reuse decimal's internal graph for the number)
62+
decimal_digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
63+
decimal_zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
64+
frac_digit = decimal_digit | decimal_zero | cross("o", "0")
65+
frac_graph = closure(frac_digit + ds) + frac_digit
66+
decimal_value = cardinal_value + ds + delete("point") + ds + insert(".") + frac_graph
67+
68+
# Optional minus/negative prefix
69+
minus = delete("minus") | delete("negative")
70+
optional_sign = closure(insert("-") + minus + ds, 0, 1)
71+
72+
# "point X" with no integer part
73+
point_only = delete("point") + ds + insert(".") + frac_graph
74+
75+
number = optional_sign + (decimal_value | cardinal_value | point_only)
76+
77+
value = insert('value: "') + number + insert('"')
78+
units = insert('units: "') + full_unit + insert('"')
79+
80+
final_graph = value + ds + insert(" ") + units
81+
self.tagger = self.add_tokens(final_graph)
82+
83+
def build_verbalizer(self):
84+
value = (
85+
delete("value:")
86+
+ self.DELETE_SPACE
87+
+ delete('"')
88+
+ self.NOT_QUOTE.plus
89+
+ delete('"')
90+
)
91+
units = (
92+
delete("units:")
93+
+ self.DELETE_SPACE
94+
+ delete('"')
95+
+ self.NOT_QUOTE.plus
96+
+ delete('"')
97+
)
98+
graph = value + self.DELETE_SPACE + insert(" ") + units
99+
self.verbalizer = self.delete_tokens(graph)

0 commit comments

Comments
 (0)