feat: 463/470 (98.5%) NeMo coverage

- cardinal: fix zero in exception list
- date: add Q2 quarter, 750BC, 3-digit year, decades => 36/36 full pass
- time: fix date vs time priority => 29/29 full pass
- whitelist: fixed via date priority => 12/12 full pass
- telephone: fix serial two_digit weight, IP combinations
- 7 full-pass rules: ordinal, decimal, measure, date, time, whitelist, money(51/52)

Author: pengzhendong

itn/english/rules/cardinal.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from pynini import closure, cross, difference, string_file, union
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from tn.processor import Processor
 from tn.utils import get_abs_path
@@ -35,7 +35,7 @@ def build_tagger(self):
 
         # 1~9, 10~19, 20~99
         one_digit = digit
-        two_digit = teen | (ties + (ds + digit | insert("0")))
+        two_digit = teen | (ties + (ds + digit | add_weight(insert("0"), 0.1)))
         self.graph_two_digit = two_digit
         up_to_99 = one_digit | two_digit
 
@@ -117,7 +117,7 @@ def _with_mag_padded(name):
 
         # exclude 0-12 from cardinal tagger (they stay as words)
         from itn.english.rules.time import _num_to_word
-        exception_labels = [_num_to_word(x) for x in range(0, 13) if _num_to_word(x)]
+        exception_labels = ["zero"] + [_num_to_word(x) for x in range(1, 13)]
         exception = union(*exception_labels).optimize()
         graph_with_exception = (difference(self.VSIGMA, exception) @ graph).optimize()
 

itn/english/rules/date.py

@@ -56,6 +56,8 @@ def build_tagger(self):
 
         # Year as two groups of two digits: "twenty twelve" => 2012
         year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen)
+        # 3-digit year: "seven fifty" => 750
+        year_three_digit = digit + ds + (two_digit | oh_digit | teen)
 
         # Year as "X thousand Y": "two thousand twelve" => 2012
         # Need zero-padded variants so "two thousand three" => 2003
@@ -116,7 +118,7 @@ def build_tagger(self):
             + po
         )
         # Year only => "twenty twelve", "two thousand three"
-        graph_y = add_weight(year, 0.01) + po
+        graph_y = year + po
 
         # Decades: "nineteen eighties" => 1980s
         decade_suffix = closure(self.ALPHA, 1) + (cross("ies", "y") | delete("s"))
@@ -125,7 +127,23 @@ def build_tagger(self):
             insert('year: "') + (teen | two_digit) + ds + decade_word + insert('0s"') + po
         )
 
-        final_graph = graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y | graph_decade
+        # Quarter: "second quarter of twenty twenty two" => Q2 2022
+        quarter_num = (
+            cross("first", "1") | cross("second", "2")
+            | cross("third", "3") | cross("fourth", "4")
+        )
+        graph_quarter = (
+            insert('day: "Q') + quarter_num + insert('"')
+            + ds + delete("quarter") + ds + delete("of") + ds
+            + insert(' year: "') + year_graph + insert('"') + po
+        )
+
+        # BC/AD suffix
+        bc_ad = ds + (cross("b c", "BC") | cross("a d", "AD"))
+        year_graph_with_3digit = year_graph | year_three_digit
+        graph_y_bc = insert('year: "') + year_graph_with_3digit + bc_ad + insert('"') + po
+
+        final_graph = graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y | graph_decade | graph_quarter | graph_y_bc
         self.tagger = self.add_tokens(final_graph)
 
     def build_verbalizer(self):
@@ -167,6 +185,8 @@ def build_verbalizer(self):
         graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year
         # year only
         graph_y = year
+        # day + year (for quarter: Q2 2022)
+        graph_dy = day + self.DELETE_SPACE + insert(" ") + year
 
-        graph = (graph_mdy | graph_dmy | graph_y) + self.DELETE_SPACE + delete_po
+        graph = (graph_mdy | graph_dmy | graph_dy | graph_y) + self.DELETE_SPACE + delete_po
         self.verbalizer = self.delete_tokens(graph)

itn/english/rules/telephone.py

@@ -91,7 +91,7 @@ def build_tagger(self):
 
         # serial: mixed alpha+digits, at least one digit, length >= 3
         serial_char = add_weight(single, 0.001) | add_weight(two_digit, -0.001) | self.ALPHA
-        serial = serial_char + closure(ds + serial_char, 2)
+        serial = serial_char + closure(ds + serial_char, 1)
         serial = serial @ (closure(self.ALPHA | self.DIGIT) + self.DIGIT + closure(self.ALPHA | self.DIGIT))
         graph |= insert('number_part: "') + add_weight(serial, 2.0) + insert('"')
 

itn/english/test/data/en_cardinal.txt

@@ -11,6 +11,7 @@ one quadrillion two hundred sixty four trillion three hundred one billion nine h
 minus sixty => -60
 forty six thousand six hundred sixty four => 46664
 sixty => 60
+zero => zero
 two million three => 2000003
 one thousand thirteen => 1013
 one thousand one => 1001

itn/english/test/data/en_date.txt

@@ -31,3 +31,6 @@ nineteen seventy six => 1976
 june twentieth twenty fourteen => june 20 2014
 nineteen seventy three => 1973
 nineteen seventy five => 1975
+eleven fifty five => 1155
+second quarter of twenty twenty two => Q2 2022
+seven fifty b c => 750BC

itn/english/test/data/en_telephone.txt

@@ -9,10 +9,12 @@ double oh three one two three five six seven eight => 003-123-5678
 one two three dot one two three dot o dot four o => 123.123.0.40
 one twenty three dot one two three dot o dot four o => 123.123.0.40
 two two five dot double five dot o dot four o => 225.55.0.40
+two two five dot double five dot o dot forty five => 225.55.0.45
 ssn is seven double nine one two three double one three => ssn is 799-12-3113
 seven nine nine => 799
 a b nine => ab9
 a b c => a b c
 five w k r a three one => 5wkra31
+x eighty six => x86
 x three eighty six => x386
 r t x forty fifty t i => RTX 4050ti

itn/english/test/data/en_whitelist.txt

@@ -9,3 +9,4 @@ r t x => RTX
 cat five e => CAT5e
 c u d n n => cuDNN
 p c i e x eight => PCIe x8
+l g a eleven fifty => LGA 1150