feat: telephone with double/IP/serial/country code, 446/470 (95%)

Author: pengzhendong

itn/english/rules/telephone.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from pynini import closure, cross, string_file
-from pynini.lib.pynutil import delete, insert
+from pynini import closure, cross, string_file, union
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from itn.english.rules.cardinal import Cardinal
 from tn.processor import Processor
@@ -30,42 +30,70 @@ def __init__(self, cardinal=None):
 
     def build_tagger(self):
         ds = delete(" ")
-
-        # Single digit: spoken word -> digit character
         digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
         zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
-        single_digit = digit | zero | cross("o", "0") | cross("oh", "0")
-
-        # 10 digits formatted as XXX-XXX-XXXX
-        ten_digits = (
-            single_digit + ds + single_digit + ds + single_digit
-            + insert("-")
-            + ds + single_digit + ds + single_digit + ds + single_digit
-            + insert("-")
-            + ds + single_digit + ds + single_digit + ds + single_digit + ds + single_digit
-        )
+        single = digit | zero | cross("o", "0") | cross("oh", "0")
+
+        # "double X" => XX
+        double = union(*[cross(f"double {w}", f"{d}{d}")
+                         for w, d in [("one","1"),("two","2"),("three","3"),("four","4"),
+                                      ("five","5"),("six","6"),("seven","7"),("eight","8"),
+                                      ("nine","9"),("zero","0"),("oh","0"),("o","0")]])
+
+        # two-digit cardinal: twenty three => 23
+        two_digit = self.cardinal.graph_no_exception @ (self.DIGIT + self.DIGIT)
 
-        # Optional country code: "plus X" or just digits before the main number
-        country_code_digits = (
-            closure(single_digit + ds, 0, 2) + single_digit
+        # a token is 1 or 2 digits
+        token = single | double | add_weight(two_digit, 0.002)
+
+        # sequence of tokens separated by spaces
+        seq = token + closure(ds + token)
+
+        # phone: XXX-XXX-XXXX
+        phone = seq @ (
+            self.DIGIT ** 3 + insert("-") + self.DIGIT ** 3 + insert("-") + self.DIGIT ** 4
         )
+
+        # country code
         country_code = (
-            closure(cross("plus ", "+"), 0, 1) + country_code_digits
+            insert('country_code: "')
+            + closure(cross("plus ", "+"), 0, 1)
+            + (closure(single + ds, 0, 2) + single | add_weight(two_digit, 0.002))
+            + insert('"')
         )
-        optional_country_code = closure(
-            country_code + insert(" ") + ds, 0, 1
+        optional_cc = closure(country_code + ds + insert(" "), 0, 1)
+
+        graph = optional_cc + insert('number_part: "') + phone + insert('"')
+
+        # SSN: XXX-XX-XXXX
+        ssn = seq @ (
+            self.DIGIT ** 3 + insert("-") + self.DIGIT ** 2 + insert("-") + self.DIGIT ** 4
         )
+        graph |= insert('number_part: "') + ssn + insert('"')
 
-        graph = optional_country_code + ten_digits
-        final_graph = insert('value: "') + graph + insert('"')
-        self.tagger = self.add_tokens(final_graph)
+        # IP: X.X.X.X
+        ip_token = single + closure(ds + single, 0, 2) | add_weight(two_digit, 0.002)
+        ip = ip_token + (cross(" dot ", ".") + ip_token) ** 3
+        graph |= insert('number_part: "') + add_weight(ip, -0.001) + insert('"')
 
-    def build_verbalizer(self):
-        value = (
-            delete("value:")
-            + self.DELETE_SPACE
-            + delete('"')
-            + self.NOT_QUOTE.plus
-            + delete('"')
+        # credit card: XXXX XXXX XXXX XXXX or XXXX XXXXXX XXXXX
+        cc = seq @ (
+            self.DIGIT ** 4 + insert(" ") + self.DIGIT ** 4
+            + insert(" ") + self.DIGIT ** 4 + insert(" ") + self.DIGIT ** 4
         )
-        self.verbalizer = self.delete_tokens(value)
+        graph |= insert('number_part: "') + cc + insert('"')
+
+        # serial: mixed alpha+digits, at least one digit, length >= 3
+        serial_char = single | add_weight(two_digit, 0.002) | self.ALPHA
+        serial = serial_char + closure(ds + serial_char, 2)
+        serial = serial @ (closure(self.ALPHA | self.DIGIT) + self.DIGIT + closure(self.ALPHA | self.DIGIT))
+        graph |= insert('number_part: "') + add_weight(serial, 0.001) + insert('"')
+
+        self.tagger = self.add_tokens(graph)
+
+    def build_verbalizer(self):
+        cc = delete('country_code: "') + self.NOT_QUOTE.plus + delete('"')
+        num = delete(' number_part: "') + self.NOT_QUOTE.plus + delete('"')
+        num_only = delete('number_part: "') + self.NOT_QUOTE.plus + delete('"')
+        graph = cc + self.DELETE_SPACE + insert(" ") + num | num_only
+        self.verbalizer = self.delete_tokens(graph)

tn/token_parser.py

@@ -32,7 +32,7 @@
     "measure": ["numerator", "denominator", "value", "units"],
     "money": ["currency", "value", "decimal", "quantity"],
     "time": ["hour", "minute", "second", "noon", "zone"],
-    "telephone": ["value"],
+    "telephone": ["country_code", "number_part"],
     "electronic": ["username", "domain", "protocol"],
 }