Refactor Korean TN cardinal and postprocessing logic based on review feedback

Author: bbae0312

nemo_text_processing/text_normalization/ko/data/number/teen.tsv

@@ -1,3 +1,4 @@
+1	십
 2	이십
 3	삼십
 4	사십

nemo_text_processing/text_normalization/ko/data/number/ty.tsv

@@ -23,7 +23,7 @@
 from pynini.export import export
 from pynini.lib import byte, pynutil, utf8
 
-from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels
+from nemo_text_processing.text_normalization.en.utils import load_labels
 from nemo_text_processing.utils.logging import logger
 
 NEMO_CHAR = utf8.VALID_UTF8_CHAR

nemo_text_processing/text_normalization/ko/graph_utils.py

@@ -28,26 +28,31 @@ def __init__(self, deterministic: bool = True):
         graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv"))
 
         digit_except_one = pynini.difference(NEMO_DIGIT, "1")
-        digit_except_zero_one = pynini.difference(digit_except_one, "0")
+        digit_except_zero_one = pynini.difference(digit_except_one, "0") <<<<<<< HEAD
 
         graph_digit_alt = digit_except_zero_one @ graph_digit
         graph_ty = pynini.string_file(get_abs_path("data/number/ty.tsv"))
         graph_teen = pynini.string_file(get_abs_path("data/number/teen.tsv"))
+=======
+      
+        graph_digit_no_zero_one = digit_except_zero_one @ graph_digit
+        graph_ty = pynini.string_file(get_abs_path("data/number/ty.tsv"))       
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
 
         # Compose all basic number forms
-        graph_all = (graph_ty + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit
+        graph_1_to_99 = (graph_ty + (graph_digit | pynutil.delete('0'))) | graph_digit
 
         hundreds = NEMO_DIGIT**3
-        graph_hundred_component = (pynini.cross('1', '백') | (graph_digit_alt + pynutil.insert('백'))) + pynini.union(
-            pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_all)
+        graph_hundred_component = (pynini.cross('1', '백') | (graph_digit_no_zero_one + pynutil.insert('백'))) + pynini.union(
+            pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_1_to_99)
         )
         graph_hundred = hundreds @ graph_hundred_component
 
         thousands = NEMO_DIGIT**4
-        graph_thousand_component = (pynini.cross('1', '천') | (graph_digit_alt + pynutil.insert('천'))) + pynini.union(
+        graph_thousand_component = (pynini.cross('1', '천') | (graph_digit_no_zero_one + pynutil.insert('천'))) + pynini.union(
             pynini.closure(pynutil.delete('0')),
             graph_hundred_component,
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_thousand = thousands @ graph_thousand_component
 
@@ -56,36 +61,44 @@ def __init__(self, deterministic: bool = True):
             pynini.closure(pynutil.delete('0')),
             graph_thousand_component,
             (pynutil.delete('0') + graph_hundred_component),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_ten_thousand = ten_thousands @ graph_ten_thousand_component
 
         hundred_thousands = NEMO_DIGIT**6
+<<<<<<< HEAD
         graph_hundred_thousand_component = ((NEMO_DIGIT**2 @ graph_all) + pynutil.insert('만')) + pynini.union(
+=======
+        graph_hundred_thousand_component = ((NEMO_DIGIT ** 2 @ graph_1_to_99) + pynutil.insert('만')) + pynini.union(
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             pynini.closure(pynutil.delete('0')),
             graph_thousand_component,
             (pynutil.delete('0') + graph_hundred_component),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component
 
         millions = NEMO_DIGIT**7
-        graph_million_component = ((NEMO_DIGIT**3 @ graph_hundred_component) + pynutil.insert('만')) + pynini.union(
+        graph_million_component = ((graph_hundred) + pynutil.insert('만')) + pynini.union(
             pynini.closure(pynutil.delete('0')),
             graph_thousand_component,
             (pynutil.delete('0') + graph_hundred_component),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_million = millions @ graph_million_component
 
         ten_millions = NEMO_DIGIT**8
+<<<<<<< HEAD
         graph_ten_million_component = (
             (NEMO_DIGIT**4 @ graph_thousand_component) + pynutil.insert('만')
         ) + pynini.union(
+=======
+        graph_ten_million_component = ((graph_thousand) + pynutil.insert('만')) + pynini.union(
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             pynini.closure(pynutil.delete('0')),
             graph_thousand_component,
             (pynutil.delete('0') + graph_hundred_component),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_ten_million = ten_millions @ graph_ten_million_component
 
@@ -98,48 +111,52 @@ def __init__(self, deterministic: bool = True):
             (pynutil.delete('000') + graph_ten_thousand_component),
             (pynutil.delete('0000') + graph_thousand_component),
             ((pynutil.delete('00000') + graph_hundred_component)),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_hundred_million = hundred_millions @ graph_hundred_million_component
 
         thousand_millions = NEMO_DIGIT**10
-        graph_thousand_million_component = ((NEMO_DIGIT**2 @ graph_all) + pynutil.insert('억')) + pynini.union(
+        graph_thousand_million_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert('억')) + pynini.union(
             pynini.closure(pynutil.delete('0')),
             graph_ten_million_component,
             (pynutil.delete('0') + graph_million_component),
             (pynutil.delete('00') + graph_hundred_thousand_component),
             (pynutil.delete('000') + graph_ten_thousand_component),
             (pynutil.delete('0000') + graph_thousand_component),
             ((pynutil.delete('00000') + graph_hundred_component)),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_thousand_million = thousand_millions @ graph_thousand_million_component
 
         billions = NEMO_DIGIT**11
-        graph_billions_component = ((NEMO_DIGIT**3 @ graph_hundred_component) + pynutil.insert('억')) + pynini.union(
+        graph_billions_component = ((graph_hundred) + pynutil.insert('억')) + pynini.union(
             pynini.closure(pynutil.delete('0')),
             graph_ten_million_component,
             (pynutil.delete('0') + graph_million_component),
             (pynutil.delete('00') + graph_hundred_thousand_component),
             (pynutil.delete('000') + graph_ten_thousand_component),
             (pynutil.delete('0000') + graph_thousand_component),
             ((pynutil.delete('00000') + graph_hundred_component)),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_billions = billions @ graph_billions_component
 
         ten_billions = NEMO_DIGIT**12
+<<<<<<< HEAD
         graph_ten_billions_component = (
             (NEMO_DIGIT**4 @ graph_thousand_component) + pynutil.insert('억')
         ) + pynini.union(
+=======
+        graph_ten_billions_component = ((graph_thousand) + pynutil.insert('억')) + pynini.union(
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             pynini.closure(pynutil.delete('0')),
             graph_ten_million_component,
             (pynutil.delete('0') + graph_million_component),
             (pynutil.delete('00') + graph_hundred_thousand_component),
             (pynutil.delete('000') + graph_ten_thousand_component),
             (pynutil.delete('0000') + graph_thousand_component),
             ((pynutil.delete('00000') + graph_hundred_component)),
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_ten_billions = ten_billions @ graph_ten_billions_component
 
@@ -156,11 +173,12 @@ def __init__(self, deterministic: bool = True):
             pynutil.delete('0000000') + graph_ten_thousand_component,
             pynutil.delete('00000000') + graph_thousand_component,
             pynutil.delete('000000000') + graph_hundred_component,
-            (pynini.closure(pynutil.delete('0')) + graph_all),
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99),
         )
         graph_hundred_billions = hundred_billions @ graph_hundred_billions_component
 
         trillion = NEMO_DIGIT**14
+<<<<<<< HEAD
         graph_trillion_component = (
             (NEMO_DIGIT**2 @ graph_all)
             + pynutil.insert('조')
@@ -177,11 +195,27 @@ def __init__(self, deterministic: bool = True):
                 pynutil.delete('00000000') + graph_thousand_component,
                 pynutil.delete('000000000') + graph_hundred_component,
                 (pynini.closure(pynutil.delete('0')) + graph_all),
+=======
+        graph_trillion_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert('조') + pynini.union(
+            pynini.closure(pynutil.delete('0')),
+            graph_ten_billions_component,
+            pynutil.delete('0') + graph_billions_component,
+            pynutil.delete('00') + graph_thousand_million_component,
+            pynutil.delete('000') + graph_hundred_million_component,
+            pynutil.delete('0000') + graph_ten_million_component,
+            pynutil.delete('00000') + graph_million_component,
+            pynutil.delete('000000') + graph_hundred_thousand_component,
+            pynutil.delete('0000000') + graph_ten_thousand_component,
+            pynutil.delete('00000000') + graph_thousand_component,
+            pynutil.delete('000000000') + graph_hundred_component,
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99)
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             )
         )
         graph_trillions = trillion @ graph_trillion_component
 
         ten_trillions = NEMO_DIGIT**15
+<<<<<<< HEAD
         graph_ten_trillions_component = (
             (NEMO_DIGIT**3 @ graph_hundred_component)
             + pynutil.insert('조')
@@ -199,10 +233,27 @@ def __init__(self, deterministic: bool = True):
                 pynutil.delete('000000000') + graph_hundred_component,
                 (pynini.closure(pynutil.delete('0')) + graph_all),
             )
+=======
+        graph_ten_trillions_component = ((graph_hundred) + pynutil.insert('조') + pynini.union(
+            pynini.closure(pynutil.delete('0')),
+            graph_ten_billions_component,
+            pynutil.delete('0') + graph_billions_component,
+            pynutil.delete('00') + graph_thousand_million_component,
+            pynutil.delete('000') + graph_hundred_million_component,
+            pynutil.delete('0000') + graph_ten_million_component,
+            pynutil.delete('00000') + graph_million_component,
+            pynutil.delete('000000') + graph_hundred_thousand_component,
+            pynutil.delete('0000000') + graph_ten_thousand_component,
+            pynutil.delete('00000000') + graph_thousand_component,
+            pynutil.delete('000000000') + graph_hundred_component,
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99)
+            )       
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
         )
         graph_ten_trillions = ten_trillions @ graph_ten_trillions_component
 
         hundred_trillions = NEMO_DIGIT**16
+<<<<<<< HEAD
         graph_hundred_trillions_component = (
             (NEMO_DIGIT**4 @ graph_thousand_component)
             + pynutil.insert('조')
@@ -219,11 +270,27 @@ def __init__(self, deterministic: bool = True):
                 pynutil.delete('00000000') + graph_thousand_component,
                 pynutil.delete('000000000') + graph_hundred_component,
                 (pynini.closure(pynutil.delete('0')) + graph_all),
+=======
+        graph_hundred_trillions_component = ((graph_thousand) + pynutil.insert('조') + pynini.union(
+            pynini.closure(pynutil.delete('0')),
+            graph_ten_billions_component,
+            pynutil.delete('0') + graph_billions_component,
+            pynutil.delete('00') + graph_thousand_million_component,
+            pynutil.delete('000') + graph_hundred_million_component,
+            pynutil.delete('0000') + graph_ten_million_component,
+            pynutil.delete('00000') + graph_million_component,
+            pynutil.delete('000000') + graph_hundred_thousand_component,
+            pynutil.delete('0000000') + graph_ten_thousand_component,
+            pynutil.delete('00000000') + graph_thousand_component,
+            pynutil.delete('000000000') + graph_hundred_component,
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99)
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             )
         )
         graph_hundred_trillions = hundred_trillions @ graph_hundred_trillions_component
 
         thousand_trillions = NEMO_DIGIT**17
+<<<<<<< HEAD
         graph_thousand_trillions_component = (
             graph_digit
             + pynutil.insert('경')
@@ -244,6 +311,25 @@ def __init__(self, deterministic: bool = True):
                 pynutil.delete('000000000000') + graph_thousand_component,
                 pynutil.delete('0000000000000') + graph_hundred_component,
                 (pynini.closure(pynutil.delete('0')) + graph_all),
+=======
+        graph_thousand_trillions_component = (graph_digit + pynutil.insert('경') + pynini.union(
+            pynini.closure(pynutil.delete('0')),
+            graph_hundred_trillions_component,
+            pynutil.delete('0') + graph_ten_trillions_component,
+            pynutil.delete('00') + graph_trillion_component,
+            pynutil.delete('000') + graph_hundred_billions_component,
+            pynutil.delete('0000') + graph_ten_billions_component,
+            pynutil.delete('00000') + graph_billions_component,
+            pynutil.delete('000000') + graph_thousand_million_component,
+            pynutil.delete('0000000') + graph_hundred_million_component,
+            pynutil.delete('00000000') + graph_ten_million_component,
+            pynutil.delete('000000000') + graph_million_component,
+            pynutil.delete('0000000000') + graph_hundred_thousand_component,
+            pynutil.delete('00000000000') + graph_ten_thousand_component,
+            pynutil.delete('000000000000') + graph_thousand_component,
+            pynutil.delete('0000000000000') + graph_hundred_component,
+            (pynini.closure(pynutil.delete('0')) + graph_1_to_99)
+>>>>>>> 68b18fa8 (Refactor Korean TN cardinal and postprocessing logic based on review feedback)
             )
         )
         graph_thousand_trillions = thousand_trillions @ graph_thousand_trillions_component
@@ -265,7 +351,7 @@ def __init__(self, deterministic: bool = True):
             graph_ten_thousand,
             graph_thousand,
             graph_hundred,
-            graph_all,
+            graph_1_to_99,
             graph_zero,
         ).optimize()
 

nemo_text_processing/text_normalization/ko/taggers/cardinal.py

@@ -18,10 +18,7 @@
 from pynini.lib import pynutil
 
 from nemo_text_processing.text_normalization.ko.graph_utils import (
-    NEMO_WHITE_SPACE,
     GraphFst,
-    delete_extra_space,
-    delete_space,
     generator_main,
 )