This reverts commit f893d89, reversing

hmlee245 · hmlee245 · commit 41ac59d79151 · 2025-05-16T13:53:59.000-07:00
changes made to 9f7e876. Signed-off-by: hmlee245 <hmlee245@gmail.com>
diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py
@@ -135,7 +135,7 @@ def __init__(
             from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import (
                 VerbalizeFinalFst,
-            )
+            )    
 
         self.tagger = ClassifyFst(
             cache_dir=cache_dir, whitelist=whitelist, overwrite_cache=overwrite_cache, input_case=input_case
@@ -180,7 +180,7 @@ def parse_args():
     parser.add_argument(
         "--language",
         help="language",
-        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh', 'hi', 'hy', 'mr', 'ja', 'ko'],
+        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh', 'hi', 'hy', 'mr', 'ja','ko'],
         default="en",
         type=str,
     )
diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py
@@ -19,7 +19,6 @@
 from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_DIGIT, GraphFst, delete_space
 from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path
 
-
 class CardinalFst(GraphFst):
     """
     Finite state transducer for classifying cardinals
@@ -35,19 +34,13 @@ def __init__(self):
         graph_zero = pynini.cross("영", "0")
         graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
 
-<<<<<<< HEAD
-=======
-        graph_negative = pynini.cross("마이너스", "-")
-        graph_negative += delete_space
-
->>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
         ten = pynutil.delete("십")
         ten_alt = pynini.cross("십", "1")
         ### Responsible for second digit of two digit number. ex) 20's 2
         graph_ten_component = pynini.union((graph_digit + ten) | ten_alt, pynutil.insert("0"))
         ### Responsible for the first digit of number. ex) 1,2,3,4,5,,,
         graph_ten_component += graph_digit | pynutil.insert("0")
-
+        
         hundred = pynutil.delete("백")
         hundred_alt = pynini.cross("백", "1")
         graph_hundred_component = pynini.union(((graph_digit + hundred) | hundred_alt), pynutil.insert("0"))
@@ -62,55 +55,46 @@ def __init__(self):
         tenthousand_alt = pynini.cross("만", "1")
         ### "만" can express next four digits of numbers until the next unit "억", so insert "0000" to allocate four digit worth of space
         ### From "만", keep adding four digits and graph_thousand_component(0000-9999), because Korean units increase every four digits
-        graph_tenthousand_component = pynini.union(
-            ((graph_thousand_component + tenthousand) | tenthousand_alt), pynutil.insert("0000")
-        )
+        graph_tenthousand_component = pynini.union(((graph_thousand_component + tenthousand) | tenthousand_alt), pynutil.insert("0000"))
         graph_tenthousand_component += graph_thousand_component
 
         hundredmillion = pynutil.delete("억")
         hundredmillion_alt = pynini.cross("억", "1")
-        graph_hundredmillion_component = pynini.union(
-            ((graph_thousand_component + hundredmillion) | hundredmillion_alt), pynutil.insert("0000")
-        )
-        graph_hundredmillion_component += graph_tenthousand_component
-
+        graph_hundredmillion_component = pynini.union(((graph_thousand_component + hundredmillion) | hundredmillion_alt), pynutil.insert("0000"))
+        graph_hundredmillion_component +=  graph_tenthousand_component
+        
         trillion = pynutil.delete("조")
         trillion_alt = pynini.cross("조", "1")
-        graph_trillion_component = pynini.union(
-            ((graph_thousand_component + trillion) | trillion_alt), pynutil.insert("0000")
-        )
+        graph_trillion_component = pynini.union(((graph_thousand_component + trillion) | trillion_alt), pynutil.insert("0000"))
         graph_trillion_component += graph_hundredmillion_component
 
         tenquadrillion = pynutil.delete("경")
         tenquadrillion_alt = pynini.cross("경", "1")
-        graph_tenquadrillion_component = pynini.union(
-            ((graph_thousand_component + tenquadrillion) | tenquadrillion_alt), pynutil.insert("0000")
-        )
+        graph_tenquadrillion_component = pynini.union(((graph_thousand_component + tenquadrillion) | tenquadrillion_alt), pynutil.insert("0000"))
         graph_tenquadrillion_component += graph_trillion_component
 
+        
         graph = pynini.union(
             ### From biggest unit to smallest, everything is included
-            graph_tenquadrillion_component
-            | graph_zero
+            graph_tenquadrillion_component|
+            graph_zero
         )
 
         leading_zero = (
             pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT)
         )
         graph_nonzero = graph @ leading_zero
         graph = pynini.union(graph_nonzero, graph_zero)
-
+        
         graph = graph @ leading_zero | graph_zero
 
         self.just_cardinals = graph
 
-        optional_sign = pynini.closure(
-            (pynini.cross("마이너스", 'negative: "-"') | pynini.cross("-", 'negative: "-"')) + delete_space, 0, 1
-        )
+        optional_sign = pynini.closure((pynini.cross("마이너스", 'negative: "-"') | pynini.cross("-", 'negative: "-"')) + delete_space,0, 1)
 
         final_graph = (
             optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"")
         ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\""))
 
         final_graph = self.add_tokens(final_graph)
-        self.fst = final_graph.optimize()
+        self.fst = final_graph.optimize()
diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py
@@ -19,9 +19,13 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import INPUT_LOWER_CASED, GraphFst, generator_main
 from nemo_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
 from nemo_text_processing.inverse_text_normalization.ko.taggers.word import WordFst
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( 
+    INPUT_LOWER_CASED,
+    GraphFst,
+    generator_main,
+)
 
 
 class ClassifyFst(GraphFst):
@@ -58,13 +62,13 @@ def __init__(
             cardinal = CardinalFst()
             cardinal_graph = cardinal.fst
             word_graph = WordFst().fst
-            classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(word_graph, 100)
-
+            classify = (pynutil.add_weight(cardinal_graph, 1.1)| pynutil.add_weight(word_graph, 100))
+           
             token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ")
             tagger = pynini.closure(token, 1)
 
             self.fst = tagger
 
             if far_file:
                 generator_main(far_file, {"tokenize_and_classify": self.fst})
-                logging.info(f"ClassifyFst grammars are saved to {far_file}.")
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")
diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/word.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/word.py
@@ -27,5 +27,6 @@ class WordFst(GraphFst):
 
     def __init__(self):
         super().__init__(name="word", kind="classify")
-        word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"")
+        word = pynutil.insert(
+            "name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"")
         self.fst = word.optimize()
diff --git a/nemo_text_processing/inverse_text_normalization/ko/utils.py b/nemo_text_processing/inverse_text_normalization/ko/utils.py
@@ -15,6 +15,9 @@
 import os
 
 
+
 def get_abs_path(rel_path):
 
     return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path
+
+
diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py
@@ -11,10 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-<<<<<<< HEAD
-=======
-
-from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
-from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
-from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst
->>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py
@@ -15,7 +15,11 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import (
+    NEMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
 
 
 class CardinalFst(GraphFst):
@@ -30,17 +34,21 @@ def __init__(self):
             pynutil.delete("negative:")
             + delete_space
             + pynutil.delete("\"")
-            + pynini.accep("-")
+            + pynini.accep("-") 
             + pynutil.delete("\"")
         )
 
         optional_sign_output = pynini.closure(negative_sign + delete_space, 0, 1)
 
-        digits_from_tag = pynini.closure(NEMO_NOT_QUOTE, 1)
+        digits_from_tag = pynini.closure(NEMO_NOT_QUOTE, 1) 
         integer_cardinal = (
-            pynutil.delete("integer:") + delete_space + pynutil.delete("\"") + digits_from_tag + pynutil.delete("\"")
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete("\"")
+            + digits_from_tag
+            + pynutil.delete("\"")
         )
 
         graph = integer_cardinal
         final_graph = optional_sign_output + graph
-        self.fst = self.delete_tokens(final_graph).optimize()
+        self.fst = self.delete_tokens(final_graph).optimize()
diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py
@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
 
 
 class VerbalizeFst(GraphFst):
@@ -30,6 +30,7 @@ def __init__(self):
         cardinal = CardinalFst()
         cardinal_graph = cardinal.fst
         word_graph = WordFst().fst
-
-        graph = cardinal_graph | word_graph
+        
+        graph = (cardinal_graph|word_graph)
         self.fst = graph
+        
diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py
@@ -18,24 +18,15 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, delete_space, generator_main
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
-
-<<<<<<< HEAD
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, delete_space, generator_main
-
-=======
-from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
-
->>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, generator_main, delete_space
 
 
 class VerbalizeFinalFst(GraphFst):
     """
     Finite state transducer that verbalizes an entire sentence, e.g.
     tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
     """
-
     def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False):
         super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
         far_file = None
diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py
@@ -18,6 +18,7 @@
 from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst
 
 
+
 class WordFst(GraphFst):
     '''
     tokens { name: "一" } -> 一
diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py
@@ -35,7 +35,7 @@ def parse_args():
     parser.add_argument(
         "--lang",
         help="language",
-        choices=["ar", "de", "en", "es", "es_en", "fr", "hi", "hy", "mr", "pt", "ru", "sv", "vi", "zh", "ja", "ko"],
+        choices=["ar", "de", "en", "es", "es_en", "fr", "hi", "hy", "mr", "pt", "ru", "sv", "vi", "zh", "ja","ko"],
         default="en",
         type=str,
     )
diff --git a/tests/nemo_text_processing/ko/test_cardinal.py b/tests/nemo_text_processing/ko/test_cardinal.py
@@ -28,12 +28,4 @@ class TestCardinal:
     @pytest.mark.unit
     def test_denorm(self, test_input, expected):
         pred = self.inverse_normalizer_ko.inverse_normalize(test_input, verbose=False)
-<<<<<<< HEAD
-        assert pred == expected
-=======
-        assert pred == expected
-
-    normalizer_with_audio_ko = (
-        NormalizerWithAudio(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) if RUN_AUDIO_BASED_TESTS else None
-    )
->>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
+        assert pred == expected
diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py
@@ -106,7 +106,7 @@ def parse_args():
             'mr',
             'ja',
             'rw',
-            'ko',
+            'ko'
         ],
         type=str,
         default='en',
