Modified/removed files

Signed-off-by: hmlee245 <hmlee245@gmail.com>

Author: hmlee245

nemo_text_processing/inverse_text_normalization/inverse_normalize.py

@@ -135,7 +135,7 @@ def __init__(
             from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import (
                 VerbalizeFinalFst,
-            )    
+            )
 
         self.tagger = ClassifyFst(
             cache_dir=cache_dir, whitelist=whitelist, overwrite_cache=overwrite_cache, input_case=input_case
@@ -180,7 +180,7 @@ def parse_args():
     parser.add_argument(
         "--language",
         help="language",
-        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh', 'hi', 'hy', 'mr', 'ja','ko'],
+        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh', 'hi', 'hy', 'mr', 'ja', 'ko'],
         default="en",
         type=str,
     )

nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py

@@ -19,6 +19,7 @@
 from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_DIGIT, GraphFst, delete_space
 from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path
 
+
 class CardinalFst(GraphFst):
     """
     Finite state transducer for classifying cardinals
@@ -34,13 +35,19 @@ def __init__(self):
         graph_zero = pynini.cross("영", "0")
         graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
 
+<<<<<<< HEAD
+=======
+        graph_negative = pynini.cross("마이너스", "-")
+        graph_negative += delete_space
+
+>>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
         ten = pynutil.delete("십")
         ten_alt = pynini.cross("십", "1")
         ### Responsible for second digit of two digit number. ex) 20's 2
         graph_ten_component = pynini.union((graph_digit + ten) | ten_alt, pynutil.insert("0"))
         ### Responsible for the first digit of number. ex) 1,2,3,4,5,,,
         graph_ten_component += graph_digit | pynutil.insert("0")
-        
+
         hundred = pynutil.delete("백")
         hundred_alt = pynini.cross("백", "1")
         graph_hundred_component = pynini.union(((graph_digit + hundred) | hundred_alt), pynutil.insert("0"))
@@ -55,46 +62,55 @@ def __init__(self):
         tenthousand_alt = pynini.cross("만", "1")
         ### "만" can express next four digits of numbers until the next unit "억", so insert "0000" to allocate four digit worth of space
         ### From "만", keep adding four digits and graph_thousand_component(0000-9999), because Korean units increase every four digits
-        graph_tenthousand_component = pynini.union(((graph_thousand_component + tenthousand) | tenthousand_alt), pynutil.insert("0000"))
+        graph_tenthousand_component = pynini.union(
+            ((graph_thousand_component + tenthousand) | tenthousand_alt), pynutil.insert("0000")
+        )
         graph_tenthousand_component += graph_thousand_component
 
         hundredmillion = pynutil.delete("억")
         hundredmillion_alt = pynini.cross("억", "1")
-        graph_hundredmillion_component = pynini.union(((graph_thousand_component + hundredmillion) | hundredmillion_alt), pynutil.insert("0000"))
-        graph_hundredmillion_component +=  graph_tenthousand_component
-        
+        graph_hundredmillion_component = pynini.union(
+            ((graph_thousand_component + hundredmillion) | hundredmillion_alt), pynutil.insert("0000")
+        )
+        graph_hundredmillion_component += graph_tenthousand_component
+
         trillion = pynutil.delete("조")
         trillion_alt = pynini.cross("조", "1")
-        graph_trillion_component = pynini.union(((graph_thousand_component + trillion) | trillion_alt), pynutil.insert("0000"))
+        graph_trillion_component = pynini.union(
+            ((graph_thousand_component + trillion) | trillion_alt), pynutil.insert("0000")
+        )
         graph_trillion_component += graph_hundredmillion_component
 
         tenquadrillion = pynutil.delete("경")
         tenquadrillion_alt = pynini.cross("경", "1")
-        graph_tenquadrillion_component = pynini.union(((graph_thousand_component + tenquadrillion) | tenquadrillion_alt), pynutil.insert("0000"))
+        graph_tenquadrillion_component = pynini.union(
+            ((graph_thousand_component + tenquadrillion) | tenquadrillion_alt), pynutil.insert("0000")
+        )
         graph_tenquadrillion_component += graph_trillion_component
 
-        
         graph = pynini.union(
             ### From biggest unit to smallest, everything is included
-            graph_tenquadrillion_component|
-            graph_zero
+            graph_tenquadrillion_component
+            | graph_zero
         )
 
         leading_zero = (
             pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT)
         )
         graph_nonzero = graph @ leading_zero
         graph = pynini.union(graph_nonzero, graph_zero)
-        
+
         graph = graph @ leading_zero | graph_zero
 
         self.just_cardinals = graph
 
-        optional_sign = pynini.closure((pynini.cross("마이너스", 'negative: "-"') | pynini.cross("-", 'negative: "-"')) + delete_space,0, 1)
+        optional_sign = pynini.closure(
+            (pynini.cross("마이너스", 'negative: "-"') | pynini.cross("-", 'negative: "-"')) + delete_space, 0, 1
+        )
 
         final_graph = (
             optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"")
         ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\""))
 
         final_graph = self.add_tokens(final_graph)
-        self.fst = final_graph.optimize()
+        self.fst = final_graph.optimize()

nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py

@@ -19,13 +19,13 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
-from nemo_text_processing.inverse_text_normalization.ko.taggers.word import WordFst
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( 
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import (
     INPUT_LOWER_CASED,
     GraphFst,
     generator_main,
 )
+from nemo_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
+from nemo_text_processing.inverse_text_normalization.ko.taggers.word import WordFst
 
 
 class ClassifyFst(GraphFst):
@@ -62,13 +62,13 @@ def __init__(
             cardinal = CardinalFst()
             cardinal_graph = cardinal.fst
             word_graph = WordFst().fst
-            classify = (pynutil.add_weight(cardinal_graph, 1.1)| pynutil.add_weight(word_graph, 100))
-           
+            classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(word_graph, 100)
+
             token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ")
             tagger = pynini.closure(token, 1)
 
             self.fst = tagger
 
             if far_file:
                 generator_main(far_file, {"tokenize_and_classify": self.fst})
-                logging.info(f"ClassifyFst grammars are saved to {far_file}.")
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")

nemo_text_processing/inverse_text_normalization/ko/taggers/word.py

@@ -27,6 +27,5 @@ class WordFst(GraphFst):
 
     def __init__(self):
         super().__init__(name="word", kind="classify")
-        word = pynutil.insert(
-            "name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"")
+        word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"")
         self.fst = word.optimize()

nemo_text_processing/inverse_text_normalization/ko/utils.py

@@ -15,9 +15,6 @@
 import os
 
 
-
 def get_abs_path(rel_path):
 
     return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path
-
-

nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py

@@ -11,3 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+<<<<<<< HEAD
+=======
+
+from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst
+>>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545

nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py

@@ -15,11 +15,7 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import (
-    NEMO_NOT_QUOTE,
-    GraphFst,
-    delete_space,
-)
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space
 
 
 class CardinalFst(GraphFst):
@@ -34,21 +30,17 @@ def __init__(self):
             pynutil.delete("negative:")
             + delete_space
             + pynutil.delete("\"")
-            + pynini.accep("-") 
+            + pynini.accep("-")
             + pynutil.delete("\"")
         )
 
         optional_sign_output = pynini.closure(negative_sign + delete_space, 0, 1)
 
-        digits_from_tag = pynini.closure(NEMO_NOT_QUOTE, 1) 
+        digits_from_tag = pynini.closure(NEMO_NOT_QUOTE, 1)
         integer_cardinal = (
-            pynutil.delete("integer:")
-            + delete_space
-            + pynutil.delete("\"")
-            + digits_from_tag
-            + pynutil.delete("\"")
+            pynutil.delete("integer:") + delete_space + pynutil.delete("\"") + digits_from_tag + pynutil.delete("\"")
         )
 
         graph = integer_cardinal
         final_graph = optional_sign_output + graph
-        self.fst = self.delete_tokens(final_graph).optimize()
+        self.fst = self.delete_tokens(final_graph).optimize()

nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
-from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
 
 
 class VerbalizeFst(GraphFst):
@@ -30,7 +30,6 @@ def __init__(self):
         cardinal = CardinalFst()
         cardinal_graph = cardinal.fst
         word_graph = WordFst().fst
-        
-        graph = (cardinal_graph|word_graph)
+
+        graph = cardinal_graph | word_graph
         self.fst = graph
-        

nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py

@@ -18,15 +18,21 @@
 import pynini
 from pynini.lib import pynutil
 
+from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, delete_space, generator_main
 from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
+<<<<<<< HEAD
 from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, generator_main, delete_space
+=======
+from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
+>>>>>>> 77da79d12b1378502cc2b382cd6933b02e7c2545
 
 
 class VerbalizeFinalFst(GraphFst):
     """
     Finite state transducer that verbalizes an entire sentence, e.g.
     tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
     """
+
     def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False):
         super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
         far_file = None

nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py

@@ -18,7 +18,6 @@
 from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst
 
 
-
 class WordFst(GraphFst):
     '''
     tokens { name: "一" } -> 一