Future Implementations for classes - Measure, Money, and Date

Signed-off-by: Namrata Gachchi <ngachchi@nvidia.com>

Author: ngachchi

nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv

@@ -0,0 +1,3 @@
+ई. पू.	ईसा पूर्व
+ई.	ईसवी
+तक  तक

nemo_text_processing/text_normalization/hi/data/measure/unit.tsv

@@ -141,14 +141,16 @@ month	महीना
 months	महीने
 ct	कैरेट
 pH	पीएच
+km/h	किलोमीटर प्रति घंटा
 km/hr	किलोमीटर प्रति घंटा
 km/min	किलोमीटर प्रति मिनट
+m/h	मीटर प्रति घंटा
 m/hr	मीटर प्रति घंटा
 mi/s	मील प्रति सेकंड
+mi/h	मील प्रति घंटा
 mi/hr	मील प्रति घंटा
 mi/min	मील प्रति मिनट
 ₹/ac	रुपए प्रति एकड़
 x	बाई
 X	बाई
 *	बाई
--	से

nemo_text_processing/text_normalization/hi/data/money/currency.tsv

@@ -1,10 +1,9 @@
 ₹	रुपए
-P	पैसे 
 £	पाउंड
 ₩	वॉन
 $	डॉलर
 ₺	लीरा
 ৳	टका
 ¥	येन
 ₦	नाइरा
-€	यूरो
+€	यूरो

nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.tsv

@@ -0,0 +1,9 @@
+रुपए	पैसे
+पाउंड	पेंस
+वॉन	जिओन
+डॉलर	सेंट
+लीरा	कुरस
+टका	पैसे
+येन	सेन
+नाइरा	कोबो
+यूरो	सेंट

nemo_text_processing/text_normalization/hi/data/time/hours.tsv

@@ -1,3 +1,4 @@
+०	शून्य
 १	एक
 २	दो
 ३	तीन

nemo_text_processing/text_normalization/hi/taggers/date.py

@@ -26,6 +26,7 @@
 
 days = pynini.string_file(get_abs_path("data/date/days.tsv"))
 months = pynini.string_file(get_abs_path("data/date/months.tsv"))
+year_suffix = pynini.string_file(get_abs_path("data/date/year_suffix.tsv"))
 
 
 class DateFst(GraphFst):
@@ -62,12 +63,17 @@ def __init__(self, cardinal: GraphFst):
 
         years_graph = pynutil.insert("year: \"") + graph_year + pynutil.insert("\"") + insert_space
 
-        graph_dd_mm = days_graph + delete_dash + months_graph
+        graph_dd_mm = days_graph + (delete_dash | pynini.accep("")) + months_graph
 
-        graph_mm_dd = months_graph + delete_dash + days_graph
+        graph_mm_dd = months_graph + (delete_dash | pynini.accep("")) + days_graph
 
         graph_mm_dd += pynutil.insert(" preserve_order: true ")
 
+        # Graph for era
+        era_graph = pynutil.insert("era: \"") + year_suffix + pynutil.insert("\"") + insert_space
+
+        range_graph = pynini.cross("-", "से")
+
         graph_dd_mm_yyyy = (
             days_graph + (delete_dash | delete_slash) + months_graph + (delete_dash | delete_slash) + years_graph
         )
@@ -78,7 +84,22 @@ def __init__(self, cardinal: GraphFst):
 
         graph_mm_dd_yyyy += pynutil.insert(" preserve_order: true ")
 
-        graph_mm_yyyy = months_graph + delete_dash + years_graph
+        graph_mm_yyyy = (
+            months_graph + (delete_dash | pynini.accep("")) + years_graph + pynutil.insert(" preserve_order: true ")
+        )
+
+        graph_year_suffix = era_graph
+
+        graph_range = (
+            pynutil.insert("text: \"")
+            + (cardinal.final_graph | graph_year)
+            + insert_space
+            + range_graph
+            + insert_space
+            + (cardinal.final_graph | graph_year)
+            + pynutil.insert("\"")
+            + pynutil.insert(" preserve_order: true ")
+        )
 
         # default assume dd_mm_yyyy
 
@@ -88,6 +109,8 @@ def __init__(self, cardinal: GraphFst):
             | pynutil.add_weight(graph_dd_mm_yyyy, -0.001)
             | graph_mm_dd_yyyy
             | graph_mm_yyyy
+            | pynutil.add_weight(graph_year_suffix, -0.001)
+            | pynutil.add_weight(graph_range, -0.005)
         )
 
         self.final_graph = final_graph.optimize()

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -44,16 +44,20 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         )
 
         # Define the unit handling
-        self.unit = pynutil.insert("units: \"") + unit_graph + pynutil.insert("\" ")
+        unit = pynutil.insert("units: \"") + unit_graph + pynutil.insert("\" ")
+
+        # Handling symbols like x, X, *
+        symbol_graph = pynini.string_map([("x", "बाई"), ("X", "बाई"), ("*", "बाई"),])
 
         graph_measurements = (
             pynutil.insert("decimal { ")
             + optional_graph_negative
             + decimal_graph
             + pynutil.insert(" }")
             + delete_space
-            + self.unit
+            + unit
         )
+
         graph_measurements |= (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
@@ -62,7 +66,27 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
             + pynutil.insert("\"")
             + pynutil.insert(" }")
             + delete_space
-            + self.unit
+            + unit
+        )
+
+        # Handling cardinal clubbed with symbol as single token
+        graph_measurements |= (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
+            + pynutil.insert(" }")
+            + pynutil.insert(" units: \"")
+            + symbol_graph
+            + pynutil.insert("\" ")
+            + pynutil.insert("} }")
+            + insert_space
+            + pynutil.insert("tokens { cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
         )
 
         graph = graph_measurements

nemo_text_processing/text_normalization/hi/taggers/money.py

@@ -24,39 +24,35 @@
 class MoneyFst(GraphFst):
     """
     Finite state transducer for classifying money, suppletive aware, e.g.
-        ₹1 -> money { currency: "रुपए" integer_part: "एक" }
-        ₹1.2 -> money { currency: "रुपए" integer_part: "एक" fractional_part: "दो" }
-        
+        ₹५० -> money { money { currency_maj: "रुपए" integer_part: "पचास" }
+        ₹५०.५० -> money { currency_maj: "रुपए" integer_part: "पचास" fractional_part: "पचास" currency_min: "centiles" }
+        ₹०.५० -> money { currency_maj: "रुपए" integer_part: "शून्य" fractional_part: "पचास" currency_min: "centiles" }
+    Note that the 'centiles' string is a placeholder to handle by the verbalizer by applying the corresponding minor currency denomination
+
     Args:
         cardinal: CardinalFst
         decimal: DecimalFst
         deterministic: if True will provide a single transduction option,
             for False multiple transduction are generated (used for audio-based normalization)
     """
 
-    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+    def __init__(self, cardinal: GraphFst):
         super().__init__(name="money", kind="classify")
 
         cardinal_graph = cardinal.final_graph
 
-        optional_graph_negative = pynini.closure(
-            pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, 0, 1,
-        )
-        self.currency = pynutil.insert("currency: \"") + currency_graph + pynutil.insert("\" ")
-        self.interger = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\" ")
-        self.fraction = pynutil.insert("fractional_part: \"") + cardinal_graph + pynutil.insert("\" ")
-
-        graph_currencies = optional_graph_negative + self.currency + insert_space + self.interger
-        graph_currencies |= (
-            optional_graph_negative
-            + self.currency
-            + insert_space
-            + self.interger
-            + pynutil.delete(".")
-            + insert_space
-            + self.fraction
+        currency_major = pynutil.insert('currency_maj: "') + currency_graph + pynutil.insert('"')
+        integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
+        fraction = pynutil.insert('fractional_part: "') + cardinal_graph + pynutil.insert('"')
+        currency_minor = pynutil.insert('currency_min: "') + pynutil.insert("centiles") + pynutil.insert('"')
+
+        graph_major_only = currency_major + insert_space + integer
+        graph_major_and_minor = (
+            currency_major + insert_space + integer + pynini.cross(".", " ") + fraction + insert_space + currency_minor
         )
-        graph = graph_currencies
-        self.graph = graph.optimize()
+
+        graph_currencies = graph_major_only | graph_major_and_minor
+
+        graph = graph_currencies.optimize()
         final_graph = self.add_tokens(graph)
         self.fst = final_graph

nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py

@@ -43,7 +43,7 @@ class ClassifyFst(GraphFst):
     Final class that composes all other classification grammars. This class can process an entire sentence including punctuation.
     For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File.
     More details to deployment at NeMo/tools/text_processing_deployment.
-    
+
     Args:
         input_case: accepting either "lower_cased" or "cased" input.
         deterministic: if True will provide a single transduction option,
@@ -68,11 +68,11 @@ def __init__(
             os.makedirs(cache_dir, exist_ok=True)
             whitelist_file = os.path.basename(whitelist) if whitelist else ""
             far_file = os.path.join(
-                cache_dir, f"hi_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far"
+                cache_dir, f"hi_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far",
             )
         if not overwrite_cache and far_file and os.path.exists(far_file):
             self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
-            logging.info(f'ClassifyFst.fst was restored from {far_file}.')
+            logging.info(f"ClassifyFst.fst was restored from {far_file}.")
         else:
             logging.info(f"Creating ClassifyFst grammars.")
 
@@ -107,7 +107,7 @@ def __init__(
             logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes")
 
             start_time = time.time()
-            money = MoneyFst(cardinal=cardinal, decimal=decimal)
+            money = MoneyFst(cardinal=cardinal)
             money_graph = money.fst
             logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes")
 

nemo_text_processing/text_normalization/hi/verbalizers/date.py

@@ -39,6 +39,10 @@ def __init__(self):
 
         year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
 
+        era = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+
+        range = pynutil.delete("text: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+
         graph_dd_mm = day + NEMO_SPACE + month
 
         graph_mm_dd = month + NEMO_SPACE + day
@@ -49,6 +53,10 @@ def __init__(self):
 
         graph_mm_yyyy = month + NEMO_SPACE + year
 
+        graph_era = era
+
+        graph_range = range
+
         optional_preserve_order = pynini.closure(
             pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
             | pynutil.delete("field_order:")
@@ -60,7 +68,7 @@ def __init__(self):
         )
 
         self.graph = (
-            (graph_dd_mm | graph_mm_dd | graph_dd_mm_yyyy | graph_mm_dd_yyyy | graph_mm_yyyy)
+            (graph_dd_mm | graph_mm_dd | graph_dd_mm_yyyy | graph_mm_dd_yyyy | graph_mm_yyyy | graph_era | graph_range)
             + delete_space
             + optional_preserve_order
         )