Merge EN riva release 22.10 (#26)

* Merge EN riva release 22.10

Signed-off-by: Anand Joseph <anajoseph@nvidia.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Code cleanup

Signed-off-by: Anand Joseph <anajoseph@nvidia.com>

---------

Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: anand-nv

.gitignore

@@ -174,4 +174,4 @@ examples/neural_graphs/*.yml
 
 .hydra/
 nemo_experiments/
-
+*.swp

nemo_text_processing/inverse_text_normalization/en/data/currency.tsv

@@ -1,35 +1,168 @@
-$	dollar
-$	us dollar
-$	united states dollar
+؋	afghan afghani
+l	albanian lek
+دج	algerian dinar
+kz	angolan kwanza
+aed	arab emirates dirham
+$	argentine peso
+֏	armenian dram
+ƒ	aruban florin
+a$	australian dollar
+₼	azerbaijani manat
+b$	bahamian dollar
+.د.ب	bahraini dinar
+৳	bangladeshi taka
+$	barbadian dollar
+br	belarusian ruble
+bz$	belize dollar
+$	bermudian dollar
+nu	bhutanese ngultrum
+₿	bitcoin
+bs	bolivian boliviano
+km	bosnia and herzegovina convertible mark
+p	botswana pula
+r$	brazilian real
 £	british pound
+b$	brunei dollar
+лв	bulgarian lev
+fbu	burundian franc
+៛	cambodian riel
+ca$	canadian dollar
+cve	cape verde escudo
+cve	cape verdean escudo
+ci$	cayman islands dollar
+cfa	cfa franc
+$	chilean peso
+元	chinese yuan
+$	colombian peso
+cf	comorian franc
+fc	congolese franc
+₡	costa rican colón
+kn	croatian kuna
+cuc$	cuban peso
+cyp	cypriot pound
+kč	czech koruna
+d	dalasi
+kr	danish krone
+fdj	djiboutian franc
+$	dollar
+rd$	dominican peso
+$	east caribbean dollar
+e£	egyptian pound
+aed	emirati dirham
+nkf	eritrean nakfa
+Ξ	ethereum
+br	ethiopian birr
 €	euro
-₩	won
-nzd	new zealand dollar
-rs	rupee
-chf	swiss franc
-dkk	danish kroner
+fk£	falkland islands pound
 fim	finnish markka
-aed	arab emirates dirham
-¥	yen
-czk	czech koruna
+₾	georgian lari
+dm	german mark
+gh₵	ghanaian cedi
+q	guatemalan quetzal
+fg	guinean franc
+g$	guyanese dollar
+g	haitian gourde
+l	honduran lempira
+hk$	hong kong dollar
+ft	hungarian forint
+₹	inr
+íkr	icelandic króna
+₹	indian rupee
+rp	indonesian rupiah
+﷼	iranian rial
+ع.د	iraqi dinar
+₪	israeli new shekel
+₪	israeli shekel
+j$	jamaican dollar
+¥	japanese yen
+ينار	jordanian dinar
+₸	kazakhstani tenge
+ksh	kenyan shilling
+ك	kuwaiti dinar
+k	kyat
+som	kyrgyzstani som
+₭	lao kip
+ლარი	lari
+ل.ل	lebanese pound
+l	lesotho loti
+ld$	liberian dollar
+ld	libyan dinar
+l	lilangeni
+ł	litecoin
+mop$	macanese pataca
+ar	malagasy ariary
+k	malawian kwacha
+rm	malaysian ringgit
+rf	maldivian rufiyaa
 mro	mauritanian ouguiya
+₨	mauritian rupee
+$	mexican peso
+l	moldovan leu
+ɱ	monero
+₮	mongolian tögrög
+dh	moroccan dirham
+mt	mozambican metical
+n$	namibian dollar
+rs	nepalese rupee
+ƒ	netherlands antillean guilder
+nt$	new taiwan dollar
+nz$	new zealand dollar
+c$	nicaraguan córdoba
+₦	nigerian naira
+₩	north korean won
+kr	norwegian krone
+ر.ع	omani rial
+um	ouguiya
 pkr	pakistani rupee
-crc costa rican colon
-hk$	hong kong dollar
-npr	nepalese rupee
-awg aruban florin
-nok	norwegian kroner
-tzs tanzanian shilling
-sek swedish kronor
-cyp	cypriot pound
+b/.	panamanian balboa
+₲	paraguayan guaraní
+s/.	peruvian sol
+₱	philippine peso
+zł	polish zloty
+£	pounds sterling
+ر.ق	qatari riyal
+£	quid
 r	real
+xrp	ripples
+lei	romanian leu
+₹	rupee
+₽	russian ruble
+r₣	rwandan franc
+shp	saint helena pounds
+db	são tomé and príncipe dobra
 sar	saudi riyal
-cve	cape verde escudo
+ден	second macedonian denar
 rsd	serbian dinar
-dm german mark
-shp saint helena pounds
-php	philippine peso
-cad	canadian dollar
-ssp	south sudanese pound
 scr	seychelles rupee
-mvr	maldivian rufiyaa
+le	sierra leonean leone
+s$	singapore dollar
+sh.so.	somali shilling
+tjs	somoni
+r	south african rand
+₩	south korean won
+ss£	south sudanese pound
+රු	sri lankan rupee
+sdg	sudanese pound
+sr$	surinamese dollar
+kr	swedish krona
+chf	swiss franc
+£s	syrian pound
+৳	taka
+tzs	tanzanian shilling
+₮	tether
+฿	thai baht
+tt$	trinidad and tobago dollar
+د.ت	tunisian dinar
+₺	turkish lira
+m	turkmen new manat
+ush	ugandan shilling
+₴	ukrainian hryvna
+$	united states dollar
+$u	uruguayan peso
+$	us dollar
+som	uzbekistan som
+bs.	venezuelan bolívar
+₫	vietnamese đồng
+₩	won
+﷼	yemeni rial
+¥	yen

nemo_text_processing/inverse_text_normalization/en/data/electronic/url_symbols.tsv

@@ -0,0 +1,5 @@
+.	dot
+-	dash
+-	hyphen
+_	underscore
+/	slash

nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import pynini
 from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path, num_to_word
 from nemo_text_processing.text_normalization.en.graph_utils import (
@@ -31,7 +30,7 @@ class CardinalFst(GraphFst):
     """
     Finite state transducer for classifying cardinals
         e.g. minus twenty three -> cardinal { integer: "23" negative: "-" } }
-    Numbers below thirteen are not converted. 
+    Numbers below thirteen are not converted.
     """
 
     def __init__(self):
@@ -67,6 +66,17 @@ def __init__(self):
 
         graph_hundreds = graph_hundred_component | graph_hundred_as_thousand
 
+        graph_ties_component = pynini.union(
+            graph_teen | pynutil.insert("00"),
+            (graph_ties | pynutil.insert("0")) + delete_space + (graph_digit | pynutil.insert("0")),
+        )
+
+        graph_ties_component_at_least_one_none_zero_digit = graph_ties_component @ (
+            pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)
+        )
+        self.graph_ties_component_at_least_one_none_zero_digit = graph_ties_component_at_least_one_none_zero_digit
+
+        # %%% International numeric format
         graph_thousands = pynini.union(
             graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("thousand"),
             pynutil.insert("000", weight=0.1),
@@ -96,8 +106,9 @@ def __init__(self):
             graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("sextillion"),
             pynutil.insert("000", weight=0.1),
         )
+        # %%%
 
-        graph = pynini.union(
+        graph_int = (
             graph_sextillion
             + delete_space
             + graph_quintillion
@@ -111,11 +122,84 @@ def __init__(self):
             + graph_million
             + delete_space
             + graph_thousands
+        )
+
+        # %% Indian numeric format simple https://en.wikipedia.org/wiki/Indian_numbering_system
+        # This only covers "standard format".
+        # Conventional format like thousand crores/lakh crores is yet to be implemented
+        graph_in_thousands = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("thousand"),
+            pynutil.insert("00", weight=0.1),
+        )
+        graph_in_lakhs = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("lakh") | pynutil.delete("lakhs")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_in_crores = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("crore") | pynutil.delete("crores")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_in_arabs = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("arab") | pynutil.delete("arabs")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_in_kharabs = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
             + delete_space
-            + graph_hundreds,
-            graph_zero,
+            + (pynutil.delete("kharab") | pynutil.delete("kharabs")),
+            pynutil.insert("00", weight=0.1),
         )
 
+        graph_in_nils = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("nil") | pynutil.delete("nils")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_in_padmas = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("padma") | pynutil.delete("padmas")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_in_shankhs = pynini.union(
+            graph_ties_component_at_least_one_none_zero_digit
+            + delete_space
+            + (pynutil.delete("shankh") | pynutil.delete("shankhs")),
+            pynutil.insert("00", weight=0.1),
+        )
+
+        graph_ind = (
+            graph_in_shankhs
+            + delete_space
+            + graph_in_padmas
+            + delete_space
+            + graph_in_nils
+            + delete_space
+            + graph_in_kharabs
+            + delete_space
+            + graph_in_arabs
+            + delete_space
+            + graph_in_crores
+            + delete_space
+            + graph_in_lakhs
+            + delete_space
+            + graph_in_thousands
+        )
+
+        graph = pynini.union((graph_int | graph_ind) + delete_space + graph_hundreds, graph_zero,)
+
         graph = graph @ pynini.union(
             pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0"
         )

nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py

@@ -35,9 +35,8 @@ def __init__(self):
             | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
         )
 
-        symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert()
-
-        accepted_username = alpha_num | symbols
+        url_symbols = pynini.string_file(get_abs_path("data/electronic/url_symbols.tsv")).invert()
+        accepted_username = alpha_num | url_symbols
         process_dot = pynini.cross("dot", ".")
         username = (alpha_num + pynini.closure(delete_extra_space + accepted_username)) | pynutil.add_weight(
             pynini.closure(NEMO_ALPHA, 1), weight=0.0001
@@ -65,7 +64,7 @@ def __init__(self):
         # .com,
         ending = (
             delete_extra_space
-            + symbols
+            + url_symbols
             + delete_extra_space
             + (domain | pynini.closure(accepted_username + delete_extra_space,) + accepted_username)
         )

nemo_text_processing/inverse_text_normalization/en/taggers/money.py

@@ -52,13 +52,14 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         )
         cardinal_graph |= with_hundred
         graph_decimal_final = decimal.final_graph_wo_negative
-
         unit = pynini.string_file(get_abs_path("data/currency.tsv"))
         unit_singular = pynini.invert(unit)
         unit_plural = get_singulars(unit_singular)
 
         graph_unit_singular = pynutil.insert("currency: \"") + convert_space(unit_singular) + pynutil.insert("\"")
-        graph_unit_plural = pynutil.insert("currency: \"") + convert_space(unit_plural) + pynutil.insert("\"")
+        graph_unit_plural = (
+            pynutil.insert("currency: \"") + convert_space(unit_plural | unit_singular) + pynutil.insert("\"")
+        )
 
         add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
         # twelve dollars (and) fifty cents, zero cents