Merge branch 'main' into staging_hi_itn

Signed-off-by: Mariana <47233618+mgrafu@users.noreply.github.com>

Author: mgrafu

Jenkinsfile

@@ -10,25 +10,24 @@ pipeline {
     disableConcurrentBuilds(abortPrevious: true)
   }
   environment {
-
-    AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
-    DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0'
-    EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0'
-    ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0'
-    ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0'
-    FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0'
-    HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0'
-    PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
-    IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
-    HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
-    MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
-    JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
-    HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-01-25-1'
-    DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
+    AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0'
+    DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
+    EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-25-0'
+    ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
+    ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
+    FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
+    HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0'
+    PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
+    RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
+    VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
+    SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
+    ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0'
+    IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0'
+    HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
+    MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
+    JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0'
+    DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {
 

nemo_text_processing/text_normalization/en/taggers/electronic.py

@@ -127,14 +127,15 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
 
         full_stop_accep = pynini.accep(".")
         dollar_accep = pynini.accep("$")  # Include for the correct transduction of the money graph
-        excluded_symbols = full_stop_accep | dollar_accep
+        excluded_symbols = full_stop_accep | dollar_accep | pynini.accep(",")
         filtered_symbols = pynini.difference(accepted_symbols, excluded_symbols)
         accepted_characters = NEMO_ALPHA | NEMO_DIGIT | filtered_symbols
         domain_component = full_stop_accep + pynini.closure(accepted_characters, 2)
-        graph_domain = (
+        graph_domain = pynutil.add_weight(
             pynutil.insert('domain: "')
             + (pynini.closure(accepted_characters, 1) + pynini.closure(domain_component, 1))
-            + pynutil.insert('"')
+            + pynutil.insert('"'),
+            0.1,
         ).optimize()
 
         graph |= pynutil.add_weight(graph_domain, MIN_NEG_WEIGHT)

tests/nemo_text_processing/en/data_text_normalization/test_cases_electronic.txt

@@ -41,4 +41,5 @@ https://www.nvidia.com/dgx-basepod/~HTTPS colon slash slash WWW dot NVIDIA dot c
 i can use your card ending in 8876~i can use your card ending in eight eight seven six
 upgrade/update~upgrade slash update
 upgrade / update~upgrade slash update
-upgrade/update/downgrade~upgrade slash update slash downgrade
+upgrade/update/downgrade~upgrade slash update slash downgrade
+5.4, or 5.5~five point four, or five point five