Fix space issue with ZH ITN (#244)

* Fix space issue with ZH ITN

Signed-off-by: Anand Joseph <anajoseph@nvidia.com>

* Update Jenkinsfile

Update FST paths

Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com>

---------

Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com>
Signed-off-by: Simon Zuberek <simon@zuberek.net>
Co-authored-by: Anand Joseph <anajoseph@nvidia.com>
Co-authored-by: anand-nv <105917641+anand-nv@users.noreply.github.com>

Author: 2 people

Jenkinsfile

@@ -22,7 +22,7 @@ pipeline {
     RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-30-24-0'
+    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
     IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'

nemo_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py

@@ -21,6 +21,7 @@
     GraphFst,
     delete_extra_space,
     delete_space,
+    delete_zero_or_one_space,
     generator_main,
 )
 from nemo_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst
@@ -92,12 +93,12 @@ def __init__(
             )
 
             punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }")
-            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ")
             token_plus_punct = (
                 pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct)
             )
 
-            graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
+            graph = token_plus_punct + pynini.closure(delete_zero_or_one_space + token_plus_punct)
             graph = delete_space + graph + delete_space
 
             self.fst = graph.optimize()

nemo_text_processing/inverse_text_normalization/zh/taggers/word.py

@@ -26,5 +26,5 @@ class WordFst(GraphFst):
 
     def __init__(self):
         super().__init__(name="word", kind="classify")
-        word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"')
+        word = pynutil.insert('name: "') + NEMO_NOT_SPACE + pynutil.insert('"')
         self.fst = word.optimize()

nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py

@@ -40,5 +40,5 @@ def __init__(self):
             + delete_space
             + pynutil.delete("}")
         )
-        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space
         self.fst = graph