Es tn romans fix (#98)

* fix es tn roman exceptions

Signed-off-by: Mariana Graterol Fuenmayor <mgrafu@gmail.com>

* update jenkinsfile

Signed-off-by: Mariana Graterol Fuenmayor <mgrafu@gmail.com>

* update eval script for ITN

Signed-off-by: Mariana Graterol Fuenmayor <mgrafu@gmail.com>

* codeql fix

Signed-off-by: Mariana Graterol Fuenmayor <mgrafu@gmail.com>

---------

Signed-off-by: Mariana Graterol Fuenmayor <mgrafu@gmail.com>

Author: mgrafu

Jenkinsfile

@@ -14,7 +14,7 @@ pipeline {
     AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-14-23-0'
-    ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
+    ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-29-23-0'
     ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-13-23-1'
     FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-16-23-1'
     HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'

nemo_text_processing/inverse_text_normalization/run_evaluate.py

@@ -55,7 +55,7 @@ def parse_args():
     if args.lang == 'en':
         from nemo_text_processing.inverse_text_normalization.en.clean_eval_data import filter_loaded_data
     file_path = args.input
-    inverse_normalizer = InverseNormalizer()
+    inverse_normalizer = InverseNormalizer(lang=args.lang)
 
     print("Loading training data: " + file_path)
     training_data = load_files([file_path])

nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv

@@ -0,0 +1,9 @@
+di
+Di
+DI
+mi
+Mi
+MI
+vi
+Vi
+VI

nemo_text_processing/text_normalization/es/taggers/ordinal.py

@@ -131,7 +131,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
 
         # Managing Romanization, excluding words that may be ambiguous
         roman_ordinals = roman_to_int(ordinal_graph)
-        exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi")
+        # exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi")
+        exceptions = pynini.string_file(get_abs_path("data/ordinals/roman_exceptions.tsv"))
         graph_exception = pynini.project(exceptions, 'input')
         roman_ordinals = (pynini.project(roman_ordinals, "input") - graph_exception.arcsort()) @ roman_ordinals