Merge branch 'NVIDIA:main' into hebrew_itn

Author: tbartley94

.pre-commit-config.yaml

@@ -22,30 +22,30 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-yaml
       - id: check-case-conflict
       - id: detect-private-key
       - id: requirements-txt-fixer
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.2.0
+    rev: 7.3.0
     hooks:
     -   id: flake8
         args:
         - --select=W605
 
   - repo: https://github.com/PyCQA/isort
-    rev: 6.0.1
+    rev: 6.1.0
     hooks:
       - id: isort
         name: Format imports
         args: [ --multi-line=3, --trailing-comma, --force-grid-wrap=0, --use-parentheses, --line-width=119, -rc, -ws ]
         exclude: docs/
 
-  - repo: https://github.com/psf/black
-    rev: 25.1.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 25.9.0
     hooks:
       - id: black
         name: Format code

Jenkinsfile

@@ -2,18 +2,17 @@ pipeline {
   agent {
         docker {
           image 'tnitn_ci_py310:24.07'
-          args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
+          args '-v /mnt/jenkins/jenkinsci/TestData:/home/jenkins/TestData -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
         }
   }
   options {
     timeout(time: 2, unit: 'HOURS')
     disableConcurrentBuilds(abortPrevious: true)
   }
   environment {
-
     AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0'
     DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
-    EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-24-0'
+    EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-25-0'
     ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
     ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
     FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
@@ -28,7 +27,7 @@ pipeline {
     HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-31-25-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/en/taggers/electronic.py

@@ -127,14 +127,15 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
 
         full_stop_accep = pynini.accep(".")
         dollar_accep = pynini.accep("$")  # Include for the correct transduction of the money graph
-        excluded_symbols = full_stop_accep | dollar_accep
+        excluded_symbols = full_stop_accep | dollar_accep | pynini.accep(",")
         filtered_symbols = pynini.difference(accepted_symbols, excluded_symbols)
         accepted_characters = NEMO_ALPHA | NEMO_DIGIT | filtered_symbols
         domain_component = full_stop_accep + pynini.closure(accepted_characters, 2)
-        graph_domain = (
+        graph_domain = pynutil.add_weight(
             pynutil.insert('domain: "')
             + (pynini.closure(accepted_characters, 1) + pynini.closure(domain_component, 1))
-            + pynutil.insert('"')
+            + pynutil.insert('"'),
+            0.1,
         ).optimize()
 
         graph |= pynutil.add_weight(graph_domain, MIN_NEG_WEIGHT)

nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_list.tsv

@@ -0,0 +1,5 @@
+हफ़्ते
+सप्ताह
+सदियां
+सदियों
+

…tion/hi/data/measure/quarterly_units.tsv …/hi/data/measure/quarterly_units_map.tsvnemo_text_processing/text_normalization/hi/data/measure/quarterly_units.tsv renamed to nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv nemo_text_processing/text_normalization/hi/data/measure/quarterly_units.tsv renamed to nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv

@@ -4,9 +4,8 @@ h	घंटे
 min	मिनट
 doz	दर्जन
 yr	साल
-yr	वर्ष
 hp	हॉर्सपॉवर
 d	दिन
 month	महीना
 months	महीने
-हफ़्ते हफ़्ते
+

nemo_text_processing/text_normalization/hi/data/measure/unit.tsv

@@ -134,7 +134,6 @@ KHz	किलोहर्ट्ज़
 N	न्यूटन
 dB	डेसीबल
 yr	साल
-yr	वर्ष
 hp	हॉर्सपॉवर
 d	दिन
 month	महीना

nemo_text_processing/text_normalization/hi/data/ordinal/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/hi/data/ordinal/exceptions.tsv

@@ -0,0 +1,12 @@
+१ला	पहला
+१ली	पहली
+२रा	दूसरा
+२री	दूसरी
+३रा	तीसरा
+३री	तीसरी
+४था	चौथा
+४थी	चौथी
+५वां	पाँचवां
+५वीं	पाँचवीं
+६ठा	छठा
+६ठी	छठी

nemo_text_processing/text_normalization/hi/data/ordinal/suffixes.tsv

@@ -0,0 +1,3 @@
+वां
+वीं
+वें

nemo_text_processing/text_normalization/hi/data/ordinal/suffixes_map.tsv

@@ -0,0 +1,2 @@
+वे	वें
+