PR: Add Vietnamese text normalization for cardinal semiotic class (#289)

* Add Vietnamese text normalization for cardinal semiotic class

Signed-off-by: folivoramanh <palasek182@gmail.com>

* Add missing init file

Signed-off-by: folivoramanh <palasek182@gmail.com>

* Fix Cardinal and optimize logic

Signed-off-by: folivoramanh <palasek182@gmail.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: folivoramanh <palasek182@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: folivoramanh

Jenkinsfile

@@ -176,7 +176,7 @@ pipeline {
       }
     }
 
-    stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') {
+    stage('L0: Create FR TN/ITN & VI TN/ITN & HU TN & IT TN') {
       when {
         anyOf {
           branch 'main'
@@ -200,6 +200,11 @@ pipeline {
             sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=vi --text="một ngàn " --cache_dir ${VI_TN_CACHE}'
           }
         }
+        stage('L0: VI TN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=vi --text="100" --cache_dir ${VI_TN_CACHE}'
+          }
+        }
         stage('L0: HU TN grammars') {
          steps {
             sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}'

nemo_text_processing/text_normalization/normalize.py

@@ -174,6 +174,9 @@ def __init__(
         elif lang == 'ja':
             from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst
+        elif lang == 'vi':
+            from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst
+            from nemo_text_processing.text_normalization.vi.verbalizers.verbalize_final import VerbalizeFinalFst
         else:
             raise NotImplementedError(f"Language {lang} has not been supported yet.")
 

nemo_text_processing/text_normalization/vi/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/vi/data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/vi/data/numbers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+1	một
+2	hai
+3	ba
+4	bốn
+5	năm
+6	sáu
+7	bảy
+8	tám
+9	chín

nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv

@@ -0,0 +1,3 @@
+1	một	mốt
+4	bốn	tư
+5	năm	lăm 

nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv

@@ -0,0 +1,5 @@
+thousand	nghìn
+million	triệu
+billion	tỷ
+hundred	trăm
+linh	linh

nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv

@@ -0,0 +1,10 @@
+10	mười
+11	mười một
+12	mười hai
+13	mười ba
+14	mười bốn
+15	mười lăm
+16	mười sáu
+17	mười bảy
+18	mười tám
+19	mười chín

nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv

@@ -0,0 +1,8 @@
+2	hai mươi
+3	ba mươi
+4	bốn mươi
+5	năm mươi
+6	sáu mươi
+7	bảy mươi
+8	tám mươi
+9	chín mươi