Skip to content

Commit c7fa942

Browse files
anand-nvanand-nv
committed
Update Jenkinfile
Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
1 parent 5c15fe2 commit c7fa942

1 file changed

Lines changed: 7 additions & 89 deletions

File tree

‎Jenkinsfile‎

Lines changed: 7 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,19 @@
11
pipeline {
22
agent {
33
docker {
4-
image 'tnitn_ci:py312'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=8g --entrypoint=""'
4+
image 'tnitn_ci2:py312'
5+
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
66
}
77
}
88
options {
9-
timeout(time: 4, unit: 'HOURS')
9+
timeout(time: 2, unit: 'HOURS')
1010
disableConcurrentBuilds(abortPrevious: true)
1111
}
1212
environment {
1313

14-
AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
15-
DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0'
16-
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0'
17-
ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0'
18-
ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0'
19-
FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0'
20-
HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0'
21-
PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
22-
RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
23-
VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
24-
SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
25-
ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
26-
IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
27-
HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
28-
MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
29-
JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
30-
HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
31-
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
14+
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-06-25-1'
15+
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0'
16+
3217
}
3318
stages {
3419

@@ -46,21 +31,14 @@ pipeline {
4631
}
4732
}
4833

49-
stage('Install test requirements') {
50-
steps {
51-
sh 'apt-get update && apt-get install -y bc'
52-
}
53-
}
54-
55-
56-
5734
stage('NeMo Installation') {
5835
steps {
5936
sh './reinstall.sh release'
6037
}
6138
}
6239

6340

41+
6442
stage('L0: Create EN TN/ITN Grammars') {
6543
when {
6644
anyOf {
@@ -70,11 +48,6 @@ pipeline {
7048
}
7149
failFast true
7250
parallel {
73-
stage('L0: Test utils') {
74-
steps {
75-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/audio_based_utils/ --cpu'
76-
}
77-
}
7851
stage('L0: En TN grammars') {
7952
steps {
8053
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir ${EN_TN_CACHE}'
@@ -94,10 +67,7 @@ pipeline {
9467
}
9568
}
9669

97-
98-
9970
// L1 Tests starts here
100-
10171
stage('L1: TN/ITN Tests CPU') {
10272
when {
10373
anyOf {
@@ -115,39 +85,6 @@ pipeline {
11585
}
11686
}
11787

118-
stage('L2: Sparrowhawk Tests') {
119-
when {
120-
anyOf {
121-
branch 'main'
122-
changeRequest target: 'main'
123-
}
124-
}
125-
failFast true
126-
parallel {
127-
stage('L2: EN ITN Run Sparrowhawk test - Lower Cased Input') {
128-
steps {
129-
sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"'
130-
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased `pwd`'
131-
132-
}
133-
}
134-
stage('L2: EN ITN Run Sparrowhawk test - Cased Input') {
135-
steps {
136-
sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"'
137-
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased `pwd`'
138-
139-
}
140-
}
141-
stage('L2: EN TN Run Sparrowhawk test') {
142-
steps {
143-
sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" '
144-
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased `pwd`'
145-
}
146-
}
147-
148-
}
149-
}
150-
15188
stage('L2: NeMo text processing') {
15289
when {
15390
anyOf {
@@ -180,25 +117,6 @@ pipeline {
180117
}
181118
}
182119

183-
184-
stage('L2: Eng alignment TN') {
185-
steps {
186-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
187-
cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \
188-
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
189-
rm -rf $NORM_OUTPUT_DIR'
190-
}
191-
}
192-
193-
stage('L2: Eng alignment ITN') {
194-
steps {
195-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
196-
cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \
197-
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
198-
rm -rf $DENORM_OUTPUT_DIR'
199-
}
200-
}
201-
202120
}
203121
}
204122
}

0 commit comments

Comments
 (0)