Skip to content

Commit 85f224f

Browse files
anand-nvanand-nv
committed
Fix Jenkinsfile for CI
Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
1 parent ac07488 commit 85f224f

1 file changed

Lines changed: 66 additions & 47 deletions

File tree

‎Jenkinsfile‎

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
pipeline {
22
agent {
33
docker {
4-
image 'tnitn_ci:py310'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
4+
image 'tnitn_ci_3:py312'
5+
args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
66
}
77
}
88
options {
@@ -11,33 +11,27 @@ pipeline {
1111
}
1212
environment {
1313

14-
AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
15-
DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0'
16-
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0'
17-
ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0'
18-
ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0'
19-
FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0'
20-
HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0'
21-
PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
22-
RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
23-
VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
24-
SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
25-
ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0'
26-
IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
27-
HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
28-
MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
29-
JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
30-
HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
31-
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
14+
AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0'
15+
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
16+
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-24-0'
17+
ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
18+
ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
19+
FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
20+
HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0'
21+
PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
22+
RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
23+
VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
24+
SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
25+
ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0'
26+
IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0'
27+
HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
28+
MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
29+
JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
30+
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0'
31+
DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
3232
}
3333
stages {
3434

35-
stage('Add git safe directory'){
36-
steps{
37-
sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH'
38-
sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH'
39-
}
40-
}
4135

4236
stage('PyTorch version') {
4337
steps {
@@ -65,7 +59,10 @@ pipeline {
6559
when {
6660
anyOf {
6761
branch 'main'
62+
branch 'staging/**'
63+
branch 'staging_*'
6864
changeRequest target: 'main'
65+
6966
}
7067
}
7168
failFast true
@@ -97,6 +94,8 @@ pipeline {
9794
when {
9895
anyOf {
9996
branch 'main'
97+
branch 'staging/**'
98+
branch 'staging_*'
10099
changeRequest target: 'main'
101100
}
102101
}
@@ -120,6 +119,8 @@ pipeline {
120119
when {
121120
anyOf {
122121
branch 'main'
122+
branch 'staging/**'
123+
branch 'staging_*'
123124
changeRequest target: 'main'
124125
}
125126
}
@@ -156,7 +157,9 @@ pipeline {
156157
stage('L0: Create AR TN/ITN Grammars') {
157158
when {
158159
anyOf {
159-
branch 'main'
160+
branch 'main'
161+
branch 'staging/**'
162+
branch 'staging_*'
160163
changeRequest target: 'main'
161164
}
162165
}
@@ -179,7 +182,9 @@ pipeline {
179182
stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') {
180183
when {
181184
anyOf {
182-
branch 'main'
185+
branch 'main'
186+
branch 'staging/**'
187+
branch 'staging_*'
183188
changeRequest target: 'main'
184189
}
185190
}
@@ -216,7 +221,9 @@ pipeline {
216221
stage('L0: Create RU TN/ITN Grammars & SV & PT') {
217222
when {
218223
anyOf {
219-
branch 'main'
224+
branch 'main'
225+
branch 'staging/**'
226+
branch 'staging_*'
220227
changeRequest target: 'main'
221228
}
222229
}
@@ -258,7 +265,9 @@ pipeline {
258265
stage('L0: Create HY TN/ITN Grammars & MR') {
259266
when {
260267
anyOf {
261-
branch 'main'
268+
branch 'main'
269+
branch 'staging/**'
270+
branch 'staging_*'
262271
changeRequest target: 'main'
263272
}
264273
}
@@ -284,7 +293,9 @@ pipeline {
284293
stage('L0: Create ZH TN/ITN Grammar') {
285294
when {
286295
anyOf {
287-
branch 'main'
296+
branch 'main'
297+
branch 'staging/**'
298+
branch 'staging_*'
288299
changeRequest target: 'main'
289300
}
290301
}
@@ -305,7 +316,9 @@ pipeline {
305316
stage('L0: Create JA ITN Grammars') {
306317
when {
307318
anyOf {
308-
branch 'main'
319+
branch 'main'
320+
branch 'staging/**'
321+
branch 'staging_*'
309322
changeRequest target: 'main'
310323
}
311324
}
@@ -325,7 +338,9 @@ pipeline {
325338
stage('L1: TN/ITN Tests CPU') {
326339
when {
327340
anyOf {
328-
branch 'main'
341+
branch 'main'
342+
branch 'staging/**'
343+
branch 'staging_*'
329344
changeRequest target: 'main'
330345
}
331346
}
@@ -412,7 +427,9 @@ pipeline {
412427
stage('L2: Sparrowhawk Tests') {
413428
when {
414429
anyOf {
415-
branch 'main'
430+
branch 'main'
431+
branch 'staging/**'
432+
branch 'staging_*'
416433
changeRequest target: 'main'
417434
}
418435
}
@@ -445,50 +462,52 @@ pipeline {
445462
stage('L2: NeMo text processing') {
446463
when {
447464
anyOf {
448-
branch 'main'
465+
branch 'main'
466+
branch 'staging/**'
467+
branch 'staging_*'
449468
changeRequest target: 'main'
450469
}
451470
}
452471
failFast true
453472
parallel {
454473
stage('L2: Eng TN') {
455474
steps {
456-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \
475+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && \
457476
cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
458-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
459-
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
477+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
478+
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkins/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
460479
cat $NORM_OUTPUT_DIR/test.pynini.txt && \
461-
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
480+
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
462481
rm -rf $NORM_OUTPUT_DIR'
463482
}
464483
}
465484

466485
stage('L2: Eng ITN export') {
467486
steps {
468-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \
487+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \
469488
cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
470-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
471-
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
472-
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
489+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
490+
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkins/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
491+
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
473492
rm -rf $DENORM_OUTPUT_DIR'
474493
}
475494
}
476495

477496

478497
stage('L2: Eng alignment TN') {
479498
steps {
480-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
499+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
481500
cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \
482-
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
501+
cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \
483502
rm -rf $NORM_OUTPUT_DIR'
484503
}
485504
}
486505

487506
stage('L2: Eng alignment ITN') {
488507
steps {
489-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
508+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
490509
cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \
491-
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
510+
cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \
492511
rm -rf $DENORM_OUTPUT_DIR'
493512
}
494513
}

0 commit comments

Comments
 (0)