Skip to content

Commit 8e56168

Browse files
tbartley94tbartley94
committed
missing jenkins
Signed-off-by: tbartley94 <tbartley@nvidia.com>
1 parent 71513b5 commit 8e56168

1 file changed

Lines changed: 53 additions & 63 deletions

File tree

‎Jenkinsfile‎

Lines changed: 53 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
pipeline {
22
agent {
3-
docker {
4-
image 'tnitn_ci_py310:24.07'
5-
args '-v /mnt/jenkins/jenkinsci/TestData:/home/jenkins/TestData -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
6-
}
3+
docker {
4+
image 'tnitn_ci_py310:24.07'
5+
args '-v /mnt/jenkins/jenkinsci/TestData:/home/jenkins/TestData -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
6+
}
77
}
88
options {
99
timeout(time: 2, unit: 'HOURS')
@@ -33,7 +33,6 @@ pipeline {
3333
}
3434
stages {
3535

36-
3736
stage('PyTorch version') {
3837
steps {
3938
sh 'python -c "import torch; print(torch.__version__)"'
@@ -47,15 +46,13 @@ pipeline {
4746
}
4847
}
4948

50-
5149
stage('L0: Create EN TN/ITN Grammars') {
5250
when {
5351
anyOf {
5452
branch 'main'
5553
branch 'staging/**'
5654
branch 'staging_*'
5755
changeRequest target: 'main'
58-
5956
}
6057
}
6158
failFast true
@@ -80,31 +77,30 @@ pipeline {
8077
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir ${EN_TN_CACHE}'
8178
}
8279
}
83-
8480
}
8581
}
82+
8683
stage('L0: Create HI TN/ITN Grammars') {
87-
when {
84+
when {
8885
anyOf {
89-
branch 'main'
90-
branch 'staging/**'
91-
branch 'staging_*'
92-
changeRequest target: 'main'
86+
branch 'main'
87+
branch 'staging/**'
88+
branch 'staging_*'
89+
changeRequest target: 'main'
9390
}
94-
}
95-
failFast true
96-
parallel {
91+
}
92+
failFast true
93+
parallel {
9794
stage('L0: Hi TN grammars') {
98-
steps {
99-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hi --text="१" --cache_dir ${HI_TN_CACHE}'
100-
}
95+
steps {
96+
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hi --text="१" --cache_dir ${HI_TN_CACHE}'
97+
}
10198
}
10299
stage('L0: Hi ITN grammars') {
103-
steps {
104-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hi --text="एक" --cache_dir ${HI_TN_CACHE}'
105-
}
100+
steps {
101+
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hi --text="एक" --cache_dir ${HI_TN_CACHE}'
102+
}
106103
}
107-
108104
}
109105
}
110106

@@ -150,7 +146,7 @@ pipeline {
150146
stage('L0: Create AR TN/ITN Grammars') {
151147
when {
152148
anyOf {
153-
branch 'main'
149+
branch 'main'
154150
branch 'staging/**'
155151
branch 'staging_*'
156152
changeRequest target: 'main'
@@ -168,14 +164,13 @@ pipeline {
168164
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ar --text="اثنان " --cache_dir ${AR_TN_CACHE}'
169165
}
170166
}
171-
172167
}
173168
}
174169

175170
stage('L0: Create FR TN/ITN & VI TN/ITN & HU TN & IT TN') {
176171
when {
177172
anyOf {
178-
branch 'main'
173+
branch 'main'
179174
branch 'staging/**'
180175
branch 'staging_*'
181176
changeRequest target: 'main'
@@ -204,7 +199,7 @@ pipeline {
204199
}
205200
}
206201
stage('L0: HU TN grammars') {
207-
steps {
202+
steps {
208203
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}'
209204
}
210205
}
@@ -219,7 +214,7 @@ pipeline {
219214
stage('L0: Create RU TN/ITN Grammars & SV & PT') {
220215
when {
221216
anyOf {
222-
branch 'main'
217+
branch 'main'
223218
branch 'staging/**'
224219
branch 'staging_*'
225220
changeRequest target: 'main'
@@ -238,31 +233,32 @@ pipeline {
238233
}
239234
}
240235
stage('L0: SV TN grammars') {
241-
steps {
236+
steps {
242237
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=sv --text="100" --cache_dir ${SV_TN_CACHE}'
243238
}
244239
}
245-
// stage('L0: SV ITN grammars') {
246-
// steps {
247-
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=sv --text="hundra " --cache_dir ${SV_TN_CACHE}'
248-
// }
249-
// }
250-
// stage('L0: PT TN grammars') {
251-
// steps {
252-
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=pt --text="2" --cache_dir ${DEFAULT_TN_CACHE}'
253-
// }
254-
// }
240+
// stage('L0: SV ITN grammars') {
241+
// steps {
242+
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=sv --text="hundra " --cache_dir ${SV_TN_CACHE}'
243+
// }
244+
// }
245+
// stage('L0: PT TN grammars') {
246+
// steps {
247+
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=pt --text="2" --cache_dir ${DEFAULT_TN_CACHE}'
248+
// }
249+
// }
255250
stage('L0: PT ITN grammars') {
256251
steps {
257252
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=pt --text="dez " --cache_dir ${PT_TN_CACHE}'
258253
}
259254
}
260255
}
261256
}
262-
stage('L0: Create He TN/ITN Grammars & MR') {
257+
258+
stage('L0: Create He TN/ITN Grammars & MR') {
263259
when {
264260
anyOf {
265-
branch 'main'
261+
branch 'main'
266262
branch 'staging/**'
267263
branch 'staging_*'
268264
changeRequest target: 'main'
@@ -277,10 +273,11 @@ pipeline {
277273
}
278274
}
279275
}
276+
280277
stage('L0: Create HY TN/ITN Grammars & MR') {
281278
when {
282279
anyOf {
283-
branch 'main'
280+
branch 'main'
284281
branch 'staging/**'
285282
branch 'staging_*'
286283
changeRequest target: 'main'
@@ -305,10 +302,11 @@ pipeline {
305302
}
306303
}
307304
}
305+
308306
stage('L0: Create ZH TN/ITN Grammar') {
309307
when {
310308
anyOf {
311-
branch 'main'
309+
branch 'main'
312310
branch 'staging/**'
313311
branch 'staging_*'
314312
changeRequest target: 'main'
@@ -328,10 +326,11 @@ pipeline {
328326
}
329327
}
330328
}
329+
331330
stage('L0: Create JA ITN Grammars') {
332331
when {
333332
anyOf {
334-
branch 'main'
333+
branch 'main'
335334
branch 'staging/**'
336335
branch 'staging_*'
337336
changeRequest target: 'main'
@@ -346,13 +345,14 @@ pipeline {
346345
}
347346
}
348347
}
348+
349349
stage('L0: Create KO TN Grammars') {
350350
when {
351351
anyOf {
352352
branch 'main'
353353
changeRequest target: 'main'
354354
}
355-
}
355+
}
356356
failFast true
357357
parallel {
358358
stage('L0: KO TN grammars') {
@@ -363,13 +363,12 @@ pipeline {
363363
}
364364
}
365365

366-
367-
// L1 Tests starts here
366+
// L1 Tests starts here
368367

369368
stage('L1: TN/ITN Tests CPU') {
370369
when {
371370
anyOf {
372-
branch 'main'
371+
branch 'main'
373372
branch 'staging/**'
374373
branch 'staging_*'
375374
changeRequest target: 'main'
@@ -441,7 +440,7 @@ pipeline {
441440
steps {
442441
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ja/ -m "not pleasefixme" --cpu --tn_cache_dir ${JA_TN_CACHE}'
443442
}
444-
}
443+
}
445444
stage('L1: Run all MR ITN tests (restore grammars from cache)') {
446445
steps {
447446
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/mr/ -m "not pleasefixme" --cpu --tn_cache_dir ${MR_TN_CACHE}'
@@ -457,18 +456,18 @@ pipeline {
457456
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/he/ -m "not pleasefixme" --cpu --tn_cache_dir ${HE_TN_CACHE}'
458457
}
459458
}
460-
stage('L1: Run all KO TN/ITN tests (restore grammars from cache)') {
459+
stage('L1: Run all KO TN/ITN tests (restore grammars from cache)') {
461460
steps {
462461
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}'
463462
}
464463
}
465464
}
466465
}
467466

468-
stage('L2: EN Sparrowhawk Tests') {
467+
stage('L2: EN Sparrowhawk Tests') {
469468
when {
470469
anyOf {
471-
branch 'main'
470+
branch 'main'
472471
branch 'staging/**'
473472
branch 'staging_*'
474473
changeRequest target: 'main'
@@ -480,14 +479,12 @@ pipeline {
480479
steps {
481480
sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"'
482481
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased `pwd`'
483-
484482
}
485483
}
486484
stage('L2: EN ITN Run Sparrowhawk test - Cased Input') {
487485
steps {
488486
sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"'
489487
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased `pwd`'
490-
491488
}
492489
}
493490
stage('L2: EN TN Run Sparrowhawk test') {
@@ -496,14 +493,13 @@ pipeline {
496493
sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased `pwd`'
497494
}
498495
}
499-
500496
}
501497
}
502-
498+
503499
stage('L2: NeMo text processing') {
504500
when {
505501
anyOf {
506-
branch 'main'
502+
branch 'main'
507503
branch 'staging/**'
508504
branch 'staging_*'
509505
changeRequest target: 'main'
@@ -522,7 +518,6 @@ pipeline {
522518
rm -rf $NORM_OUTPUT_DIR'
523519
}
524520
}
525-
526521
stage('L2: Eng ITN export') {
527522
steps {
528523
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \
@@ -533,8 +528,6 @@ pipeline {
533528
rm -rf $DENORM_OUTPUT_DIR'
534529
}
535530
}
536-
537-
538531
stage('L2: Eng alignment TN') {
539532
steps {
540533
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
@@ -543,7 +536,6 @@ pipeline {
543536
rm -rf $NORM_OUTPUT_DIR'
544537
}
545538
}
546-
547539
stage('L2: Eng alignment ITN') {
548540
steps {
549541
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
@@ -552,12 +544,10 @@ pipeline {
552544
rm -rf $DENORM_OUTPUT_DIR'
553545
}
554546
}
555-
556547
}
557548
}
558549
}
559550

560-
561551
post {
562552
always {
563553
sh 'chmod -R 777 .'

0 commit comments

Comments
 (0)