Skip to content

Commit e497dd3

Browse files
committed
Simple Jenkinsfile for debug
Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
1 parent c7fa942 commit e497dd3

1 file changed

Lines changed: 7 additions & 128 deletions

File tree

Jenkinsfile

Lines changed: 7 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -1,131 +1,10 @@
11
pipeline {
2-
agent {
3-
docker {
4-
image 'tnitn_ci2:py312'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
2+
agent any
3+
stages {
4+
stage('Print hello') {
5+
steps {
6+
echo 'Hello world!'
7+
}
68
}
7-
}
8-
options {
9-
timeout(time: 2, unit: 'HOURS')
10-
disableConcurrentBuilds(abortPrevious: true)
11-
}
12-
environment {
13-
14-
EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-06-25-1'
15-
DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0'
16-
17-
}
18-
stages {
19-
20-
stage('Add git safe directory'){
21-
steps{
22-
sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH'
23-
sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH'
24-
}
259
}
26-
27-
stage('PyTorch version') {
28-
steps {
29-
sh 'python -c "import torch; print(torch.__version__)"'
30-
sh 'python -c "import torchvision; print(torchvision.__version__)"'
31-
}
32-
}
33-
34-
stage('NeMo Installation') {
35-
steps {
36-
sh './reinstall.sh release'
37-
}
38-
}
39-
40-
41-
42-
stage('L0: Create EN TN/ITN Grammars') {
43-
when {
44-
anyOf {
45-
branch 'main'
46-
changeRequest target: 'main'
47-
}
48-
}
49-
failFast true
50-
parallel {
51-
stage('L0: En TN grammars') {
52-
steps {
53-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir ${EN_TN_CACHE}'
54-
}
55-
}
56-
stage('L0: En TN non-deterministic grammars') {
57-
steps {
58-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --text="1" --cache_dir ${EN_TN_CACHE}'
59-
}
60-
}
61-
stage('L0: En ITN grammars') {
62-
steps {
63-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir ${EN_TN_CACHE}'
64-
}
65-
}
66-
67-
}
68-
}
69-
70-
// L1 Tests starts here
71-
stage('L1: TN/ITN Tests CPU') {
72-
when {
73-
anyOf {
74-
branch 'main'
75-
changeRequest target: 'main'
76-
}
77-
}
78-
failFast true
79-
stages {
80-
stage('L1: Test EN non-deterministic TN & Run all En TN/ITN tests (restore grammars from cache)') {
81-
steps {
82-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir ${EN_TN_CACHE}'
83-
}
84-
}
85-
}
86-
}
87-
88-
stage('L2: NeMo text processing') {
89-
when {
90-
anyOf {
91-
branch 'main'
92-
changeRequest target: 'main'
93-
}
94-
}
95-
failFast true
96-
parallel {
97-
stage('L2: Eng TN') {
98-
steps {
99-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \
100-
cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
101-
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
102-
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
103-
cat $NORM_OUTPUT_DIR/test.pynini.txt && \
104-
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
105-
rm -rf $NORM_OUTPUT_DIR'
106-
}
107-
}
108-
109-
stage('L2: Eng ITN export') {
110-
steps {
111-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \
112-
cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
113-
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
114-
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
115-
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
116-
rm -rf $DENORM_OUTPUT_DIR'
117-
}
118-
}
119-
120-
}
121-
}
122-
}
123-
124-
125-
post {
126-
always {
127-
sh 'chmod -R 777 .'
128-
cleanWs()
129-
}
130-
}
131-
}
10+
}

0 commit comments

Comments
 (0)