Skip to content

Commit 0748a29

Browse files
ekmbekmb
authored
CI setup (#25)
* restarting ci Signed-off-by: ekmb <ebakhturina@nvidia.com> * restarting ci _cr Signed-off-by: ekmb <ebakhturina@nvidia.com> * revert setup tool Signed-off-by: ekmb <ebakhturina@nvidia.com> * remove pytest-runner from setup.py Signed-off-by: ekmb <ebakhturina@nvidia.com> * fix jenkins Signed-off-by: ekmb <ebakhturina@nvidia.com> * fix jenkins Signed-off-by: ekmb <ebakhturina@nvidia.com> * fix jenkins Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> * update test dir Signed-off-by: ekmb <ebakhturina@nvidia.com> --------- Signed-off-by: ekmb <ebakhturina@nvidia.com>
1 parent fa58b11 commit 0748a29

3 files changed

Lines changed: 20 additions & 16 deletions

File tree

‎Jenkinsfile‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ pipeline {
1414

1515
stage('Add git safe directory'){
1616
steps{
17-
sh 'git config --global user.name "jenkinsci"'
18-
sh 'git config --global user.email "$(whoami)@$(hostname)"'
17+
// sh 'git config --global user.name "jenkinsci"'
18+
// sh 'git config --global user.email "$(whoami)@$(hostname)"'
1919
sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH'
2020
sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH'
2121
}
@@ -55,17 +55,17 @@ pipeline {
5555
parallel {
5656
stage('En TN grammars') {
5757
steps {
58-
sh 'CUDA_VISIBLE_DEVICES="" python NTP_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/jenkinsci/nlp/text_norm/ci/grammars/11-16-22'
58+
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/jenkinsci/TestData/text_norm/ci/grammars/01-30-23'
5959
}
6060
}
6161
stage('En ITN grammars') {
6262
steps {
63-
sh 'CUDA_VISIBLE_DEVICES="" python NTP_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/jenkinsci/nlp/text_norm/ci/grammars/11-16-22'
63+
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/jenkinsci/TestData/text_norm/ci/grammars/01-30-23'
6464
}
6565
}
6666
stage('Test En non-deterministic TN & Run all En TN/ITN tests (restore grammars from cache)') {
6767
steps {
68-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/NTP_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/jenkinsci/nlp/text_norm/ci/grammars/11-16-22'
68+
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/jenkinsci/TestData/text_norm/ci/grammars/01-30-23'
6969
}
7070
}
7171

@@ -83,20 +83,24 @@ pipeline {
8383
parallel {
8484
stage('L2: Eng TN') {
8585
steps {
86-
sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/jenkinsci/nlp/text_norm/output/ --grammars=tn_grammars --cache_dir /home/jenkinsci/nlp/text_norm/ci/grammars/11-16-22 --language=en && ls -R /home/jenkinsci/nlp/text_norm/output/ && echo ".far files created "|| exit 1'
87-
sh 'cd NTP_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/nlp/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=/home/jenkinsci/nlp/text_norm/output/test.pynini.txt --verbose'
88-
sh 'cat /home/jenkinsci/nlp/text_norm/output/test.pynini.txt'
89-
sh 'cmp --silent /home/jenkinsci/nlp/text_norm/output/test.pynini.txt /home/jenkinsci/nlp/text_norm/ci/test_goal_py_05-25.txt || exit 1'
90-
sh 'rm -rf /home/jenkinsci/nlp/text_norm/output/*'
86+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \
87+
cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir /home/jenkinsci/TestData/text_norm/ci/grammars/01-30-23 --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
88+
sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \
89+
cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \
90+
cat $NORM_OUTPUT_DIR/test.pynini.txt && \
91+
cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \
92+
rm -rf $NORM_OUTPUT_DIR'
9193
}
9294
}
9395

9496
stage('L2: Eng ITN export') {
9597
steps {
96-
sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/jenkinsci/nlp/text_denorm/output/ --grammars=itn_grammars --cache_dir /home/jenkinsci/nlp/text_norm/ci/grammars/11-16-22 --language=en && ls -R /home/jenkinsci/nlp/text_denorm/output/ && echo ".far files created "|| exit 1'
97-
sh 'cd NTP_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/nlp/text_denorm/ci/test.txt --language=en --output_file=/home/jenkinsci/nlp/text_denorm/output/test.pynini.txt --verbose'
98-
sh 'cmp --silent /home/jenkinsci/nlp/text_denorm/output/test.pynini.txt /home/jenkinsci/nlp/text_denorm/ci/test_goal_py.txt || exit 1'
99-
sh 'rm -rf /home/jenkinsci/nlp/text_denorm/output/*'
98+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \
99+
cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir /home/jenkinsci/TestData/text_norm/ci/grammars/01-30-23 --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1'
100+
sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \
101+
cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \
102+
cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \
103+
rm -rf $OUTPUT_DIR'
100104
}
101105
}
102106

‎requirements/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ joblib
55
pandas
66
regex
77
sacremoses>=0.0.43
8-
setuptools==65.5.1
8+
setuptools==59.5.0
99
tqdm>=4.41.0
1010
transformers
1111
wget

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ def finalize_options(self):
221221
],
222222
packages=setuptools.find_packages(),
223223
install_requires=install_requires,
224-
setup_requires=['pytest-runner'],
224+
# setup_requires=['pytest-runner'],
225225
tests_require=tests_requirements,
226226
# List additional groups of dependencies here (e.g. development
227227
# dependencies). You can install these using the following syntax,

0 commit comments

Comments
 (0)