Skip to content

Commit 15f1423

Browse files
anand-nvanand-nv
committed
Keep only EN tests for testing CI/CD pipeline
Signed-off-by: Anand Joseph <anajoseph@nvidia.com>
1 parent 7f16408 commit 15f1423

1 file changed

Lines changed: 4 additions & 298 deletions

File tree

‎Jenkinsfile‎

Lines changed: 4 additions & 298 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
pipeline {
22
agent {
33
docker {
4-
image 'tnitn_ci:py310'
5-
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""'
4+
image 'tnitn_ci:py312'
5+
args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=8g --entrypoint=""'
66
}
77
}
88
options {
9-
timeout(time: 2, unit: 'HOURS')
9+
timeout(time: 4, unit: 'HOURS')
1010
disableConcurrentBuilds(abortPrevious: true)
1111
}
1212
environment {
@@ -93,231 +93,7 @@ pipeline {
9393

9494
}
9595
}
96-
stage('L0: Create HI TN/ITN Grammars') {
97-
when {
98-
anyOf {
99-
branch 'main'
100-
changeRequest target: 'main'
101-
}
102-
}
103-
failFast true
104-
parallel {
105-
stage('L0: Hi TN grammars') {
106-
steps {
107-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hi --text="१" --cache_dir ${HI_TN_CACHE}'
108-
}
109-
}
110-
stage('L0: Hi ITN grammars') {
111-
steps {
112-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hi --text="एक" --cache_dir ${HI_TN_CACHE}'
113-
}
114-
}
115-
116-
}
117-
}
118-
119-
stage('L0: Create DE/ES TN/ITN Grammars') {
120-
when {
121-
anyOf {
122-
branch 'main'
123-
changeRequest target: 'main'
124-
}
125-
}
126-
failFast true
127-
parallel {
128-
stage('L0: DE TN grammars') {
129-
steps {
130-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=de --text="1" --cache_dir ${DEFAULT_TN_CACHE}'
131-
}
132-
}
133-
stage('L0: DE ITN grammars') {
134-
steps {
135-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=de --text="ein hundert " --cache_dir ${DEFAULT_TN_CACHE}'
136-
}
137-
}
138-
stage('L0: ES TN grammars') {
139-
steps {
140-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=es --text="1" --cache_dir ${ES_TN_CACHE}'
141-
}
142-
}
143-
stage('L0: ES ITN grammars') {
144-
steps {
145-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=es --text="ciento uno " --cache_dir ${ES_TN_CACHE}'
146-
}
147-
}
148-
stage('L0: Codeswitched ES/EN ITN grammars') {
149-
steps {
150-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=es_en --text="ciento uno " --cache_dir ${ES_EN_TN_CACHE}'
151-
}
152-
}
153-
}
154-
}
155-
156-
stage('L0: Create AR TN/ITN Grammars') {
157-
when {
158-
anyOf {
159-
branch 'main'
160-
changeRequest target: 'main'
161-
}
162-
}
163-
failFast true
164-
parallel {
165-
stage('L0: AR TN grammars') {
166-
steps {
167-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ar --text="2" --cache_dir ${AR_TN_CACHE}'
168-
}
169-
}
170-
stage('L0: AR ITN grammars') {
171-
steps {
172-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ar --text="اثنان " --cache_dir ${AR_TN_CACHE}'
173-
}
174-
}
175-
176-
}
177-
}
178-
179-
stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') {
180-
when {
181-
anyOf {
182-
branch 'main'
183-
changeRequest target: 'main'
184-
}
185-
}
186-
failFast true
187-
parallel {
188-
stage('L0: FR TN grammars') {
189-
steps {
190-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=fr --text="2" --cache_dir ${FR_TN_CACHE}'
191-
}
192-
}
193-
stage('L0: FR ITN grammars') {
194-
steps {
195-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=fr --text="cent " --cache_dir ${FR_TN_CACHE}'
196-
}
197-
}
198-
stage('L0: VI ITN grammars') {
199-
steps {
200-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=vi --text="một ngàn " --cache_dir ${VI_TN_CACHE}'
201-
}
202-
}
203-
stage('L0: HU TN grammars') {
204-
steps {
205-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}'
206-
}
207-
}
208-
stage('L0: IT TN grammars') {
209-
steps {
210-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=it --text="122" --cache_dir ${IT_TN_CACHE}'
211-
}
212-
}
213-
}
214-
}
215-
216-
stage('L0: Create RU TN/ITN Grammars & SV & PT') {
217-
when {
218-
anyOf {
219-
branch 'main'
220-
changeRequest target: 'main'
221-
}
222-
}
223-
failFast true
224-
parallel {
225-
stage('L0: RU TN grammars') {
226-
steps {
227-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --lang=ru --text="03" --cache_dir ${RU_TN_CACHE}'
228-
}
229-
}
230-
stage('L0: RU ITN grammars') {
231-
steps {
232-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ru --text="три " --cache_dir ${RU_TN_CACHE}'
233-
}
234-
}
235-
stage('L0: SV TN grammars') {
236-
steps {
237-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=sv --text="100" --cache_dir ${SV_TN_CACHE}'
238-
}
239-
}
240-
// stage('L0: SV ITN grammars') {
241-
// steps {
242-
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=sv --text="hundra " --cache_dir ${SV_TN_CACHE}'
243-
// }
244-
// }
245-
// stage('L0: PT TN grammars') {
246-
// steps {
247-
// sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=pt --text="2" --cache_dir ${DEFAULT_TN_CACHE}'
248-
// }
249-
// }
250-
stage('L0: PT ITN grammars') {
251-
steps {
252-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=pt --text="dez " --cache_dir ${PT_TN_CACHE}'
253-
}
254-
}
255-
}
256-
}
25796

258-
stage('L0: Create HY TN/ITN Grammars & MR') {
259-
when {
260-
anyOf {
261-
branch 'main'
262-
changeRequest target: 'main'
263-
}
264-
}
265-
failFast true
266-
parallel {
267-
stage('L0: MR ITN grammars') {
268-
steps {
269-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=mr --text="शून्य " --cache_dir ${MR_TN_CACHE}'
270-
}
271-
}
272-
stage('L0: HY TN grammars') {
273-
steps {
274-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hy --text="6" --cache_dir ${HY_TN_CACHE}'
275-
}
276-
}
277-
stage('L0: HY ITN grammars') {
278-
steps {
279-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hy --text="վեց " --cache_dir ${HY_TN_CACHE}'
280-
}
281-
}
282-
}
283-
}
284-
stage('L0: Create ZH TN/ITN Grammar') {
285-
when {
286-
anyOf {
287-
branch 'main'
288-
changeRequest target: 'main'
289-
}
290-
}
291-
failFast true
292-
parallel {
293-
stage('L0: ZH ITN grammars') {
294-
steps {
295-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}'
296-
}
297-
}
298-
stage('L0: ZH TN grammars') {
299-
steps {
300-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="6" --cache_dir ${ZH_TN_CACHE}'
301-
}
302-
}
303-
}
304-
}
305-
stage('L0: Create JA ITN Grammars') {
306-
when {
307-
anyOf {
308-
branch 'main'
309-
changeRequest target: 'main'
310-
}
311-
}
312-
failFast true
313-
parallel {
314-
stage('L0: JA ITN grammars') {
315-
steps {
316-
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ja --text="100" --cache_dir ${JA_TN_CACHE}'
317-
}
318-
}
319-
}
320-
}
32197

32298

32399
// L1 Tests starts here
@@ -336,80 +112,10 @@ pipeline {
336112
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir ${EN_TN_CACHE}'
337113
}
338114
}
339-
stage('L1: Run all DE TN/ITN tests (restore grammars from cache)') {
340-
steps {
341-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/de/ -m "not pleasefixme" --cpu --tn_cache_dir ${DE_TN_CACHE}'
342-
}
343-
}
344-
stage('L1: Run all ES TN/ITN tests (restore grammars from cache)') {
345-
steps {
346-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_TN_CACHE}'
347-
}
348-
}
349-
stage('L1: Run all HI TN/ITN tests (restore grammars from cache)') {
350-
steps {
351-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}'
352-
}
353-
}
354-
stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') {
355-
steps {
356-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}'
357-
}
358-
}
359-
stage('L1: Run all AR TN/ITN tests (restore grammars from cache)') {
360-
steps {
361-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ar/ -m "not pleasefixme" --cpu --tn_cache_dir ${AR_TN_CACHE}'
362-
}
363-
}
364-
stage('L1: Run all FR TN/ITN tests (restore grammars from cache)') {
365-
steps {
366-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/fr/ -m "not pleasefixme" --cpu --tn_cache_dir ${FR_TN_CACHE}'
367-
}
368-
}
369-
stage('L1: Run all PT TN/ITN tests (restore grammars from cache)') {
370-
steps {
371-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/pt/ -m "not pleasefixme" --cpu --tn_cache_dir ${PT_TN_CACHE}'
372-
}
373-
}
374-
stage('L1: Run all VI TN/ITN tests (restore grammars from cache)') {
375-
steps {
376-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/vi/ -m "not pleasefixme" --cpu --tn_cache_dir ${VI_TN_CACHE}'
377-
}
378-
}
379-
stage('L1: Run all RU TN/ITN tests (restore grammars from cache)') {
380-
steps {
381-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ru/ -m "not pleasefixme" --cpu --tn_cache_dir ${RU_TN_CACHE}'
382-
}
383-
}
384-
// stage('L1: Run all SV TN/ITN tests (restore grammars from cache)') {
385-
// steps {
386-
// sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/sv/ -m "not pleasefixme" --cpu --tn_cache_dir ${SV_TN_CACHE}'
387-
// }
388-
// }
389-
stage('L1: Run all ZH TN/ITN tests (restore grammars from cache)') {
390-
steps {
391-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/zh/ -m "not pleasefixme" --cpu --tn_cache_dir ${ZH_TN_CACHE}'
392-
}
393-
}
394-
stage('L1: Run all JA TN/ITN tests (restore grammars from cache)') {
395-
steps {
396-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ja/ -m "not pleasefixme" --cpu --tn_cache_dir ${JA_TN_CACHE}'
397-
}
398-
}
399-
stage('L1: Run all MR ITN tests (restore grammars from cache)') {
400-
steps {
401-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/mr/ -m "not pleasefixme" --cpu --tn_cache_dir ${MR_TN_CACHE}'
402-
}
403-
}
404-
stage('L1: Run all HY TN/ITN tests (restore grammars from cache)') {
405-
steps {
406-
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hy/ -m "not pleasefixme" --cpu --tn_cache_dir ${HY_TN_CACHE}'
407-
}
408-
}
409115
}
410116
}
411117

412-
stage('L2: Sparrowhawk Tests') {
118+
stage('L2: Sparrowhawk Tests') {
413119
when {
414120
anyOf {
415121
branch 'main'

0 commit comments

Comments
 (0)