ci(speech): split L0_Unit_Tests GPU/CPU ASR and CPU SpeechLM2 into parallel buckets

.github/workflows/cicd-main-speech.yml

@@ -46,34 +46,83 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - script: L0_Unit_Tests_GPU_ASR
+          - script: L0_Unit_Tests_GPU_ASR_1
             runner: ${{ inputs.runner }}
-            timeout: 60
-          - script: L0_Unit_Tests_CPU_ASR
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_ASR_2
+            runner: ${{ inputs.runner }}
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_ASR_3
+            runner: ${{ inputs.runner }}
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_ASR_4
+            runner: ${{ inputs.runner }}
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_ASR_5
+            runner: ${{ inputs.runner }}
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_ASR_6
+            runner: ${{ inputs.runner }}
+            timeout: 15
+          - script: L0_Unit_Tests_CPU_ASR_1
             runner: ${{ inputs.runner }}
             cpu-only: true
-            timeout: 30
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_ASR_2
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_ASR_3
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_ASR_4
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_ASR_5
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_ASR_6
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
           - script: L0_Unit_Tests_GPU_TTS
             runner: ${{ inputs.runner }}
             timeout: 30
           - script: L0_Unit_Tests_CPU_TTS
             runner: ${{ inputs.runner }}
             cpu-only: true
-            timeout: 20
-          - script: L0_Unit_Tests_GPU_Audio
+          - script: L0_Unit_Tests_GPU_Audio_1
             runner: ${{ inputs.runner }}
-            timeout: 20
+            timeout: 15
+          - script: L0_Unit_Tests_GPU_Audio_2
+            runner: ${{ inputs.runner }}
+            timeout: 15
           - script: L0_Unit_Tests_CPU_Audio
             runner: ${{ inputs.runner }}
             cpu-only: true
             timeout: 20
           - script: L0_Unit_Tests_GPU_SpeechLM2
             runner: ${{ inputs.runner }}
             timeout: 20
-          - script: L0_Unit_Tests_CPU_SpeechLM2
+          - script: L0_Unit_Tests_CPU_SpeechLM2_1
             runner: ${{ inputs.runner }}
             cpu-only: true
-            timeout: 20
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_SpeechLM2_2
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_SpeechLM2_3
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
+          - script: L0_Unit_Tests_CPU_SpeechLM2_4
+            runner: ${{ inputs.runner }}
+            cpu-only: true
+            timeout: 12
     needs: [build]
     runs-on: ${{ matrix.runner }}
     name: ${{ matrix.script }}

requirements/requirements_test.txt

@@ -5,6 +5,7 @@ isort>5.1.0,<6.0.0
 parameterized
 pytest
 pytest-httpserver
+pytest-shard
 pytest-mock
 pytest-runner
 ruamel.yaml

tests/functional_tests/L0_Unit_Tests_CPU_ASR.sh → tests/functional_tests/L0_Unit_Tests_CPU_ASR_1.sh

@@ -11,4 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/collections/asr -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=0 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_SpeechLM2.sh → tests/functional_tests/L0_Unit_Tests_CPU_ASR_2.sh

@@ -11,4 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/collections/speechlm2 -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=1 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_ASR_3.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=2 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_ASR_4.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=3 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_ASR_5.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=4 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_ASR_6.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=5 --num-shards=6 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_SpeechLM2_1.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/speechlm2 \
+    --shard-id=0 --num-shards=4 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_SpeechLM2_2.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/speechlm2 \
+    --shard-id=1 --num-shards=4 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_SpeechLM2_3.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/speechlm2 \
+    --shard-id=2 --num-shards=4 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_CPU_SpeechLM2_4.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/speechlm2 \
+    --shard-id=3 --num-shards=4 \
+    -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat

tests/functional_tests/L0_Unit_Tests_GPU_ASR.sh → tests/functional_tests/L0_Unit_Tests_GPU_ASR_1.sh

@@ -11,4 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/collections/asr -m "not pleasefixme" --with_downloads
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=0 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_ASR_2.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=1 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_ASR_3.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=2 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_ASR_4.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=3 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_ASR_5.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=4 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_ASR_6.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+python -c "from nemo.collections.asr.models import ASRModel" && NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/asr \
+    --shard-id=5 --num-shards=6 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_Audio.sh → tests/functional_tests/L0_Unit_Tests_GPU_Audio_1.sh

@@ -11,4 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/collections/audio -m "not pleasefixme" --with_downloads
+NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/audio \
+    --shard-id=0 --num-shards=2 \
+    -m "not pleasefixme" --with_downloads

tests/functional_tests/L0_Unit_Tests_GPU_Audio_2.sh

@@ -0,0 +1,17 @@
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+NEMO_NUMBA_MINVER=0.53 CUDA_VISIBLE_DEVICES=0 coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest \
+    tests/collections/audio \
+    --shard-id=1 --num-shards=2 \
+    -m "not pleasefixme" --with_downloads