Make TTS L2 Tests More Resilient (#15584)

* Make all TTS tests HUBOFFLINE and deterministic

Signed-off-by: Jason <jasoli@nvidia.com>

* add pre-saved checkpoint paths for ASR and EOU

Signed-off-by: Jason <jasoli@nvidia.com>

* tighten bounds

Signed-off-by: Jason <jasoli@nvidia.com>

---------

Signed-off-by: Jason <jasoli@nvidia.com>

Author: blisc

nemo/collections/tts/models/easy_magpietts_inference.py

@@ -214,7 +214,13 @@ def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None):
             self.world_size = trainer.num_nodes * trainer.num_devices
 
         # load codec
-        codec_model = AudioCodecModel.restore_from(cfg.get('codecmodel_path'), strict=False)
+        codec_model_path = cfg.get('codecmodel_path')
+        codec_model_cfg = AudioCodecModel.restore_from(codec_model_path, return_config=True)
+        if "use_scl_loss" in codec_model_cfg:
+            codec_model_cfg.use_scl_loss = False
+        codec_model = AudioCodecModel.restore_from(
+            codec_model_path, strict=False, override_config_path=codec_model_cfg
+        )
         self.sample_rate = codec_model.sample_rate
         self.output_sample_rate = codec_model.output_sample_rate
 

tests/functional_tests/L2_TTS_Fast_dev_runs_EasyMagpietts_Nemotron.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/easy_magpietts.py \
+HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/easy_magpietts.py \
     --config-name easy_magpietts \
     name="EasyMagpieTTS-Nemotron-FastDev" \
     ~model.phoneme_tokenizer \
@@ -35,4 +35,4 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     trainer.strategy=auto \
     model.train_ds.dataloader_params.num_workers=0 \
     model.validation_ds.dataloader_params.num_workers=0 \
-    ~trainer.check_val_every_n_epoch
+    ~trainer.check_val_every_n_epoch

tests/functional_tests/L2_TTS_Fast_dev_runs_EasyMagpietts_Qwen.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/easy_magpietts.py \
+HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/easy_magpietts.py \
     --config-name easy_magpietts \
     name="EasyMagpieTTS-Qwen-FastDev" \
     ~model.phoneme_tokenizer \

tests/functional_tests/L2_TTS_Fast_dev_runs_Magpietts_DecoderContext.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
+HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
     --config-name magpietts \
     +train_ds_meta.an4.manifest_path="/home/TestData/an4_dataset/an4_train_context_v1.json" \
     +train_ds_meta.an4.audio_dir="/" \
@@ -30,4 +30,4 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     trainer.strategy=auto \
     model.train_ds.dataloader_params.num_workers=0 \
     model.validation_ds.dataloader_params.num_workers=0 \
-    ~trainer.check_val_every_n_epoch
+    ~trainer.check_val_every_n_epoch

tests/functional_tests/L2_TTS_Fast_dev_runs_Magpietts_MoE.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
+HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
     --config-name magpietts \
     name="MagpieTTS-MoE" \
     +model.use_moe=true \

tests/functional_tests/L2_TTS_Fast_dev_runs_Magpietts_MultiEncoder.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
+HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts.py \
     --config-name magpietts \
     model.model_type=multi_encoder_context_tts \
     +model.transcript_decoder_layers=[3,4,5,6,7] \

tests/functional_tests/L2_TTS_InferEvaluate_EasyMagpietts.sh

@@ -16,7 +16,8 @@
 # that is incompatible with torch.use_deterministic_algorithms(True) on CUDA >= 10.2.
 # Setting CUBLAS_WORKSPACE_CONFIG is not sufficient since the error originates inside the
 # transformers library (modeling_qwen2.py Qwen2RotaryEmbedding.forward).
-TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+CUBLAS_WORKSPACE_CONFIG=:4096:8 HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --codecmodel_path /home/TestData/tts/25fps_spectral_codec_with_bandwidth_extension.nemo \
     --nemo_files /home/TestData/tts/2603_EasyMagpieTTS/EMTTS_Pretraining_Qwen_WithCrossLingual_3_5_Delay.nemo \
     --out_dir ./emp_zs_0 \
@@ -34,5 +35,7 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --phoneme_tokenizer_path /home/TestData/tts/2603_EasyMagpieTTS/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh.json \
     --temperature 0.6 \
     --clean_up_disk \
-    --cer_target 0.20 \
-    --ssim_target 0.65
+    --cer_target 0.10 \
+    --ssim_target 0.70 \
+    --asr_model_name /home/TestData/tts/pretrained_models/parakeet-tdt-1.1b/parakeet-tdt-1.1b.nemo \
+    --eou_model_name /home/TestData/tts/pretrained_models/wav2vec2-base-960h

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_FrameStacking.sh

@@ -14,7 +14,7 @@
 
 # Tests a 4x-stacked model with local transformer inference.
 
-HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+CUBLAS_WORKSPACE_CONFIG=:4096:8 HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
     --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_MoE_ZeroShot.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+CUBLAS_WORKSPACE_CONFIG=:4096:8 HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
     --deterministic \
     --nemo_files "/home/TestData/tts/2602_MoE/moe16_sinkhorn_top1_valLoss5.0469_step2625132_epoch524.nemo" \
     --codecmodel_path "/home/TestData/tts/21fps_causal_codecmodel.nemo" \

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_SeenSpeakers.sh

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+CUBLAS_WORKSPACE_CONFIG=:4096:8 HF_HUB_OFFLINE=1 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
     --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \