add linear warmup for cr-ctc loss (#2075)

- this prevents cr-ctc loss from diverging at the beginning of the training

Author: KarelVesely84

egs/aishell/ASR/zipformer/train.py

@@ -882,8 +882,10 @@ def compute_loss(
         if params.use_ctc:
             loss += params.ctc_loss_scale * ctc_loss
             if use_cr_ctc:
-                loss += params.cr_loss_scale * cr_loss
-    
+                # linear warmup
+                cr_loss_scale = min(batch_idx_train / warm_step, 1.0) * params.cr_loss_scale
+                loss += cr_loss_scale * cr_loss
+
     assert loss.requires_grad == is_training
 
     info = MetricsTracker()

egs/librispeech/ASR/zipformer/train.py

@@ -967,7 +967,9 @@ def compute_loss(
         if params.use_ctc:
             loss += params.ctc_loss_scale * ctc_loss
             if use_cr_ctc:
-                loss += params.cr_loss_scale * cr_loss
+                # linear warmup
+                cr_loss_scale = min(batch_idx_train / warm_step, 1.0) * params.cr_loss_scale
+                loss += cr_loss_scale * cr_loss
 
         if params.use_attention_decoder:
             loss += params.attention_decoder_loss_scale * attention_decoder_loss