fix: Adjust DAG schedule (GoogleCloudPlatform#1091)

Test team identified DAGs that are using the same cluster, and their schedule time are too close to each other, adjust the schedule could reduce the resource conflict and stabilizes the cluster/DAG.

Author: chengpinglin

dags/multipod/maxtext_checkpointing.py

@@ -24,7 +24,7 @@
 from dags.multipod.configs.common import SetupMode
 
 # Run once a day at 10 am UTC (2 am PST)
-SCHEDULED_TIME = "45 5 * * *" if composer_env.is_prod_env() else None
+SCHEDULED_TIME = "15 6 * * *" if composer_env.is_prod_env() else None
 
 with models.DAG(
     dag_id="maxtext_checkpointing",

dags/multipod/maxtext_sft_trainer.py

@@ -23,7 +23,7 @@
 from dags.multipod.configs.common import SetupMode
 
 # Run once a day at 10 am UTC (2 am PST)
-SCHEDULED_TIME = '45 6 * * *' if composer_env.is_prod_env() else None
+SCHEDULED_TIME = '0 7 * * *' if composer_env.is_prod_env() else None
 HF_TOKEN = models.Variable.get('HF_TOKEN', None)
 
 with models.DAG(

dags/orbax/maxtext_emc_restore_gcs.py

@@ -18,7 +18,7 @@
 from xlml.utils.gke import zone_to_region
 
 DAG_TEST_NAME = "maxtext_emc_orbax_res_gcs"
-SCHEDULE = "45 3 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "0 10 * * *" if composer_env.is_prod_env() else None
 
 with models.DAG(
     dag_id=DAG_TEST_NAME,

dags/orbax/maxtext_emc_restore_local.py

@@ -15,7 +15,7 @@
 from xlml.utils.gke import zone_to_region
 
 DAG_TEST_NAME = "maxtext_emc_orbax_res_local"
-SCHEDULE = "30 4 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "45 10 * * *" if composer_env.is_prod_env() else None
 
 
 with models.DAG(

dags/orbax/maxtext_emc_resume_gcs.py

@@ -21,7 +21,7 @@
 from dags.orbax.util import checkpoint_util
 from xlml.utils.gke import zone_to_region
 
-SCHEDULE = "15 9 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "30 15 * * *" if composer_env.is_prod_env() else None
 DAG_TEST_NAME = "maxtext_emc_resume_from_gcs"
 
 with models.DAG(

dags/orbax/maxtext_emc_save_gcs.py

@@ -19,7 +19,7 @@
 from dags.orbax.util import test_config_util
 
 
-SCHEDULE = "30 6 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "45 12 * * *" if composer_env.is_prod_env() else None
 DAG_TEST_NAME = "maxtext_emc_save_gcs"
 
 

dags/orbax/maxtext_mtc_emergency_save_local.py

@@ -23,7 +23,7 @@
 from dags.orbax.util import test_config_util
 
 
-SCHEDULE = "45 5 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "0 12 * * *" if composer_env.is_prod_env() else None
 DAG_TEST_NAME = "maxtext_emc_and_mtc_orbax_save_local"
 
 

dags/orbax/maxtext_mtc_restore_local.py

@@ -18,7 +18,7 @@
 from xlml.utils.gke import zone_to_region
 
 DAG_TEST_NAME = "maxtext_mtc_orbax_res_local"
-SCHEDULE = "30 7 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "45 13 * * *" if composer_env.is_prod_env() else None
 
 with models.DAG(
     dag_id=DAG_TEST_NAME,

dags/orbax/maxtext_mtc_resume_gcs.py

@@ -21,7 +21,7 @@
 from dags.orbax.util import checkpoint_util
 from xlml.utils.gke import zone_to_region
 
-SCHEDULE = "45 9 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "0 16 * * *" if composer_env.is_prod_env() else None
 DAG_TEST_NAME = "maxtext_mtc_resume_from_gcs"
 
 with models.DAG(

dags/orbax/maxtext_mtc_save_gcs.py

@@ -19,7 +19,7 @@
 from dags.orbax.util import test_config_util
 
 
-SCHEDULE = "0 7 * * *" if composer_env.is_prod_env() else None
+SCHEDULE = "15 13 * * *" if composer_env.is_prod_env() else None
 DAG_TEST_NAME = "maxtext_mtc_orbax_save_gcs"