fix: Adjust DAG schedules (GoogleCloudPlatform#1126)

Co-authored-by: Andrew Yang <andrew.yang@cienet.com>

Author: chengpinglin

dags/map_reproducibility/a3ultra/llama_3_1_405b_maxtext.py

@@ -43,7 +43,7 @@
 
 with models.DAG(
     dag_id=DAG_ID,
-    schedule="0 9 * * 5",
+    schedule="0 8 * * 5",
     tags=[
         "reproducibility",
         "experimental",

dags/map_reproducibility/a3ultra/llama_3_1_405b_nemo.py

@@ -42,7 +42,7 @@
 
 with models.DAG(
     dag_id=DAG_ID,
-    schedule="0 11 * * 5",
+    schedule="0 2 * * 1",
     tags=[
         "reproducibility",
         "experimental",

dags/map_reproducibility/a3ultra/llama_3_1_70b_maxtext.py

@@ -45,7 +45,7 @@
 
 with models.DAG(
     dag_id=f"{HYPERCOMPUTER}_recipes_{MODEL_ID}_{FRAMEWORK}",
-    schedule="0 5 * * 5",
+    schedule="0 2 * * 0",
     tags=[
         "reproducibility",
         "experimental",

dags/map_reproducibility/a3ultra/llama_3_1_70b_nemo.py

@@ -42,7 +42,7 @@
 
 with models.DAG(
     dag_id=DAG_ID,
-    schedule="0 7 * * 5",
+    schedule="0 7 * * 0",
     tags=[
         "reproducibility",
         "experimental",

dags/map_reproducibility/internal_runs/dag_configs.py

@@ -106,7 +106,7 @@
         "timeout_minutes": 15,
         "backfill_group_nightly": 1,
         "backfill_group_release": 1,
-        "nightly_schedule": "15 10 * * 2,3,4,6",
+        "nightly_schedule": "0 11 * * 2,3,4,6",
         "release_schedule": "0 2 * * 2,3,4,6",
     },
     "recipes/a3ultra/a3ultra_llama3.1-8b_16gpus_fp8_maxtext.yaml": {
@@ -134,29 +134,29 @@
         "timeout_minutes": 20,
         "backfill_group_nightly": 2,
         "backfill_group_release": 2,
-        "nightly_schedule": "0 11 * * 2,3,4,6",
-        "release_schedule": "30 11 * * 2,3,4,6",
+        "nightly_schedule": "30 11 * * 2,3,4,6",
+        "release_schedule": "0 12 * * 2,3,4,6",
     },
     "recipes/a3ultra/a3ultra_llama3.1-70b_256gpus_fp8_maxtext.yaml": {
         "timeout_minutes": 15,
         "backfill_group_nightly": 3,
         "backfill_group_release": 3,
-        "nightly_schedule": "45 12 * * 2,3,4,6",
-        "release_schedule": "0 12 * * 2,3,4,6",
+        "nightly_schedule": "15 13 * * 2,3,4,6",
+        "release_schedule": "30 12 * * 2,3,4,6",
     },
     "recipes/a3ultra/a3ultra_llama3.1-405b_256gpus_fp8_maxtext.yaml": {
         "timeout_minutes": 30,
         "backfill_group_nightly": 4,
         "backfill_group_release": 4,
-        "nightly_schedule": "30 13 * * 2,3,4,6",
-        "release_schedule": "15 14 * * 2,3,4,6",
+        "nightly_schedule": "45 13 * * 2,3,4,6",
+        "release_schedule": "30 14 * * 2,3,4,6",
     },
     "recipes/a3ultra/a3ultra_llama3.1-405b_256gpus_bf16_maxtext.yaml": {
         "timeout_minutes": 40,
         "backfill_group_nightly": 5,
         "backfill_group_release": 5,
-        "nightly_schedule": "45 9 * * 2,3,4,6",
-        "release_schedule": "0 15 * * 2,3,4,6",
+        "nightly_schedule": "15 10 * * 2,3,4,6",
+        "release_schedule": "15 15 * * 2,3,4,6",
     },
 }
 
@@ -237,14 +237,14 @@
 DAG_CONFIGS_ULTRA_NEMO = {
     "recipes/a3ultra/nemo/a3ultra_llama3.1-8b_8gpus_fp8_nemo.yaml": {
         "timeout_minutes": 20,
-        "release_schedule": "30 2 * * 6",
+        "release_schedule": "30 1 * * 6",
     },
     "recipes/a3ultra/nemo/a3ultra_llama3.1-8b_8gpus_bf16_nemo.yaml": {
         "timeout_minutes": 20,
-        "release_schedule": "0 2 * * 6",
+        "release_schedule": "0 1 * * 6",
     },
     "recipes/a3ultra/nemo/a3ultra_llama3.1-70b_256gpus_fp8_nemo.yaml": {
         "timeout_minutes": 25,
-        "release_schedule": "45 10 * * 6",
+        "release_schedule": "0 16 * * 6",
     },
 }

dags/map_reproducibility/utils/common_utils.py

@@ -1128,7 +1128,7 @@ def get_scheduled_time(hardware: str, model: str, framework: str):
   schedule_map = {
       "a3ultra": {
           "mixtral-8x7b": {
-              "nemo": "0 3 * * 5",
+              "nemo": "30 2 * * 5",
               "maxtext": "0 2 * * 5",  # 6 PM PST on Thursday
           },
           "llama3-1-70b": {