fix: Resolve DAG validation errors triggered by unexpected log output from the pod_pattern regex (GoogleCloudPlatform#1154)

Resolve DAG validation errors triggered by unexpected log output from the pod_pattern regex

Author: ooops678

dags/orbax/maxtext_emc_resume_gcs.py

@@ -205,7 +205,7 @@
                 project_id=test_config.cluster.project,
                 location=zone_to_region(test_config.cluster.zone),
                 cluster_name=test_config.cluster.name,
-                pod_pattern="max.*-job-0-0",
+                pod_pattern=f"{test_config.short_id}.*-job-0-0",
                 interrupt_at_step=first_training_step - 1,
                 start_time=start_time,
                 end_time=end_time,

dags/orbax/maxtext_emc_save_gcs.py

@@ -100,7 +100,7 @@
         workload_command = test_config.generate_workload_command(
             checkpoint_dir=test_config_util.DEFAULT_RAM_DISK,
             run_name=run_name,
-            out_folder=f"maxtext_emc_orbax_save_gcs",
+            out_folder="maxtext_emc_orbax_save_gcs",
             enable_multi_tier_checkpointing=checkpointing.enable_multi_tier_checkpointing,
             slice_num=slice_num,
         )

dags/orbax/maxtext_mtc_resume_gcs.py

@@ -202,7 +202,7 @@
                 project_id=test_config.cluster.project,
                 location=zone_to_region(test_config.cluster.zone),
                 cluster_name=test_config.cluster.name,
-                pod_pattern="max.*-job-0-0",
+                pod_pattern=f"{test_config.short_id}.*-job-0-0",
                 interrupt_at_step=local_checkpoint_period,
                 check_last_two_local_saves=False,
                 start_time=start_time,

dags/orbax/maxtext_mtc_save_gcs.py

@@ -103,7 +103,7 @@
             checkpoint_dir=test_config_util.DEFAULT_RAM_DISK,
             run_name=run_name,
             slice_num=slice_num,
-            out_folder=f"maxtext_mtc_orbax_save_gcs",
+            out_folder="maxtext_mtc_orbax_save_gcs",
             enable_multi_tier_checkpointing=checkpointing.enable_multi_tier_checkpointing,
         )
 

dags/orbax/maxtext_reg_restore_gcs_with_node_disruption.py

@@ -149,7 +149,7 @@ def generate_workload_checkpoints_location(gcs_ckpt_location: str) -> str:
                 project_id=test_config.cluster.project,
                 location=zone_to_region(test_config.cluster.zone),
                 cluster_name=test_config.cluster.name,
-                pod_pattern=".*0-0",
+                pod_pattern=f"{test_config.short_id}.*0-0",
                 interrupt_at_step=step_to_interrupt,
                 start_time=start_time,
                 end_time=end_time,

dags/orbax/maxtext_reg_restore_gcs_with_resumed_workload.py

@@ -158,7 +158,7 @@
                 project_id=test_config.cluster.project,
                 location=zone_to_region(test_config.cluster.zone),
                 cluster_name=test_config.cluster.name,
-                pod_pattern="max.*-job-0-0",
+                pod_pattern=f"{test_config.short_id}.*-job-0-0",
                 interrupt_at_step=initial_step - 1,
                 start_time=start_time,
                 end_time=end_time,