Skip to content

Commit 765ab6a

Browse files
ernie-changCamilo Quinones
authored andcommitted
Add a new DAG maxtext_multi_tier_sav01_save_local (Save Locally Orbax)
[UPDATE] Update with comment [NEW] Add orbax utils: apply cpc and delete cpc to the dag [UPDATE] add delete cpc and apply cpc in orbax [UPDATE] Update format and coding style [UPDATE] [UPDATE] Modify the dag structure and add cpc operation [UPDATE] Modify the dag structure and add cpc operation [UPDATE] update for the code comment [UPDATE] update for the code comment add a new DAG 'maxtext_multi_tier_sav01_save_local' GoogleCloudPlatform#804 Add comment validate() Change log explorer text filter string to be discarded to be discarded to be discarded
1 parent 6216305 commit 765ab6a

7 files changed

Lines changed: 541 additions & 0 deletions

File tree

dags/common/test_owner.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ class Team(enum.Enum):
5555
# Multi-tier Checkpointing
5656
ABHINAV_S = "ABHINAV S."
5757
XUEFENG_G = "XUEFENG G."
58+
CAMILO = "CAMILO P."
5859

5960
# MLCompass
6061
ORTI_B = "Orti B."

dags/common/vm_resource.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,13 @@ class XpkClusters:
256256
project=Project.CLOUD_TPU_MULTIPOD_DEV.value,
257257
zone=Zone.EUROPE_WEST4_B.value,
258258
)
259+
TPU_V5P_128_CLUSTER_ORBAX = XpkClusterConfig(
260+
name="b428061876-jf-v5p-128-2",
261+
device_version=TpuVersion.V5P,
262+
core_count=128,
263+
project=Project.CLOUD_TPU_MULTIPOD_DEV.value,
264+
zone=Zone.EUROPE_WEST4_B.value,
265+
)
259266
TPU_V5E_256_CLUSTER = XpkClusterConfig(
260267
name="v5e-256-bodaborg-europe-west4",
261268
device_version=TpuVersion.V5E,

dags/gcs_bucket.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,4 @@
2727
# Multi-tier checkpointing need special permission for GCS Bucket
2828
# For further question reach out to Multi-tier Checkpointing Owners.
2929
MTC_BUCKET = "gs://mtc-bucket-us-east5/output"
30+
MTC_AUTOMATION_BUCKET = "gs://mtc-automation-bucket"
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""Add commentMore actions
2+
A DAG to run MaxText multi-tier checkpointing tests (phase2: save & validate).
3+
"""
4+
5+
import datetime
6+
from airflow import models
7+
from dags import composer_env, gcs_bucket
8+
from dags.common import test_owner
9+
from dags.common.vm_resource import DockerImage, XpkClusters
10+
from dags.multipod.configs import gke_config
11+
from dags.multipod.configs.common import SetupMode
12+
from xlml.utils import log_explorer
13+
from xlml.utils import orbax
14+
15+
SCHEDULE = "0 10 * * *" if composer_env.is_prod_env() else None
16+
17+
with models.DAG(
18+
dag_id="maxtext_multi_tier_sav01_save_local",
19+
schedule_interval=SCHEDULE,
20+
tags=[
21+
"multipod_team",
22+
"maxtext",
23+
"multi_tier_p2_chkpt_save_local",
24+
"nightly",
25+
],
26+
start_date=datetime.datetime(2025, 6, 12),
27+
catchup=False,
28+
concurrency=2,
29+
) as dag:
30+
base_output_directory = (
31+
f"{gcs_bucket.MTC_AUTOMATION_BUCKET}/maxtext_multi_tier_sav01_save_local"
32+
)
33+
docker_images = [(
34+
SetupMode.NIGHTLY,
35+
DockerImage.MAXTEXT_TPU_JAX_NIGHTLY,
36+
)]
37+
ram_disk = "/local"
38+
test_configs = {"v5p-128": [2]}
39+
clusters = {"v5p-128": XpkClusters.TPU_V5P_128_CLUSTER_ORBAX}
40+
step = 100
41+
local_checkpoint_period = 20
42+
replicator_backup_interval_minutes = 1
43+
use_replicator = "True"
44+
name_prefix = "maxtext-p2-cpt-sv"
45+
model_name = "llama2-7b"
46+
47+
for mode, image in docker_images:
48+
for accelerator, slices in test_configs.items():
49+
for slice_num in slices:
50+
cpc = (
51+
clusters[accelerator].project,
52+
clusters[accelerator].zone[:-2],
53+
clusters[accelerator].name,
54+
gcs_bucket.MTC_AUTOMATION_BUCKET.split("gs://")[1],
55+
"ct5p-hightpu-4t",
56+
"google.com/tpu",
57+
"800000Mi",
58+
)
59+
delete_cpc = orbax.delete_cpc(*cpc)
60+
apply_cpc = orbax.apply_cpc(*cpc)
61+
run_time = datetime.datetime.now().strftime("%Y-%m-%d-%H")
62+
run_name = f"{name_prefix}-{slice_num}x-{accelerator}-{run_time}"
63+
workload_command = (
64+
"export TPU_PREMAPPED_BUFFER_SIZE=52428800000 && "
65+
"export TPU_PREMAPPED_BUFFER_TRANSFER_THRESHOLD_BYTES=52428800000 && "
66+
"python3 -m MaxText.train MaxText/configs/base.yml remat_policy=full "
67+
f"global_parameter_scale=1 base_output_directory={base_output_directory} "
68+
f"dataset_type=synthetic steps={step} per_device_batch_size=1 "
69+
f"max_target_length=256 model_name={model_name} per_device_batch_size=2 "
70+
"reuse_example_batch=1 enable_emergency_checkpoint=true "
71+
f"local_checkpoint_directory={ram_disk} local_checkpoint_period={local_checkpoint_period} "
72+
f"use_replicator_service={use_replicator} replicator_backup_interval_minutes={replicator_backup_interval_minutes} "
73+
f"run_name={run_name}",
74+
)
75+
76+
start_time = log_explorer.generate_timestamp()
77+
78+
# make launch test_name unique
79+
maxtext_phase2_chkpt_test = gke_config.get_gke_config(
80+
num_slices=slice_num,
81+
cluster=clusters[accelerator],
82+
time_out_in_min=60,
83+
test_name=f"{name_prefix}",
84+
run_model_cmds=workload_command,
85+
docker_image=image.value,
86+
test_owner=test_owner.CAMILO,
87+
).run(
88+
ramdisk_directory=ram_disk,
89+
mtc_enabled=True,
90+
xpk_branch="main",
91+
skip_post_process=True,
92+
)
93+
94+
# cleanup run: unique test_name
95+
cleanup_command = (f"rm -rf {ram_disk}/*",)
96+
ram_disk_cleanup = gke_config.get_gke_config(
97+
num_slices=slice_num,
98+
cluster=clusters[accelerator],
99+
time_out_in_min=60,
100+
test_name=f"{name_prefix}-cleanup",
101+
run_model_cmds=cleanup_command,
102+
docker_image=image.value,
103+
test_owner=test_owner.CAMILO,
104+
).run(
105+
ramdisk_directory=ram_disk,
106+
mtc_enabled=True,
107+
xpk_branch="main",
108+
skip_post_process=True,
109+
)
110+
111+
end_time = log_explorer.generate_timestamp()
112+
vali_step = step - 1
113+
vali_step_list = [
114+
i for i in range(0, vali_step, local_checkpoint_period)
115+
]
116+
vali_step_list.append(vali_step)
117+
118+
validate_log = log_explorer.validate_log_with_step(
119+
project_id=clusters[accelerator].project,
120+
location=clusters[accelerator].zone[:-2],
121+
cluster_name=clusters[accelerator].name,
122+
text_filter="(blocking + background).",
123+
start_time=start_time,
124+
end_time=end_time,
125+
vali_step_list=vali_step_list,
126+
)
127+
128+
(
129+
delete_cpc
130+
>> apply_cpc
131+
>> start_time
132+
>> maxtext_phase2_chkpt_test
133+
>> ram_disk_cleanup
134+
>> end_time
135+
>> validate_log
136+
)

xlml/utils/log_explorer.py

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
"""Utilities to get workloads logs and some utils."""
2+
3+
from airflow.decorators import task
4+
from airflow.exceptions import AirflowFailException
5+
from google.cloud import logging as log_explorer
6+
from datetime import datetime, timezone, timedelta
7+
from typing import Optional
8+
from absl import logging
9+
10+
11+
@task
12+
def generate_timestamp():
13+
return datetime.now(timezone.utc)
14+
15+
16+
@task
17+
def validate_log_with_step(
18+
project_id: str,
19+
location: str,
20+
cluster_name: str,
21+
namespace: str = "default",
22+
pod_pattern: str = "*",
23+
container_name: Optional[str] = None,
24+
text_filter: Optional[str] = None,
25+
start_time: Optional[datetime] = None,
26+
end_time: Optional[datetime] = None,
27+
vali_step_list: Optional[list] = None,
28+
) -> bool:
29+
"""
30+
Validate the workload log is training correct
31+
Args:
32+
project_id: The Google Cloud project ID
33+
location: GKE cluster location
34+
cluster_name: GKE cluster name
35+
namespace: Kubernetes namespace (defaults to "default")
36+
pod_pattern: Pattern to match pod names (defaults to "*")
37+
container_name: Optional container name to filter logs
38+
text_filter: Optional comma-separated string to
39+
filter log entries by textPayload content
40+
start_time: Optional start time for log retrieval
41+
(defaults to 12 hours ago)
42+
end_time: Optional end time for log retrieval (defaults to now)
43+
vali_step_list: optional to validate list of steps
44+
Returns:
45+
bool: validate success or not
46+
"""
47+
entries = list_log_entries(
48+
project_id=project_id,
49+
location=location,
50+
cluster_name=cluster_name,
51+
namespace=namespace,
52+
pod_pattern=pod_pattern,
53+
container_name=container_name,
54+
text_filter=text_filter,
55+
start_time=start_time,
56+
end_time=end_time,
57+
)
58+
if vali_step_list is None:
59+
return False
60+
new_step_list = []
61+
for entry in entries:
62+
if not entry.payload:
63+
continue
64+
payload_str = str(entry.payload)
65+
for line in payload_str.split("\n"):
66+
if vali_step_list is not None:
67+
for step in vali_step_list:
68+
vali_str = "directory=/local/" + str(step)
69+
if vali_str in line and step not in new_step_list:
70+
logging.info(f"├─ Timestamp: {entry.timestamp}")
71+
logging.info("└─ Payload:")
72+
logging.info(f" {line}")
73+
new_step_list.append(step)
74+
if len(vali_step_list) == len(new_step_list):
75+
logging.info("Validate success")
76+
return True
77+
else:
78+
raise AirflowFailException(
79+
f"{len(vali_step_list)} saves are expected,"
80+
f"but got {len(new_step_list)}"
81+
)
82+
83+
84+
def list_log_entries(
85+
project_id: str,
86+
location: str,
87+
cluster_name: str,
88+
namespace: str = "default",
89+
pod_pattern: str = "*",
90+
container_name: Optional[str] = None,
91+
text_filter: Optional[str] = None,
92+
start_time: Optional[datetime] = None,
93+
end_time: Optional[datetime] = None,
94+
) -> list:
95+
"""
96+
List log entries for the specified Google Cloud project.
97+
This function connects to Google Cloud Logging,
98+
constructs a filter for Kubernetes container logs
99+
within a specific project, location, cluster, namespace,
100+
and pod name pattern, and retrieves log
101+
entries from the specified time range.
102+
It prints the timestamp, severity, resource information,
103+
and payload for each log entry found.
104+
Args:
105+
project_id: The Google Cloud project ID
106+
location: GKE cluster location
107+
cluster_name: GKE cluster name
108+
namespace: Kubernetes namespace (defaults to "default")
109+
pod_pattern: Pattern to match pod names (defaults to "*")
110+
container_name: Optional container name to filter logs
111+
text_filter: Optional comma-separated string to
112+
filter log entries by textPayload content
113+
start_time: Optional start time for log retrieval
114+
(defaults to 12 hours ago)
115+
end_time: Optional end time for log retrieval (defaults to now)
116+
Returns:
117+
bool: Number of log entries found
118+
"""
119+
120+
# Create a Logging Client for the specified project
121+
logging_client = log_explorer.Client(project=project_id)
122+
123+
# Set the time window for log retrieval:
124+
# default to last 12 hours if not provided
125+
if end_time is None:
126+
end_time = datetime.now(timezone.utc)
127+
if start_time is None:
128+
start_time = end_time - timedelta(hours=12)
129+
130+
# Format times as RFC3339 UTC "Zulu" format required by the Logging API
131+
start_time_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
132+
end_time_str = end_time.strftime("%Y-%m-%dT%H:%M:%SZ")
133+
134+
# Construct the log filter
135+
log_filter = (
136+
f'resource.labels.project_id="{project_id}" '
137+
f'resource.labels.location="{location}" '
138+
f'resource.labels.cluster_name="{cluster_name}" '
139+
f'resource.labels.namespace_name="{namespace}" '
140+
f'resource.labels.pod_name:"{pod_pattern}" '
141+
"severity>=DEFAULT "
142+
f'timestamp>="{start_time_str}" '
143+
f'timestamp<="{end_time_str}"'
144+
)
145+
146+
# Add container name filter if provided
147+
if container_name:
148+
log_filter += f' resource.labels.container_name="{container_name}"'
149+
150+
# Add text content filter if provided
151+
if text_filter:
152+
log_filter += f' SEARCH("{text_filter}")'
153+
154+
# Retrieve log entries matching the filter
155+
logging.info(f"Log filter constructed: {log_filter}")
156+
entries = logging_client.list_entries(filter_=log_filter)
157+
158+
return entries

0 commit comments

Comments
 (0)