Skip to content

Commit 9e16b8a

Browse files
Add a new DAG that tests node pool TTR by updating disk size (GoogleCloudPlatform#1137)
This change adds a new DAG that tests and verifies the GKE node pool's Times To Recover(TTR) metrics by triggering a disk size update and confirming the recovery time is recorded.
1 parent 5dead52 commit 9e16b8a

4 files changed

Lines changed: 280 additions & 26 deletions

File tree

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""A DAG to validate GKE node pool Times To Recover(TTR) metrics
16+
by triggering a disk size update."""
17+
18+
import datetime
19+
20+
from airflow import models
21+
from airflow.models.baseoperator import chain
22+
from airflow.utils.task_group import TaskGroup
23+
from airflow.utils.trigger_rule import TriggerRule
24+
25+
from dags import composer_env
26+
from dags.tpu_observability.configs.common import MachineConfigMap, GCS_CONFIG_PATH
27+
from dags.tpu_observability.utils import node_pool_util as node_pool
28+
29+
_DISK_SIZE_INCREMENT = 50
30+
31+
32+
with models.DAG(
33+
dag_id="node_pool_ttr_disk_size",
34+
start_date=datetime.datetime(2025, 6, 26),
35+
schedule="00 20 * * *" if composer_env.is_prod_env() else None,
36+
catchup=False,
37+
tags=[
38+
"cloud-ml-auto-solutions",
39+
"node_pool_ttr_disk_size",
40+
"tpu_obervability",
41+
"time_to_recover",
42+
],
43+
description=(
44+
"This DAG verifies the GKE node pool's Times To Recover(TTR) metrics "
45+
"by triggering a disk size update and confirming the recovery time "
46+
"is recorded"
47+
),
48+
doc_md="""
49+
# GKE Node Pool Times To Recover (TTR) Metric Validation DAG (Disk Resize)
50+
51+
### Description
52+
This DAG automates the validation of GKE node pool Times To Recover (TTR) metrics.
53+
It creates a temporary node pool, triggers a disk resize operation to force a node
54+
pool update, and checks that the TTR metric is correctly generated and reported
55+
to Google Cloud Monitoring.
56+
57+
### Prerequisites
58+
This test requires an existing GKE cluster.
59+
60+
### Procedures
61+
1. Create a temporary node pool.
62+
2. Wait for the node pool to be RUNNING.
63+
3. Trigger a disk resize update on the node pool.
64+
4. Wait for the node pool to recover and become RUNNING again.
65+
5. Wait for the Times To Recover (TTR) metrics to appear in Google Cloud Monitoring.
66+
6. Clean up the node pool after the tests.
67+
""",
68+
) as dag:
69+
for machine in MachineConfigMap:
70+
config = machine.value
71+
72+
with TaskGroup(group_id=f"v{config.tpu_version.value}"):
73+
node_pool_info = node_pool.build_node_pool_info_from_gcs_yaml(
74+
gcs_path=GCS_CONFIG_PATH,
75+
dag_name="node_pool_ttr_disk_size",
76+
is_prod=composer_env.is_prod_env(),
77+
machine_type=config.machine_version.value,
78+
tpu_topology=config.tpu_topology,
79+
)
80+
81+
create_node_pool = node_pool.create.override(task_id="create_node_pool")(
82+
node_pool=node_pool_info
83+
)
84+
85+
wait_for_provisioning = node_pool.wait_for_status.override(
86+
task_id="wait_for_provisioning"
87+
)(node_pool=node_pool_info, status=node_pool.Status.PROVISIONING)
88+
89+
wait_for_running = node_pool.wait_for_status.override(
90+
task_id="wait_for_running"
91+
)(node_pool=node_pool_info, status=node_pool.Status.RUNNING)
92+
93+
update_start_time = node_pool.update.override(task_id="update_node_pool")(
94+
node_pool=node_pool_info,
95+
spec=node_pool.NodePoolUpdateSpec.DiskSize(
96+
delta=_DISK_SIZE_INCREMENT
97+
),
98+
)
99+
100+
wait_for_recovered = node_pool.wait_for_status.override(
101+
task_id="wait_for_recovered"
102+
)(node_pool=node_pool_info, status=node_pool.Status.RUNNING)
103+
104+
wait_for_ttr = node_pool.wait_for_ttr(
105+
node_pool=node_pool_info, operation_start_time=update_start_time
106+
)
107+
108+
cleanup_node_pool = node_pool.delete.override(
109+
task_id="cleanup_node_pool", trigger_rule=TriggerRule.ALL_DONE
110+
)(node_pool=node_pool_info).as_teardown(
111+
setups=create_node_pool,
112+
)
113+
114+
chain(
115+
node_pool_info,
116+
create_node_pool,
117+
wait_for_provisioning,
118+
wait_for_running,
119+
update_start_time,
120+
wait_for_recovered,
121+
wait_for_ttr,
122+
cleanup_node_pool,
123+
)

dags/tpu_observability/node_pool_ttr_update_label.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
from airflow.utils.trigger_rule import TriggerRule
2222

2323
from dags import composer_env
24-
from dags.common.vm_resource import Region, Zone
2524
from dags.tpu_observability.configs.common import MachineConfigMap, GCS_CONFIG_PATH
2625
from dags.tpu_observability.utils import node_pool_util as node_pool
2726

@@ -93,9 +92,10 @@
9392
)
9493

9594
task_id = "update_node_pool_label"
96-
update_node_pool_label = node_pool.update_labels.override(
97-
task_id=task_id
98-
)(node_pool=node_pool_info, node_labels=LABELS_TO_UPDATE)
95+
update_node_pool_label = node_pool.update.override(task_id=task_id)(
96+
node_pool=node_pool_info,
97+
spec=node_pool.NodePoolUpdateSpec.Label(delta=LABELS_TO_UPDATE),
98+
)
9999

100100
task_id = "wait_for_recovered"
101101
wait_for_recovered = node_pool.wait_for_status.override(task_id=task_id)(

dags/tpu_observability/update_node_pool_label.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,12 @@
8888
task_id="wait_for_initial_availability"
8989
)(node_pool=node_pool_info, availability=True)
9090

91-
update_node_pool_label = node_pool.update_labels.override(
91+
update_node_pool_label = node_pool.update.override(
9292
task_id="update_node_pool_label"
93-
)(node_pool=node_pool_info, node_labels=LABELS_TO_UPDATE)
93+
)(
94+
node_pool=node_pool_info,
95+
spec=node_pool.NodePoolUpdateSpec.Label(delta=LABELS_TO_UPDATE),
96+
)
9497

9598
wait_for_unavailable = node_pool.wait_for_availability.override(
9699
task_id="wait_for_unavailability_after_update"

dags/tpu_observability/utils/node_pool_util.py

Lines changed: 148 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -605,42 +605,170 @@ def wait_for_ttr(
605605
return False
606606

607607

608+
def get_node_pool_disk_size(node_pool: Info) -> int:
609+
"""Gets the disk size of a GKE node pool using gcloud command.
610+
611+
Args:
612+
node_pool: An instance of the Info class that encapsulates the
613+
configuration and metadata of a GKE node pool.
614+
615+
Returns:
616+
The disk size of the node pool in GB.
617+
"""
618+
command = (
619+
f"gcloud container node-pools describe {node_pool.node_pool_name} "
620+
f"--project={node_pool.project_id} "
621+
f"--cluster={node_pool.cluster_name} "
622+
f"--location={node_pool.location} "
623+
f'--format="value(config.diskSizeGb)"'
624+
)
625+
626+
result = subprocess.run_exec(command).strip()
627+
628+
return int(result)
629+
630+
631+
def get_node_pool_labels(node_pool: Info) -> dict[str, str]:
632+
"""Gets the labels of a GKE node pool using gcloud command.
633+
634+
Args:
635+
node_pool: An instance of the Info class that encapsulates the
636+
configuration and metadata of a GKE node pool.
637+
638+
Returns:
639+
A dictionary contains the node pool labels.
640+
"""
641+
command = (
642+
f"gcloud container node-pools describe {node_pool.node_pool_name} "
643+
f"--project={node_pool.project_id} "
644+
f"--cluster={node_pool.cluster_name} "
645+
f"--location={node_pool.location} "
646+
f"--format='json(config.resourceLabels)'"
647+
)
648+
649+
result = (
650+
json.loads(subprocess.run_exec(command).strip())
651+
.get("config", {})
652+
.get("resourceLabels", {})
653+
)
654+
655+
return result
656+
657+
658+
class UpdateTarget(enum.Enum):
659+
"""Defines what to update on the node pool."""
660+
661+
DISK_SIZE = "disk-size"
662+
LABEL = "labels"
663+
664+
665+
@dataclasses.dataclass
666+
class NodePoolUpdateSpec:
667+
"""Configuration parameters defining a mutation on a GKE node pool.
668+
669+
Attributes:
670+
target: The specific node pool attribute to update.
671+
delta: The change to apply to the target's current state.
672+
"""
673+
674+
target: UpdateTarget
675+
delta: int | dict[str, str]
676+
677+
@staticmethod
678+
def DiskSize(delta: int) -> "NodePoolUpdateSpec":
679+
if not isinstance(delta, int):
680+
raise TypeError(f"Disk size delta must be an integer. Got: {type(delta)}")
681+
682+
if delta <= 0:
683+
raise ValueError(f"Disk size delta must be positive. Got: {delta}")
684+
685+
return NodePoolUpdateSpec(
686+
target=UpdateTarget.DISK_SIZE,
687+
delta=delta,
688+
)
689+
690+
@staticmethod
691+
def Label(delta: dict[str, str]) -> "NodePoolUpdateSpec":
692+
if not isinstance(delta, dict):
693+
raise TypeError(f"Label delta must be a dictionary. Got: {type(delta)}")
694+
695+
key_pattern = re.compile(r"^[a-z][a-z0-9_-]*$")
696+
for k, v in delta.items():
697+
if not isinstance(k, str) or not isinstance(v, str):
698+
raise TypeError(
699+
f"All label keys and values must be strings. "
700+
f"Found incompatible item: key='{k}'({type(k)}), "
701+
f"value='{v}'({type(v)})"
702+
)
703+
704+
if not key_pattern.match(k):
705+
raise ValueError(
706+
f"Invalid label key: '{k}'. "
707+
"Keys must start with a lowercase letter and contain only "
708+
"lowercase letters ([a-z]), numeric characters ([0-9]), "
709+
"underscores (_) and dashes (-)."
710+
)
711+
712+
return NodePoolUpdateSpec(
713+
target=UpdateTarget.LABEL,
714+
delta=delta,
715+
)
716+
717+
608718
@task
609-
def update_labels(node_pool: Info, node_labels: dict) -> TimeUtil:
610-
"""Updates the labels of a GKE node pool using gcloud command.
719+
def update(node_pool: Info, spec: NodePoolUpdateSpec) -> TimeUtil:
720+
"""Applies an update to a GKE node pool based on the provided specification.
611721
612-
This task updates GKE node pool labels via gcloud.
613-
It captures the current time before execution and returns it as
614-
a TimeUtil object for downstream tracking.
722+
This task performs a state-aware update. It retrieves the current node pool
723+
state, resolves the final desired configuration based on the provided `spec`,
724+
and executes the update operation.
615725
616726
Args:
617-
node_pool: An instance of the Info class.
618-
node_labels: A dictionary of labels to update or remove.
727+
node_pool: An instance of the Info class that encapsulates the
728+
configuration and metadata of a GKE node pool.
729+
spec: An instance of the NodePoolUpdateSpec class defining the
730+
update target and parameters.
619731
620732
Returns:
621733
A TimeUtil object representing the UTC timestamp when the operation started.
734+
735+
Raises:
736+
ValueError: If the target is unsupported.
622737
"""
623-
operation_start_time = TimeUtil.from_datetime(
624-
datetime.datetime.now(datetime.timezone.utc)
625-
)
738+
flags: list[str] = []
626739

627-
if not node_labels:
628-
logging.info("The specified label is empty, nothing to update.")
629-
return operation_start_time
740+
match spec.target:
741+
case UpdateTarget.DISK_SIZE:
742+
current_disk_size = get_node_pool_disk_size(node_pool=node_pool)
743+
updated_disk_size = current_disk_size + spec.delta
744+
flags.append(f"--{spec.target.value}={updated_disk_size}")
630745

631-
labels = []
746+
case UpdateTarget.LABEL:
747+
current_labels = get_node_pool_labels(node_pool=node_pool)
748+
updated_labels = []
749+
for key, val in spec.delta.items():
750+
if current_labels.get(key) == val:
751+
val += val
752+
updated_labels.append(f"{key}={val}")
753+
flags.append(f"--{spec.target.value}={','.join(updated_labels)}")
632754

633-
for key, val in node_labels.items():
634-
labels.append(f"{key}={val}")
755+
case _:
756+
raise ValueError(f"Unsupported target: {spec.target}")
635757

636-
command = (
758+
flags_str = " ".join(flags)
759+
760+
update_cmd = (
637761
f"gcloud container node-pools update {node_pool.node_pool_name} "
638762
f"--project={node_pool.project_id} "
639763
f"--cluster={node_pool.cluster_name} "
640764
f"--location={node_pool.location} "
641-
f"--labels={','.join(labels)} "
642-
"--quiet"
765+
f"--quiet "
766+
f"{flags_str}"
643767
)
644768

645-
subprocess.run_exec(command)
769+
operation_start_time = TimeUtil.from_datetime(
770+
datetime.datetime.now(datetime.timezone.utc)
771+
)
772+
773+
subprocess.run_exec(update_cmd)
646774
return operation_start_time

0 commit comments

Comments
 (0)