3030from google .cloud .monitoring_v3 import types
3131import kubernetes
3232
33- from dags .tpu_observability .utils import node_pool_util as node_pool
33+ from dags .tpu_observability .utils . node_pool_util import Info as node_pool_info
3434from dags .tpu_observability .utils import subprocess_util as subprocess
3535from dags .tpu_observability .utils .gcp_util import query_time_series
3636from dags .tpu_observability .utils .time_util import TimeUtil
@@ -224,7 +224,7 @@ class Command:
224224 """
225225
226226 @staticmethod
227- def get_credentials_command (node_pool : node_pool . Info ) -> str :
227+ def get_credentials_command (node_pool : node_pool_info ) -> str :
228228 """
229229 Returns the command to authenticate `gcloud` with the specified GKE cluster.
230230
@@ -274,7 +274,7 @@ def k8s_get_pod_name_command(kubeconfig: str, namespace: str) -> str:
274274
275275
276276def get_replica_num (
277- replica_type : str , job_name : str , node_pool : node_pool . Info
277+ replica_type : str , job_name : str , node_pool : node_pool_info
278278) -> int :
279279 """
280280 Get the number of a certain type of replicas from a running jobset.
@@ -291,7 +291,9 @@ def get_replica_num(
291291 The number of replicas of the specific type in the jobset.
292292 """
293293 api_client = gke .get_authenticated_client (
294- node_pool .project_id , node_pool .region , node_pool .cluster_name
294+ node_pool .project_id ,
295+ node_pool .region ,
296+ node_pool .cluster_name ,
295297 )
296298
297299 api = kubernetes .client .CustomObjectsApi (api_client )
@@ -322,7 +324,7 @@ def get_replica_num(
322324
323325
324326def get_running_pods (
325- node_pool : node_pool . Info , namespace = "default"
327+ node_pool : node_pool_info , namespace = "default"
326328) -> list [str ]:
327329 """
328330 Get a list of pods which are in the "running" state.
@@ -362,7 +364,7 @@ def get_running_pods(
362364
363365@task
364366def run_workload (
365- node_pool : node_pool . Info , yaml_config : str , namespace : str
367+ node_pool : node_pool_info , yaml_config : str , namespace : str
366368) -> TimeUtil :
367369 """
368370 Applies the specified YAML file to the GKE cluster.
@@ -386,11 +388,11 @@ def run_workload(
386388 subprocess .run_exec (cmd , env = env )
387389
388390 current_time_utc = datetime .datetime .now (datetime .timezone .utc )
389- return current_time_utc
391+ return TimeUtil . from_datetime ( current_time_utc )
390392
391393
392394@task
393- def end_workload (node_pool : node_pool . Info , jobset_name : str , namespace : str ):
395+ def end_workload (node_pool : node_pool_info , jobset_name : str , namespace : str ):
394396 """
395397 Deletes all JobSets from the GKE cluster to clean up resources.
396398
@@ -418,7 +420,7 @@ def end_workload(node_pool: node_pool.Info, jobset_name: str, namespace: str):
418420
419421
420422@task
421- def get_active_pods (node_pool : node_pool . Info , namespace : str ) -> list [str ]:
423+ def get_active_pods (node_pool : node_pool_info , namespace : str ) -> list [str ]:
422424 """
423425 Deletes all JobSets from the GKE cluster to clean up resources.
424426
@@ -454,9 +456,9 @@ def get_active_pods(node_pool: node_pool.Info, namespace: str) -> list[str]:
454456
455457@task .sensor (poke_interval = 30 , timeout = 900 , mode = "reschedule" )
456458def wait_for_jobset_started (
457- node_pool : node_pool . Info ,
459+ node_pool : node_pool_info ,
458460 pod_name_list : str ,
459- job_apply_time : datetime . datetime ,
461+ job_apply_time : TimeUtil ,
460462) -> bool :
461463 """
462464 Waits for the jobset to start by polling Cloud Logging for positive tensorcore
@@ -473,8 +475,10 @@ def wait_for_jobset_started(
473475 job_apply_time: The datetime object of the time the job was applied.
474476 """
475477
476- end_time_datatime = job_apply_time + datetime .timedelta (minutes = 10 )
477- start_time = TimeUtil .from_datetime (job_apply_time )
478+ end_time_datatime = job_apply_time .to_datetime () + datetime .timedelta (
479+ minutes = 10
480+ )
481+ start_time = job_apply_time
478482 end_time = TimeUtil .from_datetime (end_time_datatime )
479483
480484 if not pod_name_list :
@@ -518,7 +522,7 @@ def wait_for_jobset_started(
518522
519523
520524@task .sensor (poke_interval = 60 , timeout = 3600 , mode = "reschedule" )
521- def wait_for_jobset_ttr_to_be_found (node_pool : node_pool . Info ) -> bool :
525+ def wait_for_jobset_ttr_to_be_found (node_pool : node_pool_info ) -> bool :
522526 """
523527 Polls the jobset time_between_interruptions metric.
524528
@@ -553,7 +557,7 @@ def wait_for_jobset_ttr_to_be_found(node_pool: node_pool.Info) -> bool:
553557
554558@task .sensor (poke_interval = 30 , timeout = 600 , mode = "reschedule" )
555559def wait_for_jobset_status_occurrence (
556- replica_type : str , job_name : str , node_pool : node_pool . Info
560+ replica_type : str , job_name : str , node_pool : node_pool_info
557561):
558562 """
559563 A sensor which checks if are any jobset replicas in a status type.
@@ -574,6 +578,6 @@ def wait_for_jobset_status_occurrence(
574578
575579
576580@task .sensor (poke_interval = 30 , timeout = 600 , mode = "reschedule" )
577- def wait_for_all_pods_running (num_pods : int , node_pool : node_pool . Info ):
581+ def wait_for_all_pods_running (num_pods : int , node_pool : node_pool_info ):
578582 num_running = len (get_running_pods (node_pool = node_pool , namespace = "default" ))
579583 return num_running == num_pods
0 commit comments