Remove the labels from job run latency histogram metrics

GitOrigin-RevId: 82aee74

Author: Weiwei Yang

axlearn/cloud/common/metrics.py

@@ -9,14 +9,12 @@
 BASTION_DOWNLOAD_JOB_ALL_SECONDS = Histogram(
     "bastion_download_job_all_seconds",
     "Time in seconds consumed in each scheduling loop to download all the job specs",
-    [],
     buckets=(1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600),  # 1s to 1hr
 )
 
 JOB_TIME_TO_RUNNING_SECONDS = Histogram(
     "bastion_job_time_to_running_seconds",
     "Time in seconds from runner starts a job until until it reaches Running state",
-    ["job_name"],
     buckets=(1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600),  # 1s to 1hr
 )
 
@@ -35,7 +33,6 @@
 JOB_SUSPENDED_DURATION_SECONDS = Histogram(
     "bastion_job_suspended_duration_seconds",
     "Time in seconds the jobset stayed in SUSPENDED state",
-    ["job_name"],
     buckets=(1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600),  # 1s to 1hr
 )
 
@@ -115,15 +112,14 @@ def get_multiprocess_dir(cls) -> str:
         return cls._multiprocess_dir
 
 
-def record_job_time_to_running(job_name: str, time_seconds: float):
+def record_job_time_to_running(time_seconds: float):
     """Record the time it took for a job to reach RUNNING state.
 
     Args:
         job_name: Name of the job
         time_seconds: Time in seconds from first seen until RUNNING state
     """
-    JOB_TIME_TO_RUNNING_SECONDS.labels(job_name=job_name).observe(time_seconds)
-    logging.info("Metrics: job %s took %.2f seconds to reach RUNNING state", job_name, time_seconds)
+    JOB_TIME_TO_RUNNING_SECONDS.observe(time_seconds)
 
 
 def record_job_run_latency(job_name: str, time_seconds: float):
@@ -141,15 +137,14 @@ def record_job_wait_for_build(job_name: str, time_seconds: float):
     JOB_WAIT_FOR_BUILD_SECONDS.labels(job_name).set(time_seconds)
 
 
-def record_job_suspended_duration(job_name: str, time_seconds: float):
+def record_job_suspended_duration(time_seconds: float):
     """Record the time a job spent in SUSPENDED state.
 
     Args:
         job_name: Name of the job
         time_seconds: Time in seconds the job was in SUSPENDED state
     """
-    JOB_SUSPENDED_DURATION_SECONDS.labels(job_name=job_name).observe(time_seconds)
-    logging.info("Metrics: job %s was in SUSPENDED state for %.2f seconds", job_name, time_seconds)
+    JOB_SUSPENDED_DURATION_SECONDS.observe(time_seconds)
 
 
 def cleanup(pid=None):

axlearn/cloud/gcp/runners/gke.py

@@ -659,7 +659,7 @@ def _execute(self):
                 if suspended_since is not None:
                     # Job exited SUSPENDED state, record the duration.
                     suspended_duration = time.perf_counter() - suspended_since
-                    metrics.record_job_suspended_duration(cfg.name, suspended_duration)
+                    metrics.record_job_suspended_duration(suspended_duration)
                     suspended_since = None
 
                 # Ensure VertexAI Tensorboard Uploader is running.
@@ -670,7 +670,7 @@ def _execute(self):
                 if status == GKERunnerJob.Status.READY and status != last_job_status:
                     # Record the time to reach RUNNING state.
                     elapsed_time = time.perf_counter() - start_time
-                    metrics.record_job_time_to_running(cfg.name, elapsed_time)
+                    metrics.record_job_time_to_running(elapsed_time)
                     metrics.record_job_run_latency(cfg.name, elapsed_time)
                     self._maybe_publish(
                         cfg.name, msg="Job is running", state=JobLifecycleState.RUNNING