|
| 1 | +"""Job monitoring utilities for AWS Batch jobs.""" |
| 2 | + |
| 3 | +import time |
| 4 | +from typing import Any |
| 5 | + |
| 6 | +import boto3 |
| 7 | +from rich.console import Console |
| 8 | +from rich.live import Live |
| 9 | +from rich.table import Table |
| 10 | + |
| 11 | +from codeclash.utils.log import get_logger |
| 12 | + |
| 13 | +logger = get_logger("job_monitor", emoji="👀") |
| 14 | + |
| 15 | + |
| 16 | +class JobMonitor: |
| 17 | + def __init__(self, region: str = "us-east-1"): |
| 18 | + self.batch_client = boto3.client("batch", region_name=region) |
| 19 | + self.console = Console() |
| 20 | + |
| 21 | + def get_job_status(self, job_id: str) -> dict[str, Any]: |
| 22 | + """Get the current status of a job.""" |
| 23 | + response = self.batch_client.describe_jobs(jobs=[job_id]) |
| 24 | + if response["jobs"]: |
| 25 | + return response["jobs"][0] |
| 26 | + else: |
| 27 | + raise ValueError(f"Job {job_id} not found") |
| 28 | + |
| 29 | + @staticmethod |
| 30 | + def format_duration(seconds: float) -> str: |
| 31 | + """Format duration in HH:MM:SS""" |
| 32 | + hours = int(seconds // 3600) |
| 33 | + minutes = int((seconds % 3600) // 60) |
| 34 | + secs = int(seconds % 60) |
| 35 | + return f"{hours:02d}:{minutes:02d}:{secs:02d}" |
| 36 | + |
| 37 | + @staticmethod |
| 38 | + def get_status_color(status: str) -> str: |
| 39 | + """Get the color for a job status.""" |
| 40 | + status_lower = status.lower() |
| 41 | + if status_lower in ["submitted", "pending", "runnable"]: |
| 42 | + return "cyan" |
| 43 | + elif status_lower in ["starting", "running"]: |
| 44 | + return "yellow" |
| 45 | + elif status_lower == "succeeded": |
| 46 | + return "green" |
| 47 | + elif status_lower == "failed": |
| 48 | + return "red" |
| 49 | + else: |
| 50 | + return "" |
| 51 | + |
| 52 | + def create_status_table(self, jobs_info: dict[str, dict]) -> Table: |
| 53 | + """Create a rich table with job status information""" |
| 54 | + table = Table(title="AWS Batch Jobs Status") |
| 55 | + table.add_column("Status") |
| 56 | + table.add_column("Runtime", justify="right", style="magenta") |
| 57 | + table.add_column("Job ID", style="yellow") |
| 58 | + table.add_column("Job Name", style="blue") |
| 59 | + table.add_column("Config", style="green") |
| 60 | + |
| 61 | + for job_id, info in jobs_info.items(): |
| 62 | + status = info["status"] |
| 63 | + color = self.get_status_color(status) |
| 64 | + status_colored = f"[{color}]{status}[/{color}]" if color else status |
| 65 | + |
| 66 | + table.add_row( |
| 67 | + status_colored, |
| 68 | + info["runtime"], |
| 69 | + job_id[:13] + "...", |
| 70 | + info["job_name"], |
| 71 | + info["config"], |
| 72 | + ) |
| 73 | + |
| 74 | + return table |
| 75 | + |
| 76 | + def monitor(self, job_info_dict: dict[str, dict[str, str]]) -> None: |
| 77 | + """Monitor submitted jobs and display status in real-time. |
| 78 | +
|
| 79 | + Args: |
| 80 | + job_info_dict: Dict mapping job_id -> {job_name, config} |
| 81 | + """ |
| 82 | + jobs_info: dict[str, dict] = {} |
| 83 | + start_times: dict[str, float] = {job_id: time.time() for job_id in job_info_dict} |
| 84 | + completion_times: dict[str, float] = {} |
| 85 | + |
| 86 | + self.console.print("\n[bold green]Note:[/] You can ^C this script without your jobs dying\n") |
| 87 | + |
| 88 | + try: |
| 89 | + with Live(self.create_status_table({}), refresh_per_second=1, console=self.console) as live: |
| 90 | + while True: |
| 91 | + all_done = True |
| 92 | + for job_id, info in job_info_dict.items(): |
| 93 | + try: |
| 94 | + job_status = self.get_job_status(job_id) |
| 95 | + status = job_status["status"] |
| 96 | + |
| 97 | + # Freeze runtime when job reaches terminal state |
| 98 | + if status in ["SUCCEEDED", "FAILED"] and job_id not in completion_times: |
| 99 | + completion_times[job_id] = time.time() |
| 100 | + |
| 101 | + if job_id in completion_times: |
| 102 | + elapsed = completion_times[job_id] - start_times[job_id] |
| 103 | + else: |
| 104 | + elapsed = time.time() - start_times[job_id] |
| 105 | + |
| 106 | + jobs_info[job_id] = { |
| 107 | + "status": status, |
| 108 | + "runtime": self.format_duration(elapsed), |
| 109 | + "job_name": info["job_name"], |
| 110 | + "config": info["config"], |
| 111 | + } |
| 112 | + |
| 113 | + if status not in ["SUCCEEDED", "FAILED"]: |
| 114 | + all_done = False |
| 115 | + except Exception as e: |
| 116 | + logger.error(f"Error getting status for {job_id}: {e}", exc_info=True) |
| 117 | + if job_id not in completion_times: |
| 118 | + completion_times[job_id] = time.time() |
| 119 | + jobs_info[job_id] = { |
| 120 | + "status": "ERROR", |
| 121 | + "runtime": self.format_duration(completion_times[job_id] - start_times[job_id]), |
| 122 | + "job_name": info["job_name"], |
| 123 | + "config": info["config"], |
| 124 | + } |
| 125 | + |
| 126 | + live.update(self.create_status_table(jobs_info)) |
| 127 | + |
| 128 | + if all_done: |
| 129 | + break |
| 130 | + |
| 131 | + time.sleep(1) |
| 132 | + |
| 133 | + self.console.print("\n[bold green]All jobs completed![/]") |
| 134 | + except KeyboardInterrupt: |
| 135 | + self.console.print("\n[bold yellow]Monitoring stopped. Jobs continue running on AWS.[/]") |
0 commit comments