|
41 | 41 | ) |
42 | 42 | from dstack._internal.core.models.repos.base import Repo |
43 | 43 | from dstack._internal.core.models.resources import CPUSpec |
44 | | -from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus |
| 44 | +from dstack._internal.core.models.runs import ( |
| 45 | + JobStatus, |
| 46 | + JobSubmission, |
| 47 | + JobTerminationReason, |
| 48 | + RunStatus, |
| 49 | +) |
45 | 50 | from dstack._internal.core.services.configs import ConfigManager |
46 | 51 | from dstack._internal.core.services.diff import diff_models |
47 | 52 | from dstack._internal.utils.common import local_time |
@@ -166,12 +171,7 @@ def apply_configuration( |
166 | 171 | # We can attach to run multiple times if it goes from running to pending (retried). |
167 | 172 | while True: |
168 | 173 | with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live: |
169 | | - while run.status in ( |
170 | | - RunStatus.SUBMITTED, |
171 | | - RunStatus.PENDING, |
172 | | - RunStatus.PROVISIONING, |
173 | | - RunStatus.TERMINATING, |
174 | | - ): |
| 174 | + while not _is_ready_to_attach(run): |
175 | 175 | table = get_runs_table([run]) |
176 | 176 | live.update(table) |
177 | 177 | time.sleep(5) |
@@ -590,6 +590,20 @@ def get_run_exit_code(run: Run) -> int: |
590 | 590 | return 1 |
591 | 591 |
|
592 | 592 |
|
| 593 | +def _is_ready_to_attach(run: Run) -> bool: |
| 594 | + return not ( |
| 595 | + run.status |
| 596 | + in [ |
| 597 | + RunStatus.SUBMITTED, |
| 598 | + RunStatus.PENDING, |
| 599 | + RunStatus.PROVISIONING, |
| 600 | + RunStatus.TERMINATING, |
| 601 | + ] |
| 602 | + or run._run.jobs[0].job_submissions[-1].status |
| 603 | + in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING] |
| 604 | + ) |
| 605 | + |
| 606 | + |
593 | 607 | def _get_run_termination_reason_and_exit_status( |
594 | 608 | run: Run, |
595 | 609 | ) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]: |
|
0 commit comments