Skip to content

Commit a444b84

Browse files
authored
Fix CLI exiting while master starting (#2720)
1 parent 2ddae6e commit a444b84

1 file changed

Lines changed: 21 additions & 7 deletions

File tree

  • src/dstack/_internal/cli/services/configurators

src/dstack/_internal/cli/services/configurators/run.py

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,12 @@
4141
)
4242
from dstack._internal.core.models.repos.base import Repo
4343
from dstack._internal.core.models.resources import CPUSpec
44-
from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus
44+
from dstack._internal.core.models.runs import (
45+
JobStatus,
46+
JobSubmission,
47+
JobTerminationReason,
48+
RunStatus,
49+
)
4550
from dstack._internal.core.services.configs import ConfigManager
4651
from dstack._internal.core.services.diff import diff_models
4752
from dstack._internal.utils.common import local_time
@@ -166,12 +171,7 @@ def apply_configuration(
166171
# We can attach to run multiple times if it goes from running to pending (retried).
167172
while True:
168173
with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live:
169-
while run.status in (
170-
RunStatus.SUBMITTED,
171-
RunStatus.PENDING,
172-
RunStatus.PROVISIONING,
173-
RunStatus.TERMINATING,
174-
):
174+
while not _is_ready_to_attach(run):
175175
table = get_runs_table([run])
176176
live.update(table)
177177
time.sleep(5)
@@ -590,6 +590,20 @@ def get_run_exit_code(run: Run) -> int:
590590
return 1
591591

592592

593+
def _is_ready_to_attach(run: Run) -> bool:
594+
return not (
595+
run.status
596+
in [
597+
RunStatus.SUBMITTED,
598+
RunStatus.PENDING,
599+
RunStatus.PROVISIONING,
600+
RunStatus.TERMINATING,
601+
]
602+
or run._run.jobs[0].job_submissions[-1].status
603+
in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING]
604+
)
605+
606+
593607
def _get_run_termination_reason_and_exit_status(
594608
run: Run,
595609
) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]:

0 commit comments

Comments
 (0)