Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions src/dstack/_internal/cli/services/configurators/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@
)
from dstack._internal.core.models.repos.base import Repo
from dstack._internal.core.models.resources import CPUSpec
from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus
from dstack._internal.core.models.runs import (
JobStatus,
JobSubmission,
JobTerminationReason,
RunStatus,
)
from dstack._internal.core.services.configs import ConfigManager
from dstack._internal.core.services.diff import diff_models
from dstack._internal.utils.common import local_time
Expand Down Expand Up @@ -166,12 +171,7 @@ def apply_configuration(
# We can attach to run multiple times if it goes from running to pending (retried).
while True:
with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live:
while run.status in (
RunStatus.SUBMITTED,
RunStatus.PENDING,
RunStatus.PROVISIONING,
RunStatus.TERMINATING,
):
while not _is_ready_to_attach(run):
table = get_runs_table([run])
live.update(table)
time.sleep(5)
Expand Down Expand Up @@ -590,6 +590,20 @@ def get_run_exit_code(run: Run) -> int:
return 1


def _is_ready_to_attach(run: Run) -> bool:
return not (
run.status
in [
RunStatus.SUBMITTED,
RunStatus.PENDING,
RunStatus.PROVISIONING,
RunStatus.TERMINATING,
]
or run._run.jobs[0].job_submissions[-1].status
in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING]
)


def _get_run_termination_reason_and_exit_status(
run: Run,
) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]:
Expand Down
Loading