Skip to content

Commit 0e9bc42

Browse files
authored
Support dynamic run waiting CLI status with extra renderables (#3760)
* Support dynamic run waiting CLI status with extra renderables * Extract _get_service_url_renderable * Remove tests
1 parent 6fd1f1b commit 0e9bc42

File tree

3 files changed

+82
-5
lines changed

3 files changed

+82
-5
lines changed

src/dstack/_internal/cli/services/configurators/run.py

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@
2828
from dstack._internal.cli.services.resources import apply_resources_args, register_resources_args
2929
from dstack._internal.cli.utils.common import confirm_ask, console
3030
from dstack._internal.cli.utils.rich import MultiItemStatus
31-
from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
31+
from dstack._internal.cli.utils.run import (
32+
RunWaitStatus,
33+
get_run_wait_status,
34+
get_runs_table,
35+
print_run_plan,
36+
)
3237
from dstack._internal.core.errors import (
3338
CLIError,
3439
ConfigurationError,
@@ -192,10 +197,14 @@ def apply_configuration(
192197
try:
193198
# We can attach to run multiple times if it goes from running to pending (retried).
194199
while True:
195-
with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live:
200+
with MultiItemStatus(_get_apply_status(run), console=console) as live:
196201
while not _is_ready_to_attach(run):
197202
table = get_runs_table([run])
198-
live.update(table)
203+
live.update(
204+
table,
205+
*_get_apply_wait_renderables(run),
206+
status=_get_apply_status(run),
207+
)
199208
time.sleep(5)
200209
run.refresh()
201210

@@ -793,14 +802,38 @@ def _detect_windsurf_version(exe: str = "windsurf") -> Optional[str]:
793802
def _print_service_urls(run: Run) -> None:
794803
if run._run.run_spec.configuration.type != RunConfigurationType.SERVICE.value:
795804
return
796-
console.print(f"Service is published at:\n [link={run.service_url}]{run.service_url}[/]")
805+
console.print(_get_service_url_renderable(run))
797806
if model := run.service_model:
798807
console.print(
799808
f"Model [code]{model.name}[/] is published at:\n [link={model.url}]{model.url}[/]"
800809
)
801810
console.print()
802811

803812

813+
def _get_apply_status(run: Run) -> str:
814+
wait_status = get_run_wait_status(run._run)
815+
if wait_status is None:
816+
return f"Launching [code]{run.name}[/]..."
817+
return f"[code]{run.name}[/] is {wait_status.value}..."
818+
819+
820+
def _get_apply_wait_renderables(run: Run) -> list[str]:
821+
wait_status = get_run_wait_status(run._run)
822+
if wait_status is RunWaitStatus.WAITING_FOR_REQUESTS and run._run.service is not None:
823+
return [_get_service_url_renderable(run)]
824+
if (
825+
wait_status is RunWaitStatus.WAITING_FOR_SCHEDULE
826+
and run._run.next_triggered_at is not None
827+
):
828+
next_run = run._run.next_triggered_at.astimezone().strftime("%Y-%m-%d %H:%M %Z")
829+
return [f"Next run: {next_run}"]
830+
return []
831+
832+
833+
def _get_service_url_renderable(run: Run) -> str:
834+
return f"Service is published at:\n [link={run.service_url}]{run.service_url}[/]"
835+
836+
804837
def _print_dev_environment_connection_info(run: Run) -> None:
805838
if not FeatureFlags.CLI_PRINT_JOB_CONNECTION_INFO:
806839
return

src/dstack/_internal/cli/utils/rich.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,11 @@ def __init__(self, status: "RenderableType", *, console: Optional["Console"] = N
140140
transient=True,
141141
)
142142

143-
def update(self, *renderables: "RenderableType") -> None:
143+
def update(
144+
self, *renderables: "RenderableType", status: Optional["RenderableType"] = None
145+
) -> None:
146+
if status is not None:
147+
self._spinner.update(text=status)
144148
self._live.update(renderable=Group(self._spinner, *renderables))
145149

146150
def __enter__(self) -> "MultiItemStatus":

src/dstack/_internal/cli/utils/run.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import shutil
2+
from enum import Enum
23
from typing import Any, Dict, List, Optional
34

45
from rich.markup import escape
@@ -49,6 +50,11 @@
4950
from dstack.api import Run
5051

5152

53+
class RunWaitStatus(str, Enum):
54+
WAITING_FOR_REQUESTS = "waiting for requests"
55+
WAITING_FOR_SCHEDULE = "waiting for schedule"
56+
57+
5258
def print_offers_json(run_plan: RunPlan, run_spec):
5359
"""Print offers information in JSON format."""
5460
job_plan = run_plan.job_plans[0]
@@ -200,6 +206,40 @@ def th(s: str) -> str:
200206
console.print(NO_FLEETS_WARNING if no_fleets else NO_OFFERS_WARNING)
201207

202208

209+
def get_run_wait_status(run: CoreRun) -> Optional[RunWaitStatus]:
210+
# Only synthesize a CLI-specific waiting state when the server did not provide
211+
# a more specific run-level message such as "retrying".
212+
if run.status_message not in ("", run.status.value):
213+
return None
214+
215+
if run.status == RunStatus.PENDING and run.next_triggered_at is not None:
216+
return RunWaitStatus.WAITING_FOR_SCHEDULE
217+
218+
if _is_waiting_for_requests(run):
219+
return RunWaitStatus.WAITING_FOR_REQUESTS
220+
221+
return None
222+
223+
224+
def _is_waiting_for_requests(run: CoreRun) -> bool:
225+
if run.run_spec.configuration.type != "service":
226+
return False
227+
if run.service is None or run.next_triggered_at is not None:
228+
return False
229+
if run.status not in (RunStatus.SUBMITTED, RunStatus.PENDING):
230+
return False
231+
return not any(_is_job_active(job.job_submissions[-1].status) for job in run.jobs)
232+
233+
234+
def _is_job_active(status: JobStatus) -> bool:
235+
return status in (
236+
JobStatus.SUBMITTED,
237+
JobStatus.PROVISIONING,
238+
JobStatus.PULLING,
239+
JobStatus.RUNNING,
240+
)
241+
242+
203243
def _format_run_status(run) -> str:
204244
status_text = (
205245
run.latest_job_submission.status_message

0 commit comments

Comments
 (0)