diff --git a/frontend/src/libs/run.ts b/frontend/src/libs/run.ts index 0f3caa9942..5a622a9b6b 100644 --- a/frontend/src/libs/run.ts +++ b/frontend/src/libs/run.ts @@ -2,6 +2,7 @@ import { get as _get } from 'lodash'; import { StatusIndicatorProps } from '@cloudscape-design/components'; import { capitalize } from 'libs'; +import { finishedRunStatuses } from '../pages/Runs/constants'; import { IModelExtended } from '../pages/Models/List/types'; @@ -9,7 +10,7 @@ export const getStatusIconType = ( status: IRun['status'] | TJobStatus, terminationReason: string | null | undefined, ): StatusIndicatorProps['type'] => { - if (terminationReason === 'interrupted_by_no_capacity') { + if (finishedRunStatuses.includes(status) && terminationReason === 'interrupted_by_no_capacity') { return 'stopped'; } switch (status) { @@ -41,24 +42,26 @@ export const getStatusIconColor = ( if (terminationReason === 'failed_to_start_due_to_no_capacity' || terminationReason === 'interrupted_by_no_capacity') { return 'yellow'; } - switch (status) { + case 'submitted': + case 'pending': + return 'blue'; case 'pulling': return 'green'; case 'aborted': return 'yellow'; case 'done': - return 'blue'; + return 'grey'; default: return undefined; } }; export const getRunStatusMessage = (run: IRun): string => { - if (run.latest_job_submission?.status_message) { + if (finishedRunStatuses.includes(run.status) && run.latest_job_submission?.status_message) { return capitalize(run.latest_job_submission.status_message); } else { - return capitalize(run.status); + return capitalize(run.status_message || run.status); } }; diff --git a/frontend/src/pages/Runs/Details/RunDetails/index.tsx b/frontend/src/pages/Runs/Details/RunDetails/index.tsx index b336257d4a..0d812b1c82 100644 --- a/frontend/src/pages/Runs/Details/RunDetails/index.tsx +++ b/frontend/src/pages/Runs/Details/RunDetails/index.tsx @@ -24,6 +24,7 @@ import { Logs } from '../Logs'; import { getJobSubmissionId } from '../Logs/helpers'; import styles from './styles.module.scss'; +import { finishedRunStatuses } from 'pages/Runs/constants'; export const RunDetails = () => { const { t } = useTranslation(); @@ -47,8 +48,8 @@ export const RunDetails = () => { if (!runData) return null; - const status = runData.latest_job_submission?.status ?? runData.status; - const terminationReason = runData.latest_job_submission?.termination_reason; + const status = finishedRunStatuses.includes(runData.status) ? runData.latest_job_submission?.status ?? runData.status : runData.status; + const terminationReason = finishedRunStatuses.includes(runData.status) ? runData.latest_job_submission?.termination_reason : null; return ( <> diff --git a/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx b/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx index 244c1b07bf..62622e934b 100644 --- a/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx +++ b/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx @@ -16,6 +16,7 @@ import { getRunListItemResources, getRunListItemSpotLabelKey, } from '../helpers'; +import { finishedRunStatuses } from 'pages/Runs/constants'; export const useColumnsDefinitions = () => { const { t } = useTranslation(); @@ -65,8 +66,8 @@ export const useColumnsDefinitions = () => { id: 'status', header: t('projects.run.status'), cell: (item: IRun) => { - const status = item.latest_job_submission?.status ?? item.status; - const terminationReason = item.latest_job_submission?.termination_reason; + const status = finishedRunStatuses.includes(item.status) ? item.latest_job_submission?.status ?? item.status : item.status; + const terminationReason = finishedRunStatuses.includes(item.status) ? item.latest_job_submission?.termination_reason : null; return ( Optional[s else: return None + @root_validator + def _status_message(cls, values) -> Dict: + try: + status = values["status"] + run_spec: RunSpec = values["run_spec"] + retry_on_events = ( + run_spec.configuration.retry.on_events + if run_spec and run_spec.configuration.retry + else [] + ) + jobs = values["jobs"] + termination_reason = Run.get_last_termination_reason(jobs[0]) if jobs else None + except KeyError: + return values + values["status_message"] = Run._get_status_message( + status=status, + retry_on_events=retry_on_events, + termination_reason=termination_reason, + ) + return values + + @staticmethod + def get_last_termination_reason(job: "Job") -> Optional[JobTerminationReason]: + for submission in reversed(job.job_submissions): + if submission.termination_reason is not None: + return submission.termination_reason + return None + + @staticmethod + def _get_status_message( + status: RunStatus, + retry_on_events: List[RetryEvent], + termination_reason: Optional[JobTerminationReason], + ) -> str: + # Currently, `retrying` is shown only for `no-capacity` events + if ( + status in [RunStatus.SUBMITTED, RunStatus.PENDING] + and termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + and RetryEvent.NO_CAPACITY in retry_on_events + ): + return "retrying" + return status.value + class JobPlan(CoreModel): job_spec: JobSpec diff --git a/src/dstack/api/server/_runs.py b/src/dstack/api/server/_runs.py index 22f994cc4c..cfd70d6d5e 100644 --- a/src/dstack/api/server/_runs.py +++ b/src/dstack/api/server/_runs.py @@ -100,6 +100,7 @@ def _get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]: current_resource = plan.current_resource if current_resource is not None: current_resource_excludes = {} + current_resource_excludes["status_message"] = True apply_plan_excludes["current_resource"] = current_resource_excludes current_resource_excludes["run_spec"] = _get_run_spec_excludes(current_resource.run_spec) job_submissions_excludes = {} diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 06ad20e76a..60d2519979 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -247,6 +247,7 @@ def get_dev_env_run_dict( "submitted_at": submitted_at, "last_processed_at": last_processed_at, "status": "submitted", + "status_message": "submitted", "run_spec": { "configuration": { "entrypoint": None, @@ -510,6 +511,7 @@ async def test_lists_runs(self, test_db, session: AsyncSession, client: AsyncCli "submitted_at": run1_submitted_at.isoformat(), "last_processed_at": run1_submitted_at.isoformat(), "status": "submitted", + "status_message": "submitted", "run_spec": run1_spec.dict(), "jobs": [ { @@ -563,6 +565,7 @@ async def test_lists_runs(self, test_db, session: AsyncSession, client: AsyncCli "submitted_at": run2_submitted_at.isoformat(), "last_processed_at": run2_submitted_at.isoformat(), "status": "submitted", + "status_message": "submitted", "run_spec": run2_spec.dict(), "jobs": [], "latest_job_submission": None,