Make run and job termination_reason a string

Author: r4victor

src/dstack/_internal/core/models/runs.py

@@ -344,15 +344,17 @@ class JobSubmission(CoreModel):
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
     submitted_at: datetime
     last_processed_at: datetime
-    finished_at: Optional[datetime]
-    inactivity_secs: Optional[int]
+    finished_at: Optional[datetime] = None
+    inactivity_secs: Optional[int] = None
     status: JobStatus
     status_message: str = ""  # default for backward compatibility
-    termination_reason: Optional[JobTerminationReason]
-    termination_reason_message: Optional[str]
-    exit_status: Optional[int]
-    job_provisioning_data: Optional[JobProvisioningData]
-    job_runtime_data: Optional[JobRuntimeData]
+    # termination_reason stores JobTerminationReason.
+    # str allows adding new enum members without breaking compatibility with old clients.
+    termination_reason: Optional[str] = None
+    termination_reason_message: Optional[str] = None
+    exit_status: Optional[int] = None
+    job_provisioning_data: Optional[JobProvisioningData] = None
+    job_runtime_data: Optional[JobRuntimeData] = None
     error: Optional[str] = None
     probes: list[Probe] = []
 
@@ -502,7 +504,9 @@ class Run(CoreModel):
     last_processed_at: datetime
     status: RunStatus
     status_message: str = ""  # default for backward compatibility
-    termination_reason: Optional[RunTerminationReason] = None
+    # termination_reason stores RunTerminationReason.
+    # str allows adding new enum members without breaking compatibility with old clients.
+    termination_reason: Optional[str] = None
     run_spec: RunSpec
     jobs: List[Job]
     latest_job_submission: Optional[JobSubmission] = None

src/dstack/_internal/server/background/tasks/process_runs.py

@@ -574,7 +574,7 @@ def _should_retry_job(run: Run, job: Job, job_model: JobModel) -> Optional[datet
 
     if (
         last_provisioned_submission.termination_reason is not None
-        and last_provisioned_submission.termination_reason.to_retry_event()
+        and JobTerminationReason(last_provisioned_submission.termination_reason).to_retry_event()
         in job.job_spec.retry.on_events
     ):
         return common.get_current_datetime() - last_provisioned_submission.last_processed_at

src/dstack/_internal/server/services/jobs/__init__.py

@@ -152,7 +152,9 @@ def job_model_to_job_submission(
         inactivity_secs=job_model.inactivity_secs,
         status=job_model.status,
         status_message=status_message,
-        termination_reason=job_model.termination_reason,
+        termination_reason=job_model.termination_reason.value
+        if job_model.termination_reason
+        else None,
         termination_reason_message=job_model.termination_reason_message,
         exit_status=job_model.exit_status,
         job_provisioning_data=job_provisioning_data,

src/dstack/_internal/server/services/runs.py

@@ -714,7 +714,9 @@ def run_model_to_run(
         last_processed_at=run_model.last_processed_at,
         status=run_model.status,
         status_message=status_message,
-        termination_reason=run_model.termination_reason,
+        termination_reason=run_model.termination_reason.value
+        if run_model.termination_reason
+        else None,
         run_spec=run_spec,
         jobs=jobs,
         latest_job_submission=latest_job_submission,