Skip to content

Commit 69483b6

Browse files
authored
Slack integration (#19)
* init slack integration * fix status output
1 parent 093044b commit 69483b6

1 file changed

Lines changed: 5 additions & 5 deletions

File tree

app/core/monitor.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ async def monitor_jobs(self):
153153
f"Job {job_id} disappeared from API, probably canceled by user."
154154
)
155155
await send_slack_notification(
156-
f"Job `{job_info.status}`: `{job_info.job_name} ({job_id})` by `{job_info.user_id}`\nModel: `{job_info.model_name}` | Started: `{job_info.created_at}`"
156+
f"Job `{job_info.status.value}`: `{job_info.job_name} ({job_id})` by `{job_info.user_id}`\nModel: `{job_info.model_name}` | Started: `{job_info.created_at}`"
157157
)
158158
self.notified_queued_jobs.discard(job_id)
159159

@@ -185,7 +185,7 @@ async def monitor_jobs(self):
185185
prev_job_info = await db_manager.get_job(job_id)
186186
if prev_job_info and prev_job_info.status.lower() != status.lower():
187187
logger.info(
188-
f"Job {job_id} status changed from {prev_job_info.status} to {status}"
188+
f"Job {job_id} status changed from {prev_job_info.status.value} to {status}"
189189
)
190190
# Notify if job in queue
191191
if (
@@ -194,7 +194,7 @@ async def monitor_jobs(self):
194194
and job_id not in self.notified_queued_jobs
195195
):
196196
await send_slack_notification(
197-
f"Job `{TrainingJobStatus.map_status(status)}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Queue Position: `{queue_positions.get(job_id, 'N/A')}`"
197+
f"Job `{TrainingJobStatus.map_status(status).value}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Queue Position: `{queue_positions.get(job_id, 'N/A')}`"
198198
)
199199
self.notified_queued_jobs.add(job_id)
200200
# Notify on start
@@ -204,12 +204,12 @@ async def monitor_jobs(self):
204204
!= KubeflowStatusEnum.running.lower()
205205
):
206206
await send_slack_notification(
207-
f"Job `{TrainingJobStatus.map_status(status)}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Started: `{prev_job_info.created_at}`"
207+
f"Job `{TrainingJobStatus.map_status(status).value}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Started: `{prev_job_info.created_at}`"
208208
)
209209
# Notify on stop
210210
elif status in TrainingJobStatus.stopped_states:
211211
await send_slack_notification(
212-
f"Job `{TrainingJobStatus.map_status(status)}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Started: `{prev_job_info.created_at}` | Ended: `{job.status.completion_time}`"
212+
f"Job `{TrainingJobStatus.map_status(status).value}`: `{prev_job_info.job_name} ({job_id})` by `{prev_job_info.user_id}`\nModel: `{prev_job_info.model_name}` | Started: `{prev_job_info.created_at}` | Ended: `{job.status.completion_time}`"
213213
)
214214
self.notified_queued_jobs.discard(job_id)
215215

0 commit comments

Comments
 (0)