fix(notification): preserve watchdog on reschedule failure

Author: adityathebe

notification/job.go

@@ -558,18 +558,32 @@ func SyncWatchdogJob(ctx context.Context, scheduler *cron.Cron, notificationID s
 		if schedule != nil {
 			scheduleChanged = existingJob.Schedule != *schedule
 		}
+	}
 
-		if schedule == nil || scheduleChanged {
+	if schedule == nil {
+		if existingJob != nil {
 			ctx.Debugf("deleting existing watchdog job for %s", notificationID)
 			existingJob.Unschedule()
 			watchdogJobs.Delete(notificationID)
 		}
+		return nil
 	}
 
-	if schedule != nil && (scheduleChanged || existingJob == nil) {
+	if existingJob == nil {
 		return scheduleWatchdogJob(ctx, scheduler, notificationID, *schedule)
 	}
 
+	if !scheduleChanged {
+		return nil
+	}
+
+	if err := scheduleWatchdogJob(ctx, scheduler, notificationID, *schedule); err != nil {
+		return err
+	}
+
+	ctx.Debugf("deleting existing watchdog job for %s after replacement", notificationID)
+	existingJob.Unschedule()
+
 	return nil
 }
 

notification/watchdog_test.go

@@ -77,6 +77,22 @@ var _ = ginkgo.Describe("Notification watchdog", func() {
 		Expect(scheduler.Entries()).To(BeEmpty())
 	})
 
+	ginkgo.It("should keep the existing watchdog when schedule update fails", func() {
+		scheduler := cron.New()
+		ginkgo.DeferCleanup(func() {
+			scheduler.Stop()
+		})
+
+		notificationID := uuid.New().String()
+		schedule := "*/1 * * * *"
+		Expect(notification.SyncWatchdogJob(DefaultContext, scheduler, notificationID, &schedule)).To(BeNil())
+		Expect(scheduler.Entries()).To(HaveLen(1))
+
+		invalidSchedule := "not-a-cron-expression"
+		Expect(notification.SyncWatchdogJob(DefaultContext, scheduler, notificationID, &invalidSchedule)).To(HaveOccurred())
+		Expect(scheduler.Entries()).To(HaveLen(1))
+	})
+
 	ginkgo.It("should resolve watchdog statistics from the event id", func() {
 		notif := createWatchdogNotification()