Resolve concurrency problem causing skipping taint by surfacing error and retry with SQS (#1279)

Co-authored-by: mcornea mcornea@redhat.com

Author: hyeong01

pkg/interruptionevent/draincordon/handler.go

@@ -81,7 +81,18 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error {
 	}
 
 	if drainEvent.PreDrainTask != nil {
-		h.commonHandler.RunPreDrainTask(nodeName, drainEvent)
+		if err := h.commonHandler.RunPreDrainTask(nodeName, drainEvent); err != nil {
+			log.Err(err).Str("nodeName", nodeName).Msg("Pre-drain task failed; aborting to allow SQS retry")
+			h.commonHandler.InterruptionEventStore.CancelInterruptionEvent(drainEvent.EventID)
+
+			// If the node is missing and the user opted for DeleteSqsMsgIfNodeNotFound then delete the SQS message
+			if !nodeFound && h.commonHandler.NthConfig.DeleteSqsMsgIfNodeNotFound && drainEvent.PostDrainTask != nil {
+				h.commonHandler.RunPostDrainTask(nodeName, drainEvent)
+				return nil
+			}
+			
+			return err
+		}
 	}
 
 	podNameList, err := h.commonHandler.Node.FetchPodNameList(nodeName)

pkg/interruptionevent/internal/common/handler.go

@@ -44,7 +44,7 @@ func (h *Handler) GetNodeName(drainEvent *monitor.InterruptionEvent) (string, er
 	return nodeName, nil
 }
 
-func (h *Handler) RunPreDrainTask(nodeName string, drainEvent *monitor.InterruptionEvent) {
+func (h *Handler) RunPreDrainTask(nodeName string, drainEvent *monitor.InterruptionEvent) error {
 	err := drainEvent.PreDrainTask(*drainEvent, h.Node)
 	if err != nil {
 		log.Err(err).Msg("There was a problem executing the pre-drain task")
@@ -53,6 +53,7 @@ func (h *Handler) RunPreDrainTask(nodeName string, drainEvent *monitor.Interrupt
 		h.Recorder.Emit(nodeName, observability.Normal, observability.PreDrainReason, observability.PreDrainMsg)
 	}
 	h.Metrics.NodeActionsInc("pre-drain", nodeName, drainEvent.EventID, err)
+	return err
 }
 
 func (h *Handler) RunCancelDrainTask(nodeName string, drainEvent *monitor.InterruptionEvent) {

pkg/monitor/sqsevent/spot-itn-event.go

@@ -96,7 +96,7 @@ func (m SQSMonitor) spotITNTerminationToInterruptionEvent(event *EventBridgeEven
 		if err != nil {
 			log.Err(err).Msgf("Unable to taint node with taint %s:%s", node.SpotInterruptionTaint, interruptionEvent.EventID)
 		}
-		return nil
+		return err
 	}
 	return &interruptionEvent, nil
 }