Add missing timer handling

Author: tom1299

internal/pkg/handler/pause_deployment.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -14,6 +15,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// Keeps track of currently active timers for paused deployments
+var (
+	activeTimers = make(map[string]*time.Timer)
+	timerLock    = sync.Mutex{}
+)
+
+// Returns unique key for the activeTimers map
+func getTimerKey(namespace, deploymentName string) string {
+	return fmt.Sprintf("%s/%s", namespace, deploymentName)
+}
+
 // IsPaused checks if a deployment is currently paused
 func IsPaused(deployment *app.Deployment) bool {
 	return deployment.Spec.Paused
@@ -78,19 +90,75 @@ func PauseDeployment(deployment *app.Deployment, clients kube.Clients, deploymen
 			deployment.Annotations = make(map[string]string)
 		}
 		deployment.Annotations[options.PauseDeploymentTimeAnnotation] = time.Now().Format(time.RFC3339)
+		deployment.Annotations[options.PauseDeploymentAnnotation] = pauseIntervalValue
 
 		CreateResumeTimer(deployment, clients, deploymentName, namespace, pauseDuration)
+	} else if IsPausedByReloader(deployment) {
+		logrus.Debugf("Deployment '%s' in namespace '%s' is already paused by reloader", deploymentName, namespace)
+
+		// Check if a timer already exists for this deployment
+		timerKey := getTimerKey(namespace, deploymentName)
+		timerLock.Lock()
+		_, timerExists := activeTimers[timerKey]
+		timerLock.Unlock()
+
+		if !timerExists {
+			logrus.Warnf("Timer does not exist for already paused deployment '%s' in namespace '%s', creating new one",
+				deploymentName, namespace)
+			HandleMissingTimer(deployment, pauseDuration, clients, deploymentName, namespace)
+		}
+
 	} else {
-		logrus.Infof("Deployment '%s' in namespace '%s' is already paused", deploymentName, namespace)
+		logrus.Infof("Deployment '%s' in namespace '%s' already paused", deploymentName, namespace)
 	}
 	return nil
 }
 
+func HandleMissingTimer(deployment *app.Deployment, pauseDuration time.Duration, clients kube.Clients, deploymentName, namespace string) {
+	pauseStartTime, err := GetPauseStartTime(deployment)
+	if err != nil {
+		logrus.Errorf("Error parsing pause start time for deployment '%s' in namespace '%s': %v. Resuming deployment immediately",
+			deploymentName, namespace, err)
+		ResumeDeployment(deploymentName, namespace, clients)
+	} else if pauseStartTime != nil {
+		elapsedTime := time.Since(*pauseStartTime)
+		remainingTime := pauseDuration - elapsedTime
+
+		if remainingTime <= 0 {
+			logrus.Infof("Pause period for deployment '%s' in namespace '%s' has expired. Resuming immediately",
+				deploymentName, namespace)
+			ResumeDeployment(deploymentName, namespace, clients)
+		} else {
+			logrus.Infof("Creating missing timer for already paused deployment '%s' in namespace '%s' with remaining time %s",
+				deploymentName, namespace, remainingTime)
+			CreateResumeTimer(deployment, clients, deploymentName, namespace, remainingTime)
+		}
+	}
+}
+
 // CreateResumeTimer creates a timer to resume the deployment after the specified duration
 func CreateResumeTimer(deployment *app.Deployment, clients kube.Clients, deploymentName, namespace string, pauseDuration time.Duration) {
-	time.AfterFunc(pauseDuration, func() {
+	timerKey := getTimerKey(namespace, deploymentName)
+
+	// Check if there's an existing timer for this deployment
+	timerLock.Lock()
+	if _, exists := activeTimers[timerKey]; exists {
+		logrus.Debugf("Timer already exists for deployment '%s' in namespace '%s', Skipping creation",
+			deploymentName, namespace)
+		return
+	}
+
+	// Create and store the new timer
+	timer := time.AfterFunc(pauseDuration, func() {
 		ResumeDeployment(deploymentName, namespace, clients)
 	})
+
+	// Add the new timer to the map
+	activeTimers[timerKey] = timer
+	timerLock.Unlock()
+
+	logrus.Debugf("Created pause timer for deployment '%s' in namespace '%s' with duration %s",
+		deploymentName, namespace, pauseDuration)
 }
 
 // ResumeDeployment resumes a deployment that has been paused by reloader
@@ -106,6 +174,16 @@ func ResumeDeployment(deploymentName, namespace string, clients kube.Clients) {
 		return
 	}
 
+	// Remove the timer itself
+	timerKey := getTimerKey(namespace, deploymentName)
+	timerLock.Lock()
+	if timer, exists := activeTimers[timerKey]; exists {
+		timer.Stop()
+		delete(activeTimers, timerKey)
+		logrus.Debugf("Removed pause timer for deployment '%s' in namespace '%s'", deploymentName, namespace)
+	}
+	timerLock.Unlock()
+
 	deployment.Spec.Paused = false
 	delete(deployment.Annotations, options.PauseDeploymentTimeAnnotation)