Skip to content

Commit 8359d87

Browse files
tom1299tom1299
committed
Add pausing deployments during upgrades
1 parent 574129d commit 8359d87

5 files changed

Lines changed: 267 additions & 0 deletions

File tree

‎internal/pkg/handler/pause_deployment.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package handler
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/sirupsen/logrus"
8+
"github.com/stakater/Reloader/internal/pkg/options"
9+
"github.com/stakater/Reloader/pkg/kube"
10+
app "k8s.io/api/apps/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
)
13+
14+
func PauseDeployment(deployment *app.Deployment, clients kube.Clients, deploymentName, namespace, pauseIntervalValue string) error {
15+
pauseDuration, err := ParsePauseDuration(pauseIntervalValue)
16+
if err != nil {
17+
return err
18+
}
19+
20+
if !deployment.Spec.Paused {
21+
deployment.Spec.Paused = true
22+
logrus.Infof("Pausing Deployment '%s' in namespace '%s' for %s seconds", deploymentName, namespace, pauseDuration)
23+
24+
if deployment.Annotations == nil {
25+
deployment.Annotations = make(map[string]string)
26+
}
27+
deployment.Annotations[options.PauseDeploymentTimeAnnotation] = time.Now().Format(time.RFC3339)
28+
29+
CreateResumeTimer(deployment, clients, deploymentName, namespace, pauseDuration)
30+
} else {
31+
logrus.Infof("Deployment '%s' in namespace '%s' is already paused", deploymentName, namespace)
32+
}
33+
return nil
34+
}
35+
36+
func CreateResumeTimer(deployment *app.Deployment, clients kube.Clients, deploymentName, namespace string, pauseDuration time.Duration) {
37+
time.AfterFunc(pauseDuration, func() {
38+
ResumeDeployment(deploymentName, namespace, clients)
39+
})
40+
}
41+
42+
func ResumeDeployment(deploymentName, namespace string, clients kube.Clients) {
43+
deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{})
44+
if err != nil {
45+
logrus.Errorf("Failed to get deployment '%s' in namespace '%s': %v", deploymentName, namespace, err)
46+
return
47+
}
48+
49+
if !deployment.Spec.Paused {
50+
logrus.Infof("Deployment '%s' in namespace '%s' not paused. Skipping resume", deploymentName, namespace)
51+
return
52+
}
53+
54+
pausedAtAnnotationValue := deployment.Annotations[options.PauseDeploymentTimeAnnotation]
55+
if pausedAtAnnotationValue == "" {
56+
logrus.Infof("Deployment '%s' in namespace '%s' was not paused by Reloader. Skipping resume", deploymentName, namespace)
57+
return
58+
}
59+
60+
deployment.Spec.Paused = false
61+
delete(deployment.Annotations, options.PauseDeploymentTimeAnnotation)
62+
63+
_, err = clients.KubernetesClient.AppsV1().Deployments(namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
64+
if err != nil {
65+
logrus.Errorf("Failed to resume deployment '%s' in namespace '%s': %v", deploymentName, namespace, err)
66+
}
67+
68+
logrus.Infof("Successfully resumed deployment '%s' in namespace '%s'", deploymentName, namespace)
69+
}
70+
71+
func ParsePauseDuration(pauseIntervalValue string) (time.Duration, error) {
72+
pauseDuration, err := time.ParseDuration(pauseIntervalValue)
73+
if err != nil {
74+
logrus.Warnf("Failed to parse pause interval value '%s': %v", pauseIntervalValue, err)
75+
return 0, err
76+
}
77+
return pauseDuration, nil
78+
}

‎internal/pkg/handler/upgrade.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/stakater/Reloader/internal/pkg/options"
2222
"github.com/stakater/Reloader/internal/pkg/util"
2323
"github.com/stakater/Reloader/pkg/kube"
24+
app "k8s.io/api/apps/v1"
2425
v1 "k8s.io/api/core/v1"
2526
"k8s.io/apimachinery/pkg/api/meta"
2627
"k8s.io/apimachinery/pkg/runtime"
@@ -219,6 +220,7 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba
219220
typedAutoAnnotationEnabledValue, foundTypedAuto := annotations[config.TypedAutoAnnotation]
220221
excludeConfigmapAnnotationValue, foundExcludeConfigmap := annotations[options.ConfigmapExcludeReloaderAnnotation]
221222
excludeSecretAnnotationValue, foundExcludeSecret := annotations[options.SecretExcludeReloaderAnnotation]
223+
pauseInterval, foundPauseInterval := annotations[options.PauseDeploymentAnnotation]
222224

223225
if !found && !foundAuto && !foundTypedAuto && !foundSearchAnn {
224226
annotations = upgradeFuncs.PodAnnotationsFunc(i)
@@ -274,6 +276,25 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba
274276
}
275277

276278
if result == constants.Updated {
279+
280+
if foundPauseInterval {
281+
282+
accessor, err := meta.Accessor(i)
283+
if err != nil {
284+
return err
285+
}
286+
287+
itemName := accessor.GetName()
288+
itemNamespace := accessor.GetNamespace()
289+
290+
deployment, ok := i.(*app.Deployment)
291+
if !ok {
292+
logrus.Warnf("Annotation '%s' only applicable for deployments", options.PauseDeploymentAnnotation)
293+
} else {
294+
PauseDeployment(deployment, clients, itemName, itemNamespace, pauseInterval)
295+
}
296+
}
297+
277298
accessor, err := meta.Accessor(i)
278299
if err != nil {
279300
return err

‎internal/pkg/handler/upgrade_test.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/stakater/Reloader/internal/pkg/testutil"
1818
"github.com/stakater/Reloader/internal/pkg/util"
1919
"github.com/stakater/Reloader/pkg/kube"
20+
appsv1 "k8s.io/api/apps/v1"
2021
"k8s.io/apimachinery/pkg/api/meta"
2122
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2223
"k8s.io/apimachinery/pkg/runtime"
@@ -49,6 +50,7 @@ var (
4950
arsConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5)
5051
arsSecretWithExcludeSecretAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5)
5152
arsConfigmapWithExcludeConfigMapAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5)
53+
arsConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5)
5254

5355
ersNamespace = "test-handler-" + testutil.RandSeq(5)
5456
ersConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5)
@@ -72,6 +74,7 @@ var (
7274
ersConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5)
7375
ersSecretWithSecretExcludeAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5)
7476
ersConfigmapWithConfigMapExcludeAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5)
77+
ersConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5)
7578
)
7679

7780
func TestMain(m *testing.M) {
@@ -200,6 +203,12 @@ func setupArs() {
200203
logrus.Errorf("Error in configmap creation: %v", err)
201204
}
202205

206+
// Creating configmap for testing pausing deployments
207+
_, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment, "www.google.com")
208+
if err != nil {
209+
logrus.Errorf("Error in configmap creation: %v", err)
210+
}
211+
203212
// Creating secret used with secret auto annotation
204213
_, err = testutil.CreateSecret(clients.KubernetesClient, arsNamespace, arsSecretWithExcludeSecretAnnotation, data)
205214
if err != nil {
@@ -421,6 +430,12 @@ func setupArs() {
421430
if err != nil {
422431
logrus.Errorf("Error in Deployment with both annotations: %v", err)
423432
}
433+
434+
// Creating Deployment with pause annotation
435+
_, err = testutil.CreateDeploymentWithAnnotations(clients.KubernetesClient, arsConfigmapWithPausedDeployment, arsNamespace, map[string]string{options.PauseDeploymentAnnotation: "10s"}, false)
436+
if err != nil {
437+
logrus.Errorf("Error in Deployment with configmap creation: %v", err)
438+
}
424439
}
425440

426441
func teardownArs() {
@@ -622,6 +637,12 @@ func teardownArs() {
622637
logrus.Errorf("Error while deleting statefulSet with secret as env var source %v", statefulSetError)
623638
}
624639

640+
// Deleting Deployment with pasuse annotation
641+
deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment)
642+
if deploymentError != nil {
643+
logrus.Errorf("Error while deleting deployment with configmap %v", deploymentError)
644+
}
645+
625646
// Deleting Configmap
626647
err := testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapName)
627648
if err != nil {
@@ -735,6 +756,12 @@ func teardownArs() {
735756
logrus.Errorf("Error while deleting the configmap used with configmap auto annotations: %v", err)
736757
}
737758

759+
// Deleting configmap for testing pausing deployments
760+
err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment)
761+
if err != nil {
762+
logrus.Errorf("Error while deleting the configmap: %v", err)
763+
}
764+
738765
// Deleting namespace
739766
testutil.DeleteNamespace(arsNamespace, clients.KubernetesClient)
740767

@@ -794,6 +821,12 @@ func setupErs() {
794821
logrus.Errorf("Error in configmap creation: %v", err)
795822
}
796823

824+
// Creating configmap for testing pausing deployments
825+
_, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment, "www.google.com")
826+
if err != nil {
827+
logrus.Errorf("Error in configmap creation: %v", err)
828+
}
829+
797830
// Creating secret
798831
_, err = testutil.CreateSecret(clients.KubernetesClient, ersNamespace, ersSecretWithInitEnv, data)
799832
if err != nil {
@@ -970,6 +1003,12 @@ func setupErs() {
9701003
logrus.Errorf("Error in Deployment with configmap and with configmap exclude annotation: %v", err)
9711004
}
9721005

1006+
// Creating Deployment with pause annotation
1007+
_, err = testutil.CreateDeploymentWithAnnotations(clients.KubernetesClient, ersConfigmapWithPausedDeployment, ersNamespace, map[string]string{options.PauseDeploymentAnnotation: "10s"}, false)
1008+
if err != nil {
1009+
logrus.Errorf("Error in Deployment with configmap creation: %v", err)
1010+
}
1011+
9731012
// Creating DaemonSet with configmap
9741013
_, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersConfigmapName, ersNamespace, true)
9751014
if err != nil {
@@ -1254,6 +1293,12 @@ func teardownErs() {
12541293
logrus.Errorf("Error while deleting statefulSet with secret as env var source %v", statefulSetError)
12551294
}
12561295

1296+
// Deleting Deployment for testing pausing deployments
1297+
deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment)
1298+
if deploymentError != nil {
1299+
logrus.Errorf("Error while deleting deployment with configmap %v", deploymentError)
1300+
}
1301+
12571302
// Deleting Configmap
12581303
err := testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapName)
12591304
if err != nil {
@@ -1367,6 +1412,12 @@ func teardownErs() {
13671412
logrus.Errorf("Error while deleting the configmap used with configmap exclude annotation: %v", err)
13681413
}
13691414

1415+
// Deleting ConfigMap for testins pausing deployments
1416+
err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment)
1417+
if err != nil {
1418+
logrus.Errorf("Error while deleting the configmap: %v", err)
1419+
}
1420+
13701421
// Deleting namespace
13711422
testutil.DeleteNamespace(ersNamespace, clients.KubernetesClient)
13721423

@@ -3548,3 +3599,94 @@ func TestFailedRollingUpgradeUsingErs(t *testing.T) {
35483599
t.Errorf("Counter by namespace was not increased")
35493600
}
35503601
}
3602+
3603+
func TestPausingDeploymentUsingErs(t *testing.T) {
3604+
options.ReloadStrategy = constants.EnvVarsReloadStrategy
3605+
testPausingDeployment(t, options.ReloadStrategy, ersConfigmapWithPausedDeployment, ersNamespace)
3606+
}
3607+
3608+
func TestPausingDeploymentUsingArs(t *testing.T) {
3609+
options.ReloadStrategy = constants.AnnotationsReloadStrategy
3610+
testPausingDeployment(t, options.ReloadStrategy, arsConfigmapWithPausedDeployment, arsNamespace)
3611+
}
3612+
3613+
func testPausingDeployment(t *testing.T, reloadStrategy string, testName string, namespace string) {
3614+
options.ReloadStrategy = reloadStrategy
3615+
envVarPostfix := constants.ConfigmapEnvVarPostfix
3616+
3617+
shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, testName, "pause.stakater.com")
3618+
config := getConfigWithAnnotations(envVarPostfix, testName, shaData, options.ConfigmapUpdateOnChangeAnnotation, options.ConfigmapReloaderAutoAnnotation)
3619+
deploymentFuncs := GetDeploymentRollingUpgradeFuncs()
3620+
collectors := getCollectors()
3621+
3622+
_ = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy)
3623+
3624+
if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 {
3625+
t.Errorf("Counter was not increased")
3626+
}
3627+
3628+
if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": namespace})) != 1 {
3629+
t.Errorf("Counter by namespace was not increased")
3630+
}
3631+
3632+
logrus.Infof("Verifying deployment has been paused")
3633+
items := deploymentFuncs.ItemsFunc(clients, config.Namespace)
3634+
deploymentPaused, err := isDeploymentPaused(items, testName)
3635+
if err != nil {
3636+
t.Errorf(err.Error())
3637+
}
3638+
if !deploymentPaused {
3639+
t.Errorf("Deployment has not been paused")
3640+
}
3641+
3642+
shaData = testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, testName, "pause-changed.stakater.com")
3643+
config = getConfigWithAnnotations(envVarPostfix, testName, shaData, options.ConfigmapUpdateOnChangeAnnotation, options.ConfigmapReloaderAutoAnnotation)
3644+
3645+
_ = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy)
3646+
3647+
if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 {
3648+
t.Errorf("Counter was not increased")
3649+
}
3650+
3651+
if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": namespace})) != 2 {
3652+
t.Errorf("Counter by namespace was not increased")
3653+
}
3654+
3655+
logrus.Infof("Verifying deployment is still paused")
3656+
items = deploymentFuncs.ItemsFunc(clients, config.Namespace)
3657+
deploymentPaused, err = isDeploymentPaused(items, testName)
3658+
if err != nil {
3659+
t.Errorf("%s", err.Error())
3660+
}
3661+
if !deploymentPaused {
3662+
t.Errorf("Deployment should still be paused")
3663+
}
3664+
3665+
logrus.Infof("Verifying deployment has been resumed after pause interval")
3666+
time.Sleep(11 * time.Second)
3667+
items = deploymentFuncs.ItemsFunc(clients, config.Namespace)
3668+
deploymentPaused, err = isDeploymentPaused(items, testName)
3669+
if err != nil {
3670+
t.Errorf("%s", err.Error())
3671+
}
3672+
if deploymentPaused {
3673+
t.Errorf("Deployment should have been resumed after pause interval")
3674+
}
3675+
}
3676+
3677+
func isDeploymentPaused(deployments []runtime.Object, deploymentName string) (bool, error) {
3678+
for _, deployment := range deployments {
3679+
accessor, err := meta.Accessor(deployment)
3680+
if err != nil {
3681+
return false, fmt.Errorf("Error getting accessor for item: %v", err)
3682+
}
3683+
if accessor.GetName() == deploymentName {
3684+
deploymentObj, ok := deployment.(*appsv1.Deployment)
3685+
if !ok {
3686+
return false, fmt.Errorf("Failed to cast to Deployment")
3687+
}
3688+
return deploymentObj.Spec.Paused, nil
3689+
}
3690+
}
3691+
return false, nil
3692+
}

‎internal/pkg/options/flags.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ var (
3838
SearchMatchAnnotation = "reloader.stakater.com/match"
3939
// RolloutStrategyAnnotation is an annotation to define rollout update strategy
4040
RolloutStrategyAnnotation = "reloader.stakater.com/rollout-strategy"
41+
// PauseDeploymentAnnotation is an annotation to define the time period to pause a deployment after
42+
// a configmap/secret change has been detected. Valid values are described here: https://pkg.go.dev/time#ParseDuration
43+
// only positive values are allowed
44+
PauseDeploymentAnnotation = "deployment.reloader.stakater.com/pause-period"
45+
// Annotation set by reloader to indicate that the deployment has been paused
46+
PauseDeploymentTimeAnnotation = "deployment.reloader.stakater.com/paused-at"
4147
// LogFormat is the log format to use (json, or empty string for default)
4248
LogFormat = ""
4349
// LogLevel is the log level to use (trace, debug, info, warning, error, fatal and panic)

‎internal/pkg/testutil/kube.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,26 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp
794794
return deployment, err
795795
}
796796

797+
// CreateDeployment creates a deployment in given namespace and returns the Deployment
798+
func CreateDeploymentWithAnnotations(client kubernetes.Interface, deploymentName string, namespace string, additionalAnnotations map[string]string, volumeMount bool) (*appsv1.Deployment, error) {
799+
logrus.Infof("Creating Deployment")
800+
deploymentClient := client.AppsV1().Deployments(namespace)
801+
var deploymentObj *appsv1.Deployment
802+
if volumeMount {
803+
deploymentObj = GetDeployment(namespace, deploymentName)
804+
} else {
805+
deploymentObj = GetDeploymentWithEnvVars(namespace, deploymentName)
806+
}
807+
808+
for annotationKey, annotationValue := range additionalAnnotations {
809+
deploymentObj.Annotations[annotationKey] = annotationValue
810+
}
811+
812+
deployment, err := deploymentClient.Create(context.TODO(), deploymentObj, metav1.CreateOptions{})
813+
time.Sleep(3 * time.Second)
814+
return deployment, err
815+
}
816+
797817
// CreateDeploymentConfig creates a deploymentConfig in given namespace and returns the DeploymentConfig
798818
func CreateDeploymentConfig(client appsclient.Interface, deploymentName string, namespace string, volumeMount bool) (*openshiftv1.DeploymentConfig, error) {
799819
logrus.Infof("Creating DeploymentConfig")

0 commit comments

Comments
 (0)