support recreating k8s jobs when configmap/secret changed (#808)

* support recreating k8s jobs when configmap/secret changed

Co-authored-by: Mohamed Sekour <mohamed.sekour@exfo.com>

* add unit tests

* fix tests

Author: mohsek

README.md

@@ -50,13 +50,13 @@ spec:
     metadata:
 ```
 
-This will discover deploymentconfigs/deployments/daemonsets/statefulset/rollouts automatically where `foo-configmap` or `foo-secret` is being used either via environment variable or from volume mount. And it will perform rolling upgrade on related pods when `foo-configmap` or `foo-secret`are updated.
+This will discover deploymentconfigs/deployments/daemonsets/statefulset/rollouts/cronjobs/jobs automatically where `foo-configmap` or `foo-secret` is being used either via environment variable or from volume mount. And it will perform rolling upgrade on related pods when `foo-configmap` or `foo-secret`are updated.
 
 You can filter it by the type of monitored resource and use typed versions of `auto` annotation. If you want to discover changes only in mounted `Secret`s and ignore changes in `ConfigMap`s, add `secret.reloader.stakater.com/auto` annotation instead. Analogously, you can use `configmap.reloader.stakater.com/auto` annotation to look for changes in mounted `ConfigMap`, changes in any of mounted `Secret`s will not trigger a rolling upgrade on related pods.
 
 You can also restrict this discovery to only `ConfigMap` or `Secret` objects that
 are tagged with a special annotation. To take advantage of that, annotate
-your deploymentconfigs/deployments/daemonsets/statefulset/rollouts like this:
+your deploymentconfigs/deployments/daemonsets/statefulset/rollouts/cronjobs/jobs like this:
 
 ```yaml
 kind: Deployment
@@ -91,7 +91,7 @@ not.
 
 Similarly, `reloader.stakater.com/auto` and its typed version (`secret.reloader.stakater.com/auto` or `configmap.reloader.stakater.com/auto`) do not work together. If you have both annotations in your deployment, then only one of them needs to be true to trigger the restart. For example, having both `reloader.stakater.com/auto: "true"` and `secret.reloader.stakater.com/auto: "false"` or both `reloader.stakater.com/auto: "false"` and `secret.reloader.stakater.com/auto: "true"` will restart upon a change in a secret it uses.
 
-We can also specify a specific configmap or secret which would trigger rolling upgrade only upon change in our specified configmap or secret, this way, it will not trigger rolling upgrade upon changes in all configmaps or secrets used in a `deploymentconfig`, `deployment`, `daemonset`, `statefulset` or `rollout`.
+We can also specify a specific configmap or secret which would trigger rolling upgrade only upon change in our specified configmap or secret, this way, it will not trigger rolling upgrade upon changes in all configmaps or secrets used in a `deploymentconfig`, `deployment`, `daemonset`, `statefulset`, `rollout`, `cronJob` or `job`.
 To do this either set the auto annotation to `"false"` (`reloader.stakater.com/auto: "false"`) or remove it altogether, and use annotations for [Configmap](.#Configmap) or [Secret](.#Secret).
 
 It's also possible to enable auto reloading for all resources, by setting the `--auto-reload-all` flag.
@@ -158,7 +158,7 @@ spec:
 
 - Reloader also supports [sealed-secrets](https://github.com/bitnami-labs/sealed-secrets). [Here](docs/Reloader-with-Sealed-Secrets.md) are the steps to use sealed-secrets with Reloader.
 - For [`rollouts`](https://github.com/argoproj/argo-rollouts/) Reloader simply triggers a change is up to you how you configure the `rollout` strategy.
-- `reloader.stakater.com/auto: "true"` will only reload the pod, if the configmap or secret is used (as a volume mount or as an env) in `DeploymentConfigs/Deployment/Daemonsets/Statefulsets`
+- `reloader.stakater.com/auto: "true"` will only reload the pod, if the configmap or secret is used (as a volume mount or as an env) in `DeploymentConfigs/Deployment/Daemonsets/Statefulsets/CronJobs/Jobs`
 - `secret.reloader.stakater.com/reload` or `configmap.reloader.stakater.com/reload` annotation will reload the pod upon changes in specified configmap or secret, irrespective of the usage of configmap or secret.
 - you may override the auto annotation with the `--auto-annotation` flag
 - you may override the secret typed auto annotation with the `--secret-auto-annotation` flag

deployments/kubernetes/chart/reloader/templates/clusterrole.yaml

@@ -89,6 +89,9 @@ rules:
       - jobs
     verbs:
       - create
+      - delete
+      - list
+      - get
 {{- if .Values.reloader.enableHA }}
   - apiGroups:
       - "coordination.k8s.io"

deployments/kubernetes/chart/reloader/templates/role.yaml

@@ -80,6 +80,9 @@ rules:
       - jobs
     verbs:
       - create
+      - delete
+      - list
+      - get
 {{- if .Values.reloader.enableHA }}
   - apiGroups:
       - "coordination.k8s.io"

deployments/kubernetes/manifests/clusterrole.yaml

@@ -48,6 +48,9 @@ rules:
       - jobs
     verbs:
       - create
+      - delete
+      - list
+      - get
   - apiGroups:
       - ""
     resources:

deployments/kubernetes/reloader.yaml

@@ -52,6 +52,9 @@ rules:
   - jobs
   verbs:
   - create
+  - delete
+  - list
+  - get
 - apiGroups:
   - ""
   resources:

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
+	github.com/stretchr/testify v1.9.0
 	k8s.io/api v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.1
@@ -47,6 +48,7 @@ require (
 	github.com/moul/http2curl v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

go.sum

@@ -178,8 +178,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
-github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
 github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

internal/pkg/callbacks/rolling_upgrade.go

@@ -92,6 +92,26 @@ func GetCronJobItems(clients kube.Clients, namespace string) []runtime.Object {
 	return items
 }
 
+// GetJobItems returns the jobs in given namespace
+func GetJobItems(clients kube.Clients, namespace string) []runtime.Object {
+	jobs, err := clients.KubernetesClient.BatchV1().Jobs(namespace).List(context.TODO(), meta_v1.ListOptions{})
+	if err != nil {
+		logrus.Errorf("Failed to list jobs %v", err)
+	}
+
+	items := make([]runtime.Object, len(jobs.Items))
+	// Ensure we always have pod annotations to add to
+	for i, v := range jobs.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			annotations := make(map[string]string)
+			jobs.Items[i].Spec.Template.ObjectMeta.Annotations = annotations
+		}
+		items[i] = &jobs.Items[i]
+	}
+
+	return items
+}
+
 // GetDaemonSetItems returns the daemonSets in given namespace
 func GetDaemonSetItems(clients kube.Clients, namespace string) []runtime.Object {
 	daemonSets, err := clients.KubernetesClient.AppsV1().DaemonSets(namespace).List(context.TODO(), meta_v1.ListOptions{})
@@ -178,6 +198,11 @@ func GetCronJobAnnotations(item runtime.Object) map[string]string {
 	return item.(*batchv1.CronJob).ObjectMeta.Annotations
 }
 
+// GetJobAnnotations returns the annotations of given job
+func GetJobAnnotations(item runtime.Object) map[string]string {
+	return item.(*batchv1.Job).ObjectMeta.Annotations
+}
+
 // GetDaemonSetAnnotations returns the annotations of given daemonSet
 func GetDaemonSetAnnotations(item runtime.Object) map[string]string {
 	return item.(*appsv1.DaemonSet).ObjectMeta.Annotations
@@ -208,6 +233,11 @@ func GetCronJobPodAnnotations(item runtime.Object) map[string]string {
 	return item.(*batchv1.CronJob).Spec.JobTemplate.Spec.Template.ObjectMeta.Annotations
 }
 
+// GetJobPodAnnotations returns the pod's annotations of given job
+func GetJobPodAnnotations(item runtime.Object) map[string]string {
+	return item.(*batchv1.Job).Spec.Template.ObjectMeta.Annotations
+}
+
 // GetDaemonSetPodAnnotations returns the pod's annotations of given daemonSet
 func GetDaemonSetPodAnnotations(item runtime.Object) map[string]string {
 	return item.(*appsv1.DaemonSet).Spec.Template.ObjectMeta.Annotations
@@ -238,6 +268,11 @@ func GetCronJobContainers(item runtime.Object) []v1.Container {
 	return item.(*batchv1.CronJob).Spec.JobTemplate.Spec.Template.Spec.Containers
 }
 
+// GetJobContainers returns the containers of given job
+func GetJobContainers(item runtime.Object) []v1.Container {
+	return item.(*batchv1.Job).Spec.Template.Spec.Containers
+}
+
 // GetDaemonSetContainers returns the containers of given daemonSet
 func GetDaemonSetContainers(item runtime.Object) []v1.Container {
 	return item.(*appsv1.DaemonSet).Spec.Template.Spec.Containers
@@ -268,6 +303,11 @@ func GetCronJobInitContainers(item runtime.Object) []v1.Container {
 	return item.(*batchv1.CronJob).Spec.JobTemplate.Spec.Template.Spec.InitContainers
 }
 
+// GetJobInitContainers returns the containers of given job
+func GetJobInitContainers(item runtime.Object) []v1.Container {
+	return item.(*batchv1.Job).Spec.Template.Spec.InitContainers
+}
+
 // GetDaemonSetInitContainers returns the containers of given daemonSet
 func GetDaemonSetInitContainers(item runtime.Object) []v1.Container {
 	return item.(*appsv1.DaemonSet).Spec.Template.Spec.InitContainers
@@ -307,6 +347,38 @@ func CreateJobFromCronjob(clients kube.Clients, namespace string, resource runti
 	return err
 }
 
+// ReCreateJobFromjob performs rolling upgrade on job
+func ReCreateJobFromjob(clients kube.Clients, namespace string, resource runtime.Object) error {
+	oldJob := resource.(*batchv1.Job)
+	job := oldJob.DeepCopy()
+
+	// Delete the old job
+	policy := meta_v1.DeletePropagationBackground
+	err := clients.KubernetesClient.BatchV1().Jobs(namespace).Delete(context.TODO(), job.Name, meta_v1.DeleteOptions{PropagationPolicy: &policy})
+	if err != nil {
+		return err
+	}
+
+	// Remove fields that should not be specified when creating a new Job
+	job.ObjectMeta.ResourceVersion = ""
+	job.ObjectMeta.UID = ""
+	job.ObjectMeta.CreationTimestamp = meta_v1.Time{}
+	job.Status = batchv1.JobStatus{}
+
+	// Remove problematic labels
+	delete(job.Spec.Template.Labels, "controller-uid")
+	delete(job.Spec.Template.Labels, batchv1.ControllerUidLabel)
+	delete(job.Spec.Template.Labels, batchv1.JobNameLabel)
+	delete(job.Spec.Template.Labels, "job-name")
+
+	// Remove the selector to allow it to be auto-generated
+	job.Spec.Selector = nil
+
+	// Create the new job with same spec
+	_, err = clients.KubernetesClient.BatchV1().Jobs(namespace).Create(context.TODO(), job, meta_v1.CreateOptions{FieldManager: "Reloader"})
+	return err
+}
+
 // UpdateDaemonSet performs rolling upgrade on daemonSet
 func UpdateDaemonSet(clients kube.Clients, namespace string, resource runtime.Object) error {
 	daemonSet := resource.(*appsv1.DaemonSet)
@@ -352,6 +424,11 @@ func GetCronJobVolumes(item runtime.Object) []v1.Volume {
 	return item.(*batchv1.CronJob).Spec.JobTemplate.Spec.Template.Spec.Volumes
 }
 
+// GetJobVolumes returns the Volumes of given job
+func GetJobVolumes(item runtime.Object) []v1.Volume {
+	return item.(*batchv1.Job).Spec.Template.Spec.Volumes
+}
+
 // GetDaemonSetVolumes returns the Volumes of given daemonSet
 func GetDaemonSetVolumes(item runtime.Object) []v1.Volume {
 	return item.(*appsv1.DaemonSet).Spec.Template.Spec.Volumes