K8SPSMDB-1608 remove stale external-dns annotations when expose.externalDNS changes

e2e-tests/external-dns/conf/external-dns.yml

@@ -0,0 +1,51 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: external-dns
+spec:
+  #platform: openshift
+  image:
+  unsafeFlags:
+    tls: true
+  tls:
+    mode: disabled
+  backup:
+    enabled: false
+  sharding:
+    enabled: true
+    configsvrReplSet:
+      size: 3
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 1Gi
+      expose:
+        enabled: false
+    mongos:
+      size: 3
+      expose:
+        type: ClusterIP
+        externalDNS:
+          prefix: mongos-p
+          domain: mongo.example.com
+          ttl: 120
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    expose:
+      enabled: true
+      type: ClusterIP
+      externalDNS:
+        prefix: prod
+        domain: mongo.example.com
+        ttl: 300
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+  secrets:
+    users: some-users

e2e-tests/external-dns/run

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+set_debug
+
+ANNOTATION_HOSTNAME='external-dns.alpha.kubernetes.io/hostname'
+ANNOTATION_TTL='external-dns.alpha.kubernetes.io/ttl'
+ANNOTATION_MANAGED='percona.com/external-dns-managed'
+
+get_annotation() {
+	local svc=$1
+	local key=$2
+
+	kubectl_bin get service/$svc -o json | jq -r ".metadata.annotations[\"$key\"] // \"\""
+}
+
+wait_annotation() {
+	local svc=$1
+	local key=$2
+	local expected=$3
+
+	echo -n "waiting for annotation $key=\"$expected\" on service/$svc"
+	local timeout=0
+	until [[ $(get_annotation $svc $key) == "$expected" ]]; do
+		sleep 2
+		timeout=$((timeout + 2))
+		echo -n '.'
+		if [[ ${timeout} -gt 240 ]]; then
+			echo
+			echo "Waiting timeout has been reached. Annotation $key on service/$svc is \"$(get_annotation $svc $key)\", expected \"$expected\". Exiting..."
+			exit 1
+		fi
+	done
+	echo ".OK"
+}
+
+check_annotation_stays() {
+	local svc=$1
+	local key=$2
+	local expected=$3
+
+	sleep 30 # a few reconcile loops
+	local actual=$(get_annotation $svc $key)
+	if [[ $actual != "$expected" ]]; then
+		echo "Annotation $key on service/$svc is \"$actual\", expected it to stay \"$expected\". Exiting..."
+		exit 1
+	fi
+	echo "annotation $key=\"$expected\" on service/$svc preserved.OK"
+}
+
+main() {
+	create_infra $namespace
+
+	desc 'create secrets and start client'
+	kubectl_bin apply \
+		-f $conf_dir/client.yml \
+		-f $conf_dir/secrets.yml
+
+	desc 'create sharded PSMDB cluster with expose.externalDNS on rs0 and mongos'
+	local cluster="external-dns"
+	apply_cluster $test_dir/conf/$cluster.yml
+	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-cfg 3 "false"
+	wait_for_running $cluster-mongos 3
+
+	desc 'check external-dns annotations on replset per-pod services'
+	for i in 0 1 2; do
+		wait_annotation $cluster-rs0-$i $ANNOTATION_HOSTNAME "prod-rs0-$i.mongo.example.com"
+		wait_annotation $cluster-rs0-$i $ANNOTATION_TTL "300"
+		wait_annotation $cluster-rs0-$i $ANNOTATION_MANAGED "true"
+	done
+
+	desc 'check headless replset service has no external-dns annotations'
+	if [[ -n $(get_annotation $cluster-rs0 $ANNOTATION_HOSTNAME) ]]; then
+		echo "headless service $cluster-rs0 must not have $ANNOTATION_HOSTNAME annotation. Exiting..."
+		exit 1
+	fi
+
+	desc 'check single mongos service gets hostname without pod index'
+	wait_annotation $cluster-mongos $ANNOTATION_HOSTNAME "mongos-p-mongos.mongo.example.com"
+	wait_annotation $cluster-mongos $ANNOTATION_TTL "120"
+	wait_annotation $cluster-mongos $ANNOTATION_MANAGED "true"
+
+	desc 'remove ttl from rs0 externalDNS: stale ttl annotation must be removed'
+	kubectl_bin patch psmdb $cluster --type=json --patch '[
+		{
+			"op": "replace",
+			"path": "/spec/replsets/0/expose/externalDNS",
+			"value": {"prefix": "prod", "domain": "mongo.example.com"}
+		}]'
+	for i in 0 1 2; do
+		wait_annotation $cluster-rs0-$i $ANNOTATION_TTL ""
+		wait_annotation $cluster-rs0-$i $ANNOTATION_HOSTNAME "prod-rs0-$i.mongo.example.com"
+	done
+
+	desc 'remove externalDNS from rs0: all managed annotations must be removed'
+	kubectl_bin patch psmdb $cluster --type=json --patch '[
+		{
+			"op": "remove",
+			"path": "/spec/replsets/0/expose/externalDNS"
+		}]'
+	for i in 0 1 2; do
+		wait_annotation $cluster-rs0-$i $ANNOTATION_HOSTNAME ""
+		wait_annotation $cluster-rs0-$i $ANNOTATION_MANAGED ""
+	done
+
+	desc 'manually added external-dns annotations (no marker) must be preserved'
+	kubectl_bin annotate service/$cluster-rs0-0 "$ANNOTATION_HOSTNAME=manual.example.com" --overwrite
+	check_annotation_stays $cluster-rs0-0 $ANNOTATION_HOSTNAME "manual.example.com"
+
+	desc 'enable servicePerPod: per-pod mongos services get indexed hostnames'
+	kubectl_bin patch psmdb $cluster --type=merge -p '{"spec":{"sharding":{"mongos":{"expose":{"servicePerPod":true}}}}}'
+	wait_for_running $cluster-mongos 3
+	for i in 0 1 2; do
+		wait_annotation $cluster-mongos-$i $ANNOTATION_HOSTNAME "mongos-p-mongos-$i.mongo.example.com"
+	done
+
+	desc "delete PSMDB cluster $cluster"
+	kubectl_bin delete \
+		-f $test_dir/conf/$cluster.yml
+
+	destroy $namespace
+
+	desc 'test passed'
+}
+
+main

e2e-tests/run-pr.csv

@@ -52,6 +52,7 @@ demand-backup-snapshot
 demand-backup-snapshot-vault
 disabled-auth
 expose-sharded
+external-dns
 finalizer
 ignore-labels-annotations
 init-deploy

e2e-tests/run-release.csv

@@ -53,6 +53,7 @@ demand-backup-snapshot
 demand-backup-snapshot-vault
 disabled-auth
 expose-sharded
+external-dns
 finalizer
 ignore-labels-annotations
 init-deploy

pkg/controller/perconaservermongodb/psmdb_controller.go

@@ -1817,14 +1817,31 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateSvc(ctx context.Context, c
 	}
 
 	if saveOldMeta {
-		svc.SetAnnotations(util.MapMerge(oldSvc.GetAnnotations(), svc.GetAnnotations()))
+		oldAnnotations := oldSvc.GetAnnotations()
+		removeStaleExternalDNSAnnotations(oldAnnotations, svc.GetAnnotations())
+		svc.SetAnnotations(util.MapMerge(oldAnnotations, svc.GetAnnotations()))
 		svc.SetLabels(util.MapMerge(oldSvc.GetLabels(), svc.GetLabels()))
 	}
 	setIgnoredAnnotations(cr, svc, oldSvc)
 	setIgnoredLabels(cr, svc, oldSvc)
 	return r.createOrUpdate(ctx, svc)
 }
 
+func removeStaleExternalDNSAnnotations(oldAnnotations, desired map[string]string) {
+	if oldAnnotations[naming.AnnotationExternalDNSManaged] == "" && desired[naming.AnnotationExternalDNSManaged] == "" {
+		return
+	}
+	for _, k := range []string{
+		naming.AnnotationExternalDNSHostname,
+		naming.AnnotationExternalDNSTTL,
+		naming.AnnotationExternalDNSManaged,
+	} {
+		if _, ok := desired[k]; !ok {
+			delete(oldAnnotations, k)
+		}
+	}
+}
+
 func setIgnoredAnnotations(cr *api.PerconaServerMongoDB, obj, oldObject client.Object) {
 	oldAnnotations := oldObject.GetAnnotations()
 	if len(oldAnnotations) == 0 {

pkg/controller/perconaservermongodb/service_test.go

@@ -289,3 +289,68 @@ func yamlCompare(t *testing.T, ns string, filename string, compare any) {
 		t.Fatalf("yaml resources doesn't match:\nexpected:\n%s\ngot:\n%s", string(expected), string(data))
 	}
 }
+
+func TestRemoveStaleExternalDNSAnnotations(t *testing.T) {
+	tests := map[string]struct {
+		old      map[string]string
+		desired  map[string]string
+		expected map[string]string
+	}{
+		"externalDNS removed from CR: managed annotations dropped": {
+			old: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "prod-rs0-0.mongo.example.com",
+				"external-dns.alpha.kubernetes.io/ttl":      "300",
+				"percona.com/external-dns-managed":          "true",
+				"cloud.google.com/neg":                      `{"ingress":true}`,
+			},
+			desired: map[string]string{},
+			expected: map[string]string{
+				"cloud.google.com/neg": `{"ingress":true}`,
+			},
+		},
+		"ttl removed from CR: stale ttl dropped, hostname kept": {
+			old: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "prod-rs0-0.mongo.example.com",
+				"external-dns.alpha.kubernetes.io/ttl":      "300",
+				"percona.com/external-dns-managed":          "true",
+			},
+			desired: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "prod-rs0-0.mongo.example.com",
+				"percona.com/external-dns-managed":          "true",
+			},
+			expected: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "prod-rs0-0.mongo.example.com",
+				"percona.com/external-dns-managed":          "true",
+			},
+		},
+		"manually added external-dns annotations (no marker) are preserved": {
+			old: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "manual.example.com",
+				"external-dns.alpha.kubernetes.io/ttl":      "60",
+			},
+			desired: map[string]string{},
+			expected: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "manual.example.com",
+				"external-dns.alpha.kubernetes.io/ttl":      "60",
+			},
+		},
+		"externalDNS enabled over manual annotations: operator takes ownership": {
+			old: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "manual.example.com",
+			},
+			desired: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "prod-rs0-0.mongo.example.com",
+				"percona.com/external-dns-managed":          "true",
+			},
+			expected: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname": "manual.example.com",
+			},
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			removeStaleExternalDNSAnnotations(tt.old, tt.desired)
+			assert.Equal(t, tt.expected, tt.old)
+		})
+	}
+}

pkg/naming/annotations.go

@@ -8,3 +8,9 @@ const (
 )
 
 const AnnotationKubectlRestartedAt = "kubectl.kubernetes.io/restartedAt"
+
+const (
+	AnnotationExternalDNSHostname = "external-dns.alpha.kubernetes.io/hostname"
+	AnnotationExternalDNSTTL      = "external-dns.alpha.kubernetes.io/ttl"
+	AnnotationExternalDNSManaged  = perconaPrefix + "external-dns-managed"
+)

pkg/psmdb/mongos.go

@@ -464,10 +464,11 @@ func MongosService(cr *api.PerconaServerMongoDB, name string) corev1.Service {
 			} else {
 				hostname = BuildDNSHostnameWithoutIndex(dns, "mongos")
 			}
-			annotations["external-dns.alpha.kubernetes.io/hostname"] = hostname
+			annotations[naming.AnnotationExternalDNSHostname] = hostname
 			if dns.TTL > 0 {
-				annotations["external-dns.alpha.kubernetes.io/ttl"] = strconv.Itoa(dns.TTL)
+				annotations[naming.AnnotationExternalDNSTTL] = strconv.Itoa(dns.TTL)
 			}
+			annotations[naming.AnnotationExternalDNSManaged] = "true"
 		}
 
 		svc.Annotations = annotations