Skip to content

Commit 1822be7

Browse files
committed
Add AC finalizer management
Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
1 parent 82c6f3e commit 1822be7

13 files changed

Lines changed: 314 additions & 2 deletions

File tree

api/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

api/go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
9191
replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
9292

9393
replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
94+
95+
replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

api/v1beta1/watcher_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ type WatcherStatus struct {
6161

6262
// DecisionEngineServiceReadyCount defines the number or replicas ready from watcher-decision-engine
6363
DecisionEngineServiceReadyCount int32 `json:"decisionengineServiceReadyCount,omitempty"`
64+
65+
// ApplicationCredentialSecret - the AC secret watcher is currently
66+
// consuming and protecting with the openstack.org/watcher-ac-consumer
67+
// finalizer. Tracked so the controller can remove its finalizer from the
68+
// old secret when the openstack-operator rotates the reference.
69+
ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"`
6470
}
6571

6672
// WatcherDBPurge defines the parameters for the Watcher database purging cron job

config/crd/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
141141
replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
142142

143143
replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
144+
145+
replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20 h1:iyxfh2SDvQrOrsHItYAE3A3+8Ku9UnzWAq9jnLJDLjg=
2+
github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20/go.mod h1:SpO4CL7c5/1HG+61fP6kWhL2+3aqR+5SNatdZueKrz8=
13
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
24
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
35
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -120,8 +122,6 @@ github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyU
120122
github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
121123
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4=
122124
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs=
123-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f h1:28WYAUIef3uion0Pps6doCSSbgZtIcodGzwG6BHhCOw=
124-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f/go.mod h1:4ryvbSYuoN522BIPijnm0wMemPgJVKf7jCv8BNDq46I=
125125
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg=
126126
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs=
127127
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 h1:kUfZlcl+EbUBEWe6EGLXjzlUeYj7xZ21QsPA5jMJlwE=

internal/controller/watcher_controller.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,24 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
478478
return ctrl.Result{}, err
479479
}
480480

481+
// Add consumer finalizer to the new AC secret early, before deployment.
482+
// The old secret's finalizer is removed later (after all services deploy)
483+
// so that rapid rotations don't revoke a credential still in use by pods.
484+
if instance.Spec.Auth.ApplicationCredentialSecret != "" {
485+
if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace,
486+
instance.Spec.Auth.ApplicationCredentialSecret,
487+
"",
488+
watcher.ACConsumerFinalizer); err != nil {
489+
instance.Status.Conditions.Set(condition.FalseCondition(
490+
condition.ServiceConfigReadyCondition,
491+
condition.ErrorReason,
492+
condition.SeverityWarning,
493+
condition.ServiceConfigReadyErrorMessage,
494+
err.Error()))
495+
return ctrl.Result{}, err
496+
}
497+
}
498+
481499
instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
482500
// End of config generation for dbsync
483501

@@ -523,6 +541,27 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
523541
}
524542
// End of Watcher Applier deploy
525543

544+
// Manage the old AC secret's finalizer and status tracking.
545+
// On rotation (old != new), only remove the old secret's finalizer after
546+
// all sub-services are ready with the new credentials. This prevents
547+
// premature revocation during rapid rotations.
548+
isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret
549+
550+
if isRotation {
551+
allServicesReady := instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherAPIReadyCondition) &&
552+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherApplierReadyCondition) &&
553+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherDecisionEngineReadyCondition)
554+
if allServicesReady {
555+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
556+
instance.Status.ApplicationCredentialSecret, watcher.ACConsumerFinalizer); err != nil {
557+
return ctrl.Result{}, err
558+
}
559+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
560+
}
561+
} else {
562+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
563+
}
564+
526565
//
527566
// remove finalizers from unused MariaDBAccount records
528567
// this assumes all database-depedendent deployments are up and
@@ -1347,6 +1386,17 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch
13471386
}
13481387
//
13491388

1389+
// Remove consumer finalizer from AC secrets watcher was consuming.
1390+
for _, secretName := range []string{
1391+
instance.Status.ApplicationCredentialSecret,
1392+
instance.Spec.Auth.ApplicationCredentialSecret,
1393+
} {
1394+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
1395+
secretName, watcher.ACConsumerFinalizer); err != nil {
1396+
return ctrl.Result{}, err
1397+
}
1398+
}
1399+
13501400
controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
13511401
Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name))
13521402
return ctrl.Result{}, nil

internal/watcher/constants.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,7 @@ const (
5050

5151
// scriptVolume is the name of the volume used to ship scripts into pods
5252
scriptVolume = "scripts-volume"
53+
54+
// ACConsumerFinalizer is added to AC secrets that watcher is actively consuming
55+
ACConsumerFinalizer = "openstack.org/watcher-ac-consumer"
5356
)

test/functional/watcher_controller_test.go

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
. "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers"
1818
mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
1919
watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1"
20+
"github.com/openstack-k8s-operators/watcher-operator/internal/watcher"
2021
corev1 "k8s.io/api/core/v1"
2122
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2223
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -1858,6 +1859,188 @@ var _ = Describe("Watcher controller", func() {
18581859
})
18591860
})
18601861

1862+
When("ApplicationCredential consumer finalizer is managed", func() {
1863+
var acSecretName string
1864+
1865+
BeforeEach(func() {
1866+
acSecretName = "ac-watcher-consumer-fnz-secret" //nolint:gosec
1867+
1868+
acSecret := &corev1.Secret{
1869+
ObjectMeta: metav1.ObjectMeta{
1870+
Name: acSecretName,
1871+
Namespace: watcherTest.Instance.Namespace,
1872+
},
1873+
Data: map[string][]byte{
1874+
keystonev1beta1.ACIDSecretKey: []byte("consumer-test-ac-id"),
1875+
keystonev1beta1.ACSecretSecretKey: []byte("consumer-test-ac-secret"), //nolint:gosec
1876+
},
1877+
}
1878+
Expect(k8sClient.Create(ctx, acSecret)).To(Succeed())
1879+
DeferCleanup(k8sClient.Delete, ctx, acSecret)
1880+
1881+
DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret"))
1882+
1883+
memcachedSpec := memcachedv1.MemcachedSpec{
1884+
MemcachedSpecCore: memcachedv1.MemcachedSpecCore{
1885+
Replicas: ptr.To(int32(1)),
1886+
},
1887+
}
1888+
DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.Watcher.Namespace, MemcachedInstance, memcachedSpec))
1889+
infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace)
1890+
1891+
DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace))
1892+
1893+
DeferCleanup(
1894+
k8sClient.Delete, ctx, th.CreateSecret(
1895+
types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "metric-storage-prometheus-endpoint"},
1896+
map[string][]byte{
1897+
"host": []byte("prometheus.example.com"),
1898+
"port": []byte("9090"),
1899+
},
1900+
))
1901+
1902+
DeferCleanup(
1903+
k8sClient.Delete, ctx, th.CreateSecret(
1904+
types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName},
1905+
map[string][]byte{
1906+
"WatcherPassword": []byte("password"),
1907+
},
1908+
))
1909+
1910+
// Create Watcher CR after all secrets and dependencies are in place
1911+
// so sub-CR controllers don't enter long exponential backoff.
1912+
spec := GetDefaultWatcherSpec()
1913+
spec["auth"] = map[string]any{"applicationCredentialSecret": acSecretName}
1914+
DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, spec))
1915+
1916+
DeferCleanup(
1917+
mariadb.DeleteDBService,
1918+
mariadb.CreateDBService(
1919+
watcherTest.Instance.Namespace,
1920+
*GetWatcher(watcherTest.Instance).Spec.DatabaseInstance,
1921+
corev1.ServiceSpec{
1922+
Ports: []corev1.ServicePort{{Port: 3306}},
1923+
},
1924+
),
1925+
)
1926+
1927+
mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount)
1928+
mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName)
1929+
infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL)
1930+
1931+
keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName)
1932+
th.SimulateJobSuccess(watcherTest.WatcherDBSync)
1933+
})
1934+
1935+
It("should add the consumer finalizer to the AC secret", func() {
1936+
Eventually(func(g Gomega) {
1937+
secret := th.GetSecret(types.NamespacedName{
1938+
Namespace: watcherTest.Instance.Namespace,
1939+
Name: acSecretName,
1940+
})
1941+
g.Expect(secret.Finalizers).To(
1942+
ContainElement(watcher.ACConsumerFinalizer))
1943+
}, timeout, interval).Should(Succeed())
1944+
})
1945+
1946+
It("should track the consumed AC secret in status", func() {
1947+
Eventually(func(g Gomega) {
1948+
w := GetWatcher(watcherTest.Instance)
1949+
g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName))
1950+
}, timeout, interval).Should(Succeed())
1951+
})
1952+
1953+
It("should move the finalizer from the old to the new secret on rotation", func() {
1954+
Eventually(func(g Gomega) {
1955+
secret := th.GetSecret(types.NamespacedName{
1956+
Namespace: watcherTest.Instance.Namespace,
1957+
Name: acSecretName,
1958+
})
1959+
g.Expect(secret.Finalizers).To(
1960+
ContainElement(watcher.ACConsumerFinalizer))
1961+
}, timeout, interval).Should(Succeed())
1962+
1963+
newACSecretName := "ac-watcher-consumer-rotated-secret" //nolint:gosec
1964+
newSecret := &corev1.Secret{
1965+
ObjectMeta: metav1.ObjectMeta{
1966+
Namespace: watcherTest.Instance.Namespace,
1967+
Name: newACSecretName,
1968+
},
1969+
Data: map[string][]byte{
1970+
keystonev1beta1.ACIDSecretKey: []byte("rotated-ac-id"),
1971+
keystonev1beta1.ACSecretSecretKey: []byte("rotated-ac-secret-value"), //nolint:gosec
1972+
},
1973+
}
1974+
DeferCleanup(k8sClient.Delete, ctx, newSecret)
1975+
Expect(k8sClient.Create(ctx, newSecret)).To(Succeed())
1976+
1977+
Eventually(func(g Gomega) {
1978+
w := GetWatcher(watcherTest.Instance)
1979+
w.Spec.Auth.ApplicationCredentialSecret = newACSecretName
1980+
g.Expect(k8sClient.Update(ctx, w)).Should(Succeed())
1981+
}, timeout, interval).Should(Succeed())
1982+
1983+
// New secret gets the consumer finalizer immediately (early in reconcile)
1984+
Eventually(func(g Gomega) {
1985+
secret := th.GetSecret(types.NamespacedName{
1986+
Namespace: watcherTest.Instance.Namespace,
1987+
Name: newACSecretName,
1988+
})
1989+
g.Expect(secret.Finalizers).To(
1990+
ContainElement(watcher.ACConsumerFinalizer))
1991+
}, timeout, interval).Should(Succeed())
1992+
1993+
// Old secret keeps the finalizer until all services deploy (split pattern)
1994+
secret := th.GetSecret(types.NamespacedName{
1995+
Namespace: watcherTest.Instance.Namespace,
1996+
Name: acSecretName,
1997+
})
1998+
Expect(secret.Finalizers).To(
1999+
ContainElement(watcher.ACConsumerFinalizer))
2000+
2001+
// Simulate all watcher services deploying successfully
2002+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet)
2003+
keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName)
2004+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet)
2005+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet)
2006+
2007+
// Now the old secret's finalizer is removed and status updated
2008+
Eventually(func(g Gomega) {
2009+
secret := th.GetSecret(types.NamespacedName{
2010+
Namespace: watcherTest.Instance.Namespace,
2011+
Name: acSecretName,
2012+
})
2013+
g.Expect(secret.Finalizers).NotTo(
2014+
ContainElement(watcher.ACConsumerFinalizer))
2015+
}, timeout, interval).Should(Succeed())
2016+
2017+
Eventually(func(g Gomega) {
2018+
w := GetWatcher(watcherTest.Instance)
2019+
g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(newACSecretName))
2020+
}, timeout, interval).Should(Succeed())
2021+
})
2022+
2023+
It("should remove the consumer finalizer from AC secret on CR deletion", func() {
2024+
Eventually(func(g Gomega) {
2025+
secret := th.GetSecret(types.NamespacedName{
2026+
Namespace: watcherTest.Instance.Namespace,
2027+
Name: acSecretName,
2028+
})
2029+
g.Expect(secret.Finalizers).To(
2030+
ContainElement(watcher.ACConsumerFinalizer))
2031+
}, timeout, interval).Should(Succeed())
2032+
2033+
th.DeleteInstance(GetWatcher(watcherTest.Instance))
2034+
2035+
secret := th.GetSecret(types.NamespacedName{
2036+
Namespace: watcherTest.Instance.Namespace,
2037+
Name: acSecretName,
2038+
})
2039+
Expect(secret.Finalizers).NotTo(
2040+
ContainElement(watcher.ACConsumerFinalizer))
2041+
})
2042+
})
2043+
18612044
When("ApplicationCredential is adopted on existing deployment", func() {
18622045
var appCredSecretName string
18632046
var appCredID string

test/kuttl/test-suites/default/appcred-tests/02-assert.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,23 @@ commands:
4646
fi
4747
echo "✓ watcher pods restarted after appcred secret became available"
4848
49+
echo "Checking consumer finalizer on AC secret..."
50+
finalizers=$(oc get -n "${NS}" secret/ac-watcher-test-secret -o jsonpath='{.metadata.finalizers}')
51+
if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then
52+
echo "ERROR: AC secret missing watcher consumer finalizer"
53+
echo " finalizers: ${finalizers}"
54+
exit 1
55+
fi
56+
echo "✓ AC secret has openstack.org/watcher-ac-consumer finalizer"
57+
58+
echo "Checking watcher status tracks the consumed AC secret..."
59+
status_ac=$(oc get -n "${NS}" watcher/watcher-kuttl -o jsonpath='{.status.applicationCredentialSecret}')
60+
if [ "${status_ac}" != "ac-watcher-test-secret" ]; then
61+
echo "ERROR: watcher.status.applicationCredentialSecret expected ac-watcher-test-secret, got ${status_ac}"
62+
exit 1
63+
fi
64+
echo "✓ watcher.status.applicationCredentialSecret = ${status_ac}"
65+
4966
echo "Checking watcher config contains application_credential_id..."
5067
oc exec -n "${NS}" pod/watcher-kuttl-api-0 -c watcher-api -- \
5168
bash -c "grep -q \"^application_credential_id = ${ac_id}$\" /etc/watcher/watcher.conf.d/00-default.conf"

0 commit comments

Comments
 (0)