Skip to content

Commit 3fdfdd3

Browse files
committed
Add AC finalizer management
Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
1 parent 82c6f3e commit 3fdfdd3

15 files changed

Lines changed: 386 additions & 13 deletions

File tree

api/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

api/go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
9191
replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
9292

9393
replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
94+
95+
replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

api/v1beta1/watcher_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ type WatcherStatus struct {
6161

6262
// DecisionEngineServiceReadyCount defines the number or replicas ready from watcher-decision-engine
6363
DecisionEngineServiceReadyCount int32 `json:"decisionengineServiceReadyCount,omitempty"`
64+
65+
// ApplicationCredentialSecret - the AC secret watcher is currently
66+
// consuming and protecting with the openstack.org/watcher-ac-consumer
67+
// finalizer. Tracked so the controller can remove its finalizer from the
68+
// old secret when the openstack-operator rotates the reference.
69+
ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"`
6470
}
6571

6672
// WatcherDBPurge defines the parameters for the Watcher database purging cron job

config/crd/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
141141
replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
142142

143143
replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
144+
145+
replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20 h1:iyxfh2SDvQrOrsHItYAE3A3+8Ku9UnzWAq9jnLJDLjg=
2+
github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20/go.mod h1:SpO4CL7c5/1HG+61fP6kWhL2+3aqR+5SNatdZueKrz8=
13
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
24
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
35
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -120,8 +122,6 @@ github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyU
120122
github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
121123
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4=
122124
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs=
123-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f h1:28WYAUIef3uion0Pps6doCSSbgZtIcodGzwG6BHhCOw=
124-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f/go.mod h1:4ryvbSYuoN522BIPijnm0wMemPgJVKf7jCv8BNDq46I=
125125
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg=
126126
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs=
127127
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 h1:kUfZlcl+EbUBEWe6EGLXjzlUeYj7xZ21QsPA5jMJlwE=

internal/controller/watcher_controller.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,24 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
478478
return ctrl.Result{}, err
479479
}
480480

481+
// Add consumer finalizer to the new AC secret early, before deployment.
482+
// The old secret's finalizer is removed later (after all services deploy)
483+
// so that rapid rotations don't revoke a credential still in use by pods.
484+
if instance.Spec.Auth.ApplicationCredentialSecret != "" {
485+
if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace,
486+
instance.Spec.Auth.ApplicationCredentialSecret,
487+
"",
488+
watcher.ACConsumerFinalizer); err != nil {
489+
instance.Status.Conditions.Set(condition.FalseCondition(
490+
condition.ServiceConfigReadyCondition,
491+
condition.ErrorReason,
492+
condition.SeverityWarning,
493+
condition.ServiceConfigReadyErrorMessage,
494+
err.Error()))
495+
return ctrl.Result{}, err
496+
}
497+
}
498+
481499
instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
482500
// End of config generation for dbsync
483501

@@ -523,6 +541,27 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
523541
}
524542
// End of Watcher Applier deploy
525543

544+
// Manage the old AC secret's finalizer and status tracking.
545+
// On rotation (old != new), only remove the old secret's finalizer after
546+
// all sub-services are ready with the new credentials. This prevents
547+
// premature revocation during rapid rotations.
548+
isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret
549+
550+
if isRotation {
551+
allServicesReady := instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherAPIReadyCondition) &&
552+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherApplierReadyCondition) &&
553+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherDecisionEngineReadyCondition)
554+
if allServicesReady {
555+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
556+
instance.Status.ApplicationCredentialSecret, watcher.ACConsumerFinalizer); err != nil {
557+
return ctrl.Result{}, err
558+
}
559+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
560+
}
561+
} else {
562+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
563+
}
564+
526565
//
527566
// remove finalizers from unused MariaDBAccount records
528567
// this assumes all database-depedendent deployments are up and
@@ -1347,6 +1386,17 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch
13471386
}
13481387
//
13491388

1389+
// Remove consumer finalizer from AC secrets watcher was consuming.
1390+
for _, secretName := range []string{
1391+
instance.Status.ApplicationCredentialSecret,
1392+
instance.Spec.Auth.ApplicationCredentialSecret,
1393+
} {
1394+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
1395+
secretName, watcher.ACConsumerFinalizer); err != nil {
1396+
return ctrl.Result{}, err
1397+
}
1398+
}
1399+
13501400
controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
13511401
Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name))
13521402
return ctrl.Result{}, nil

internal/watcher/constants.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,7 @@ const (
5050

5151
// scriptVolume is the name of the volume used to ship scripts into pods
5252
scriptVolume = "scripts-volume"
53+
54+
// ACConsumerFinalizer is added to AC secrets that watcher is actively consuming
55+
ACConsumerFinalizer = "openstack.org/watcher-ac-consumer"
5356
)

test/functional/watcher_controller_test.go

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
. "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers"
1818
mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
1919
watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1"
20+
"github.com/openstack-k8s-operators/watcher-operator/internal/watcher"
2021
corev1 "k8s.io/api/core/v1"
2122
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2223
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -1858,6 +1859,206 @@ var _ = Describe("Watcher controller", func() {
18581859
})
18591860
})
18601861

1862+
When("ApplicationCredential consumer finalizer is managed", func() {
1863+
var acSecretName string
1864+
1865+
BeforeEach(func() {
1866+
acSecretName = "ac-watcher-consumer-fnz-secret" //nolint:gosec
1867+
1868+
acSecret := &corev1.Secret{
1869+
ObjectMeta: metav1.ObjectMeta{
1870+
Name: acSecretName,
1871+
Namespace: watcherTest.Instance.Namespace,
1872+
},
1873+
Data: map[string][]byte{
1874+
keystonev1beta1.ACIDSecretKey: []byte("consumer-test-ac-id"),
1875+
keystonev1beta1.ACSecretSecretKey: []byte("consumer-test-ac-secret"), //nolint:gosec
1876+
},
1877+
}
1878+
Expect(k8sClient.Create(ctx, acSecret)).To(Succeed())
1879+
DeferCleanup(k8sClient.Delete, ctx, acSecret)
1880+
1881+
DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret"))
1882+
1883+
memcachedSpec := memcachedv1.MemcachedSpec{
1884+
MemcachedSpecCore: memcachedv1.MemcachedSpecCore{
1885+
Replicas: ptr.To(int32(1)),
1886+
},
1887+
}
1888+
DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.Watcher.Namespace, MemcachedInstance, memcachedSpec))
1889+
infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace)
1890+
1891+
DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace))
1892+
1893+
DeferCleanup(
1894+
k8sClient.Delete, ctx, th.CreateSecret(
1895+
types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "metric-storage-prometheus-endpoint"},
1896+
map[string][]byte{
1897+
"host": []byte("prometheus.example.com"),
1898+
"port": []byte("9090"),
1899+
},
1900+
))
1901+
1902+
DeferCleanup(
1903+
k8sClient.Delete, ctx, th.CreateSecret(
1904+
types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName},
1905+
map[string][]byte{
1906+
"WatcherPassword": []byte("password"),
1907+
},
1908+
))
1909+
1910+
// Create Watcher CR after all secrets and dependencies are in place
1911+
// so sub-CR controllers don't enter long exponential backoff.
1912+
spec := GetDefaultWatcherSpec()
1913+
spec["auth"] = map[string]any{"applicationCredentialSecret": acSecretName}
1914+
DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, spec))
1915+
1916+
DeferCleanup(
1917+
mariadb.DeleteDBService,
1918+
mariadb.CreateDBService(
1919+
watcherTest.Instance.Namespace,
1920+
*GetWatcher(watcherTest.Instance).Spec.DatabaseInstance,
1921+
corev1.ServiceSpec{
1922+
Ports: []corev1.ServicePort{{Port: 3306}},
1923+
},
1924+
),
1925+
)
1926+
1927+
mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount)
1928+
mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName)
1929+
infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL)
1930+
1931+
keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName)
1932+
th.SimulateJobSuccess(watcherTest.WatcherDBSync)
1933+
})
1934+
1935+
It("should add the consumer finalizer to the AC secret", func() {
1936+
Eventually(func(g Gomega) {
1937+
secret := th.GetSecret(types.NamespacedName{
1938+
Namespace: watcherTest.Instance.Namespace,
1939+
Name: acSecretName,
1940+
})
1941+
g.Expect(secret.Finalizers).To(
1942+
ContainElement(watcher.ACConsumerFinalizer))
1943+
}, timeout, interval).Should(Succeed())
1944+
})
1945+
1946+
It("should track the consumed AC secret in status", func() {
1947+
Eventually(func(g Gomega) {
1948+
w := GetWatcher(watcherTest.Instance)
1949+
g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName))
1950+
}, timeout, interval).Should(Succeed())
1951+
})
1952+
1953+
It("should move the finalizer from the old to the new secret on rotation", func() {
1954+
Eventually(func(g Gomega) {
1955+
secret := th.GetSecret(types.NamespacedName{
1956+
Namespace: watcherTest.Instance.Namespace,
1957+
Name: acSecretName,
1958+
})
1959+
g.Expect(secret.Finalizers).To(
1960+
ContainElement(watcher.ACConsumerFinalizer))
1961+
}, timeout, interval).Should(Succeed())
1962+
1963+
// Simulate all watcher services deploying successfully
1964+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet)
1965+
keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName)
1966+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet)
1967+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet)
1968+
1969+
Eventually(func(g Gomega) {
1970+
w := GetWatcher(watcherTest.Instance)
1971+
g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName))
1972+
}, timeout, interval).Should(Succeed())
1973+
1974+
th.ExpectCondition(
1975+
watcherTest.Instance,
1976+
ConditionGetterFunc(WatcherConditionGetter),
1977+
condition.ReadyCondition,
1978+
corev1.ConditionTrue,
1979+
)
1980+
1981+
newACSecretName := "ac-watcher-consumer-rotated-secret" //nolint:gosec
1982+
newSecret := &corev1.Secret{
1983+
ObjectMeta: metav1.ObjectMeta{
1984+
Namespace: watcherTest.Instance.Namespace,
1985+
Name: newACSecretName,
1986+
},
1987+
Data: map[string][]byte{
1988+
keystonev1beta1.ACIDSecretKey: []byte("rotated-ac-id"),
1989+
keystonev1beta1.ACSecretSecretKey: []byte("rotated-ac-secret-value"), //nolint:gosec
1990+
},
1991+
}
1992+
DeferCleanup(k8sClient.Delete, ctx, newSecret)
1993+
Expect(k8sClient.Create(ctx, newSecret)).To(Succeed())
1994+
1995+
Eventually(func(g Gomega) {
1996+
w := GetWatcher(watcherTest.Instance)
1997+
w.Spec.Auth.ApplicationCredentialSecret = newACSecretName
1998+
g.Expect(k8sClient.Update(ctx, w)).Should(Succeed())
1999+
}, timeout, interval).Should(Succeed())
2000+
2001+
// New secret gets the consumer finalizer immediately (early in reconcile)
2002+
Eventually(func(g Gomega) {
2003+
secret := th.GetSecret(types.NamespacedName{
2004+
Namespace: watcherTest.Instance.Namespace,
2005+
Name: newACSecretName,
2006+
})
2007+
g.Expect(secret.Finalizers).To(
2008+
ContainElement(watcher.ACConsumerFinalizer))
2009+
}, timeout, interval).Should(Succeed())
2010+
2011+
// Old secret keeps the finalizer until all services deploy (split pattern)
2012+
secret := th.GetSecret(types.NamespacedName{
2013+
Namespace: watcherTest.Instance.Namespace,
2014+
Name: acSecretName,
2015+
})
2016+
Expect(secret.Finalizers).To(
2017+
ContainElement(watcher.ACConsumerFinalizer))
2018+
2019+
// Simulate all watcher services deploying successfully
2020+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet)
2021+
keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName)
2022+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet)
2023+
th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet)
2024+
2025+
// Now the old secret's finalizer is removed and status updated
2026+
Eventually(func(g Gomega) {
2027+
secret := th.GetSecret(types.NamespacedName{
2028+
Namespace: watcherTest.Instance.Namespace,
2029+
Name: acSecretName,
2030+
})
2031+
g.Expect(secret.Finalizers).NotTo(
2032+
ContainElement(watcher.ACConsumerFinalizer))
2033+
}, timeout, interval).Should(Succeed())
2034+
2035+
Eventually(func(g Gomega) {
2036+
w := GetWatcher(watcherTest.Instance)
2037+
g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(newACSecretName))
2038+
}, timeout, interval).Should(Succeed())
2039+
})
2040+
2041+
It("should remove the consumer finalizer from AC secret on CR deletion", func() {
2042+
Eventually(func(g Gomega) {
2043+
secret := th.GetSecret(types.NamespacedName{
2044+
Namespace: watcherTest.Instance.Namespace,
2045+
Name: acSecretName,
2046+
})
2047+
g.Expect(secret.Finalizers).To(
2048+
ContainElement(watcher.ACConsumerFinalizer))
2049+
}, timeout, interval).Should(Succeed())
2050+
2051+
th.DeleteInstance(GetWatcher(watcherTest.Instance))
2052+
2053+
secret := th.GetSecret(types.NamespacedName{
2054+
Namespace: watcherTest.Instance.Namespace,
2055+
Name: acSecretName,
2056+
})
2057+
Expect(secret.Finalizers).NotTo(
2058+
ContainElement(watcher.ACConsumerFinalizer))
2059+
})
2060+
})
2061+
18612062
When("ApplicationCredential is adopted on existing deployment", func() {
18622063
var appCredSecretName string
18632064
var appCredID string

0 commit comments

Comments
 (0)