Skip to content

Commit 9f5b18d

Browse files
committed
Add AC finalizer management
Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
1 parent b85e441 commit 9f5b18d

19 files changed

Lines changed: 407 additions & 34 deletions

api/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

api/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ toolchain go1.24.6
66

77
require (
88
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31
9-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0
9+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587
1010
k8s.io/api v0.31.14
1111
k8s.io/apimachinery v0.31.14
1212
k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d

api/go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
7676
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
7777
github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps=
7878
github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE=
79-
github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc=
80-
github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
79+
github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA=
80+
github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
8181
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4=
8282
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs=
83-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg=
84-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs=
83+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 h1:p03uEXoSreyu7LpFmb9YyYM8tEx2D2+7qqhLXNWHTq0=
84+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:JC04T5G4E/he5ukonV1oCqa0QzFkLv761VbLruVghJM=
8585
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
8686
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
8787
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

api/v1beta1/watcher_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ type WatcherStatus struct {
6161

6262
// DecisionEngineServiceReadyCount defines the number or replicas ready from watcher-decision-engine
6363
DecisionEngineServiceReadyCount int32 `json:"decisionengineServiceReadyCount,omitempty"`
64+
65+
// ApplicationCredentialSecret - the AC secret watcher is currently
66+
// consuming and protecting with the openstack.org/watcher-ac-consumer
67+
// finalizer. Tracked so the controller can remove its finalizer from the
68+
// old secret when the openstack-operator rotates the reference.
69+
ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"`
6470
}
6571

6672
// WatcherDBPurge defines the parameters for the Watcher database purging cron job

config/crd/bases/watcher.openstack.org_watchers.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,13 @@ spec:
737737
from watcher-api
738738
format: int32
739739
type: integer
740+
applicationCredentialSecret:
741+
description: |-
742+
ApplicationCredentialSecret - the AC secret watcher is currently
743+
consuming and protecting with the openstack.org/watcher-ac-consumer
744+
finalizer. Tracked so the controller can remove its finalizer from the
745+
old secret when the openstack-operator rotates the reference.
746+
type: string
740747
applierServiceReadyCount:
741748
description: ApplierServiceReadyCount defines the number or replicas
742749
ready from watcher-applier

go.mod

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ require (
66
github.com/go-logr/logr v1.4.3
77
github.com/google/uuid v1.6.0
88
github.com/onsi/ginkgo/v2 v2.28.2
9-
github.com/onsi/gomega v1.40.0
9+
github.com/onsi/gomega v1.41.0
1010
github.com/openshift/api v3.9.0+incompatible
1111
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31
12-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f
13-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0
14-
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0
12+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf
13+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587
14+
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587
1515
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65
1616
github.com/openstack-k8s-operators/watcher-operator/api v0.0.0-00010101000000-000000000000
1717
go.uber.org/zap v1.28.0
@@ -62,8 +62,8 @@ require (
6262
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
6363
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
6464
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
65-
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 // indirect
66-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162 // indirect
65+
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0 // indirect
66+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0 // indirect
6767
github.com/pkg/errors v0.9.1 // indirect
6868
github.com/prometheus/client_golang v1.22.0 // indirect
6969
github.com/prometheus/client_model v0.6.2 // indirect

go.sum

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -114,22 +114,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
114114
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
115115
github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps=
116116
github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE=
117-
github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc=
118-
github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
117+
github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA=
118+
github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
119119
github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U=
120120
github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
121121
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4=
122122
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs=
123-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f h1:28WYAUIef3uion0Pps6doCSSbgZtIcodGzwG6BHhCOw=
124-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f/go.mod h1:4ryvbSYuoN522BIPijnm0wMemPgJVKf7jCv8BNDq46I=
125-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg=
126-
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs=
127-
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 h1:kUfZlcl+EbUBEWe6EGLXjzlUeYj7xZ21QsPA5jMJlwE=
128-
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI=
129-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162 h1:b/E0t3klHOxMUzpp/TaMB6OwLliv9mwsHxE94i8Rifc=
130-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162/go.mod h1:xSRQQt2ygoUdjq0pJelQ7X2aeKPIgAj71O9NpV10CQY=
131-
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 h1:mG3QhS/QWv9Y/AkZZ5OzO6hu6+l5oDXnI/Q5ZUbj6Zs=
132-
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:ZYG9CQe7cOePOKQbenEZFA28kPdkUOe9QKbDRwGhEV0=
123+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf h1:FoKK0zNo48i4ZMFxScupCK/YAmy6Ps4IILz3CK4BCTI=
124+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf/go.mod h1:VNX1Mda2u5+yGxycIyVrgABucitMDR9ct3Lj6ROS92I=
125+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 h1:p03uEXoSreyu7LpFmb9YyYM8tEx2D2+7qqhLXNWHTq0=
126+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:JC04T5G4E/he5ukonV1oCqa0QzFkLv761VbLruVghJM=
127+
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0 h1:kMie+G0aHlGwDHjimjj8AUxTl2R7LGfai/8pev2T+TY=
128+
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI=
129+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0 h1:wuzSibIT9F/5RbMmxvBVFj6fy2vtKo58nibzmk5L4PM=
130+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:tft3oDiN+v6wX3ILPXGUM/gCLJz6QtrPN63hxpJ3E24=
131+
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587 h1:jpouKcgs2Kc5z2JHIpvsXMxEonfXLgzX3KswuBoeKQ0=
132+
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:nLS2oK4pBo756JNN1cPgr44S0X9V11QScgVla89Ojok=
133133
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65 h1:MNjmU326MKwYU6xT6AL2aKbr/0ids87wP5B+s5CL2O0=
134134
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65/go.mod h1:I2LO+wGIzbirVczQ9qo/49y2F5Zo/MEg/pabkXOrY2M=
135135
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

internal/controller/watcher_controller.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,24 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
478478
return ctrl.Result{}, err
479479
}
480480

481+
// Add consumer finalizer to the new AC secret early, before deployment.
482+
// The old secret's finalizer is removed later (after all services deploy)
483+
// so that rapid rotations don't revoke a credential still in use by pods.
484+
if instance.Spec.Auth.ApplicationCredentialSecret != "" {
485+
if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace,
486+
instance.Spec.Auth.ApplicationCredentialSecret,
487+
"",
488+
watcher.ACConsumerFinalizer); err != nil {
489+
instance.Status.Conditions.Set(condition.FalseCondition(
490+
condition.ServiceConfigReadyCondition,
491+
condition.ErrorReason,
492+
condition.SeverityWarning,
493+
condition.ServiceConfigReadyErrorMessage,
494+
err.Error()))
495+
return ctrl.Result{}, err
496+
}
497+
}
498+
481499
instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
482500
// End of config generation for dbsync
483501

@@ -523,6 +541,27 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
523541
}
524542
// End of Watcher Applier deploy
525543

544+
// Manage the old AC secret's finalizer and status tracking.
545+
// On rotation (old != new), only remove the old secret's finalizer after
546+
// all sub-services are ready with the new credentials. This prevents
547+
// premature revocation during rapid rotations.
548+
isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret
549+
550+
if isRotation {
551+
allServicesReady := instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherAPIReadyCondition) &&
552+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherApplierReadyCondition) &&
553+
instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherDecisionEngineReadyCondition)
554+
if allServicesReady {
555+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
556+
instance.Status.ApplicationCredentialSecret, watcher.ACConsumerFinalizer); err != nil {
557+
return ctrl.Result{}, err
558+
}
559+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
560+
}
561+
} else {
562+
instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
563+
}
564+
526565
//
527566
// remove finalizers from unused MariaDBAccount records
528567
// this assumes all database-depedendent deployments are up and
@@ -1347,6 +1386,17 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch
13471386
}
13481387
//
13491388

1389+
// Remove consumer finalizer from AC secrets watcher was consuming.
1390+
for _, secretName := range []string{
1391+
instance.Status.ApplicationCredentialSecret,
1392+
instance.Spec.Auth.ApplicationCredentialSecret,
1393+
} {
1394+
if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
1395+
secretName, watcher.ACConsumerFinalizer); err != nil {
1396+
return ctrl.Result{}, err
1397+
}
1398+
}
1399+
13501400
controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
13511401
Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name))
13521402
return ctrl.Result{}, nil

internal/watcher/constants.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,7 @@ const (
5050

5151
// scriptVolume is the name of the volume used to ship scripts into pods
5252
scriptVolume = "scripts-volume"
53+
54+
// ACConsumerFinalizer is added to AC secrets that watcher is actively consuming
55+
ACConsumerFinalizer = "openstack.org/watcher-ac-consumer"
5356
)

0 commit comments

Comments
 (0)