Add transport URL secret rotation with consumer finalizer

When infra-operator rotates a RabbitMQ transport URL (creating a new
secret and user), consumer operators must hold a consumer finalizer on
the old secret until all their pods have rolled out with the new
credentials. Without this, infra-operator cleans up the old RabbitMQ
user while pods are still connected with old credentials, causing
message bus outages.

Design:

1. Add consumer finalizer to the current transport URL secret early
   in reconcile. Set instance.Status.TransportURLSecret for first-time
   setup only (when empty or unchanged); during rotation the status
   is updated solely by FinalizeSecretRotation at end of reconcile.

2. Pass transportURL.Status.SecretName directly to sub-CR creation
   functions and config generation as a parameter — never read from
   instance.Status.TransportURLSecret for sub-CR specs.

3. Use statefulset.IsReady() / deployment.IsReady() from lib-common
   in all sub-CR controllers for accurate rollout status.

4. Use object.ManageRotationGracePeriod() to enforce a 60-second
   grace period before evaluating the rotation guard. This gives
   sub-CRs time to detect config changes, update their workloads,
   and roll pods — without relying on informer cache freshness.

5. Guard: CredentialRotationGuardReady(true, conditions) — evaluates
   AllSubConditionIsTrue after the grace period expires. Only when
   all sub-CR conditions are True does FinalizeSecretRotation remove
   the consumer finalizer from the old secret.

The same pattern applies to notification transport URL secrets and
application credential secrets where applicable.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: lmiccini

api/go.mod

@@ -74,6 +74,10 @@ require (
 	sigs.k8s.io/yaml v1.6.0 // indirect
 )
 
+replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594
+
+replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462
+
 // mschuppert: map to latest commit from release-4.18 tag
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e //allow-merging

api/go.sum

@@ -68,6 +68,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4=
+github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8=
+github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594 h1:rMCoyJyTQaekhmKJYsIgK/u6w2Gat0dd/9xgBiUlyxM=
+github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -85,10 +89,6 @@ github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4=
 github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
 github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U=
 github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
-github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc=
-github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:OVFoNXzinsI0rq8gbegu8TnlDPkO409iyVoWhU4nEdQ=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:3cyU3HUhCoV7vscqea6ZUbkwxNSAJd1Rwk0P15vsUZw=

api/v1beta1/keystoneapplicationcredential.go

@@ -17,17 +17,8 @@ limitations under the License.
 package v1beta1
 
 import (
-	"context"
 	"fmt"
 	"strings"
-
-	corev1 "k8s.io/api/core/v1"
-	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/object"
 )
 
 // ApplicationCredentialData contains AC ID/Secret extracted from a Secret
@@ -67,68 +58,3 @@ func (ac *KeystoneApplicationCredential) IsEDPMService() bool {
 	return ac.GetAnnotations()[EDPMServiceAnnotation] != "false"
 }
 
-// ManageACSecretFinalizer ensures consumerFinalizer is present on the AC secret
-// identified by newSecretName and absent from the one identified by
-// oldSecretName. It is a no-op when both names are equal.
-func ManageACSecretFinalizer(
-	ctx context.Context,
-	h *helper.Helper,
-	namespace string,
-	newSecretName string,
-	oldSecretName string,
-	consumerFinalizer string,
-) error {
-	if newSecretName == oldSecretName {
-		return nil
-	}
-
-	var newObj, oldObj client.Object
-
-	if newSecretName != "" {
-		secret := &corev1.Secret{}
-		key := types.NamespacedName{Name: newSecretName, Namespace: namespace}
-		if err := h.GetClient().Get(ctx, key, secret); err != nil {
-			return fmt.Errorf("failed to get new AC secret %s: %w", newSecretName, err)
-		}
-		newObj = secret
-	}
-
-	if oldSecretName != "" {
-		secret := &corev1.Secret{}
-		key := types.NamespacedName{Name: oldSecretName, Namespace: namespace}
-		if err := h.GetClient().Get(ctx, key, secret); err != nil {
-			if !k8s_errors.IsNotFound(err) {
-				return fmt.Errorf("failed to get old AC secret %s: %w", oldSecretName, err)
-			}
-		} else {
-			oldObj = secret
-		}
-	}
-
-	return object.ManageConsumerFinalizer(ctx, h, newObj, oldObj, consumerFinalizer)
-}
-
-// RemoveACSecretConsumerFinalizer removes consumerFinalizer from the AC secret
-// identified by secretName. It is a no-op when secretName is empty or the
-// secret no longer exists.
-func RemoveACSecretConsumerFinalizer(
-	ctx context.Context,
-	h *helper.Helper,
-	namespace string,
-	secretName string,
-	consumerFinalizer string,
-) error {
-	if secretName == "" {
-		return nil
-	}
-
-	secret := &corev1.Secret{}
-	key := types.NamespacedName{Name: secretName, Namespace: namespace}
-	if err := h.GetClient().Get(ctx, key, secret); err != nil {
-		if k8s_errors.IsNotFound(err) {
-			return nil
-		}
-		return err
-	}
-	return object.RemoveConsumerFinalizer(ctx, h, secret, consumerFinalizer)
-}

go.mod

@@ -118,6 +118,10 @@ require (
 
 replace github.com/openstack-k8s-operators/keystone-operator/api => ./api
 
+replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594
+
+replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462
+
 // mschuppert: map to latest commit from release-4.18 tag
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e //allow-merging

go.sum

@@ -100,6 +100,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4=
+github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8=
+github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594 h1:rMCoyJyTQaekhmKJYsIgK/u6w2Gat0dd/9xgBiUlyxM=
+github.com/lmiccini/lib-common/modules/common v0.0.0-20260623081928-d67d41c03594/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
@@ -120,10 +124,6 @@ github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4=
 github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
 github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U=
 github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
-github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc=
-github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0=
 github.com/openstack-k8s-operators/lib-common/modules/edpm v0.0.0-20260618132757-fe8e60d1d8a6 h1:Iq7reRH6OtQ3MCqA/Tiz6e8fqfYi+h1o5cyZv0RNv0E=
 github.com/openstack-k8s-operators/lib-common/modules/edpm v0.0.0-20260618132757-fe8e60d1d8a6/go.mod h1:xsKeDFU3/xEObaVDqd6XEYV3MzvFswbWMlnr2Z3q3ZI=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:OVFoNXzinsI0rq8gbegu8TnlDPkO409iyVoWhU4nEdQ=

internal/controller/keystoneapi_controller.go

@@ -42,6 +42,7 @@ import (
 	job "github.com/openstack-k8s-operators/lib-common/modules/common/job"
 	labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels"
 	nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/object"
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
 	oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
@@ -522,6 +523,11 @@ func (r *KeystoneAPIReconciler) reconcileDelete(ctx context.Context, instance *k
 		}
 	}
 
+	if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+		instance.Status.TransportURLSecret, keystone.TransportConsumerFinalizer); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// Service is deleted so remove the finalizer.
 	controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
 	Log.Info("Reconciled Service delete successfully")
@@ -1069,9 +1075,9 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
 		Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op)))
 	}
 
-	instance.Status.TransportURLSecret = transportURL.Status.SecretName
+	currentTransportSecret := transportURL.Status.SecretName
 
-	if instance.Status.TransportURLSecret == "" {
+	if currentTransportSecret == "" {
 		// Since the TransportURL secret is automatically created by the Infra operator,
 		// we treat this as an info (because the user is not responsible for manually creating it).
 		Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name))
@@ -1084,6 +1090,19 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
 	}
 	Log.Info(fmt.Sprintf("TransportURL secret name %s", transportURL.Status.SecretName))
 	instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage)
+
+	// Set status early for first-time setup so PatchInstance persists it
+	// even on early returns. During rotation (old != current), the status
+	// is only updated by FinalizeSecretRotation at end of reconcile.
+	if instance.Status.TransportURLSecret == "" ||
+		instance.Status.TransportURLSecret == currentTransportSecret {
+		instance.Status.TransportURLSecret = currentTransportSecret
+	}
+
+	if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+		currentTransportSecret, keystone.TransportConsumerFinalizer); err != nil {
+		return ctrl.Result{}, err
+	}
 	// run check rabbitmq - end
 
 	//
@@ -1152,7 +1171,7 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
 	// - %-config configmap holding minimal keystone config required to get the service up, user can add additional files to be added to the service
 	// - parameters which has passwords gets added from the OpenStack secret via the init container
 	//
-	err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db)
+	err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db, currentTransportSecret)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -1468,6 +1487,30 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
 		return ctrl.Result{}, err
 	}
 
+	rotationPending := instance.Status.TransportURLSecret != "" &&
+		instance.Status.TransportURLSecret != currentTransportSecret
+	result, graceActive, err := object.ManageRotationGracePeriod(
+		ctx, r.Client, instance, rotationPending, 60*time.Second)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	if graceActive {
+		return result, nil
+	}
+
+	guardReady := condition.CredentialRotationGuardReady(true, &instance.Status.Conditions)
+	transportSecretName, err := object.FinalizeSecretRotation(
+		ctx, helper, instance.Namespace,
+		instance.Status.TransportURLSecret,
+		currentTransportSecret,
+		keystone.TransportConsumerFinalizer,
+		guardReady,
+	)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	instance.Status.TransportURLSecret = transportSecretName
+
 	Log.Info("Reconciled Service successfully")
 	return ctrl.Result{}, nil
 }
@@ -1520,6 +1563,7 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
 	envVars *map[string]env.Setter,
 	mc *memcachedv1.Memcached,
 	db *mariadbv1.Database,
+	transportURLSecretName string,
 ) error {
 	//
 	// create Configmap/Secret required for keystone input
@@ -1545,7 +1589,7 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
 	}
 	maps.Copy(customData, instance.Spec.DefaultConfigOverwrite)
 
-	transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace)
+	transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, transportURLSecretName, instance.Namespace)
 	if err != nil {
 		return err
 	}

internal/keystone/const.go

@@ -53,6 +53,8 @@ const (
 	FederationMultiRealmSecret = "keystone-multirealm-federation-secret"
 	// FederationDefaultMountPath - if user doesn't specify otherwise, this location is used
 	FederationDefaultMountPath = "/var/lib/config-data/default/multirealm-federation"
+	// TransportConsumerFinalizer is the finalizer added to transport URL secrets
+	TransportConsumerFinalizer = "openstack.org/keystone-transport-consumer"
 )
 
 // KeystonePropagation is the definition of the Keystone propagation service