Add application credential finalizer management

Heat now tracks which AC secret it is consuming via Status.ApplicationCredentialSecret and manages the openstack.org/heat-ac-consumer finalizer on that secret. This ensures keystone-operator does not prematurely revoke the application credential while Heat is still using it. On rotation (when the spec reference changes), the finalizer is moved from the old secret to the new one. On Heat CR deletion, the finalizer is cleaned up from all referenced secrets.

Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>

Author: Deydra71

api/bases/heat.openstack.org_heats.yaml

@@ -2093,6 +2093,13 @@ spec:
           status:
             description: HeatStatus defines the observed state of Heat
             properties:
+              applicationCredentialSecret:
+                description: |-
+                  ApplicationCredentialSecret - the AC secret Heat is currently
+                  consuming and protecting with the openstack.org/heat-ac-consumer
+                  finalizer. Tracked so the controller can remove its finalizer from the
+                  old secret when the openstack-operator rotates the reference.
+                type: string
               conditions:
                 description: Conditions
                 items:

api/v1beta1/heat_types.go

@@ -42,7 +42,6 @@ const (
 	HeatDatabaseMigrationAnnotation = "heat.openstack.org/database-migration"
 )
 
-
 // HeatSpec defines the desired state of Heat
 type HeatSpec struct {
 	HeatSpecBase `json:",inline"`
@@ -177,6 +176,12 @@ type HeatStatus struct {
 	// ReadyCount of Heat Engine instance
 	HeatEngineReadyCount int32 `json:"heatEngineReadyCount,omitempty"`
 
+	// ApplicationCredentialSecret - the AC secret Heat is currently
+	// consuming and protecting with the openstack.org/heat-ac-consumer
+	// finalizer. Tracked so the controller can remove its finalizer from the
+	// old secret when the openstack-operator rotates the reference.
+	ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"`
+
 	// ObservedGeneration - the most recent generation observed for this
 	// service. If the observed generation is less than the spec generation,
 	// then the controller has not processed the latest changes injected by

config/crd/bases/heat.openstack.org_heats.yaml

@@ -2093,6 +2093,13 @@ spec:
           status:
             description: HeatStatus defines the observed state of Heat
             properties:
+              applicationCredentialSecret:
+                description: |-
+                  ApplicationCredentialSecret - the AC secret Heat is currently
+                  consuming and protecting with the openstack.org/heat-ac-consumer
+                  finalizer. Tracked so the controller can remove its finalizer from the
+                  old secret when the openstack-operator rotates the reference.
+                type: string
               conditions:
                 description: Conditions
                 items:

go.mod

@@ -7,6 +7,7 @@ replace github.com/openstack-k8s-operators/heat-operator/api => ./api
 require (
 	github.com/go-logr/logr v1.4.3
 	github.com/google/uuid v1.6.0
+	github.com/gophercloud/gophercloud/v2 v2.8.0
 	github.com/onsi/ginkgo/v2 v2.28.2
 	github.com/onsi/gomega v1.41.0
 	github.com/openstack-k8s-operators/heat-operator/api v0.3.1-0.20240214134649-6643d1b09d49
@@ -52,7 +53,6 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
-	github.com/gophercloud/gophercloud/v2 v2.8.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

internal/controller/heat_controller.go

@@ -466,6 +466,19 @@ func (r *HeatReconciler) reconcileDelete(ctx context.Context, instance *heatv1be
 		}
 	}
 
+	// Remove consumer finalizer from AC secrets Heat was consuming.
+	// Check both status and spec to handle the edge case where the reconciler
+	// crashed after adding the finalizer but before updating the status.
+	for _, secretName := range []string{
+		instance.Status.ApplicationCredentialSecret,
+		instance.Spec.Auth.ApplicationCredentialSecret,
+	} {
+		if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+			secretName, heat.ACConsumerFinalizer); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	// Service is deleted so remove the finalizer.
 	controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
 	Log.Info("Reconciled Heat delete successfully")
@@ -813,6 +826,24 @@ func (r *HeatReconciler) reconcileNormal(ctx context.Context, instance *heatv1be
 
 	// Create Secrets - end
 
+	// Add consumer finalizer to the new AC secret early, before deployment.
+	// The old secret's finalizer is removed later (after all services deploy)
+	// so that rapid rotations don't revoke a credential still in use by pods.
+	if instance.Spec.Auth.ApplicationCredentialSecret != "" {
+		if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace,
+			instance.Spec.Auth.ApplicationCredentialSecret,
+			"",
+			heat.ACConsumerFinalizer); err != nil {
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				condition.ServiceConfigReadyCondition,
+				condition.ErrorReason,
+				condition.SeverityWarning,
+				condition.ServiceConfigReadyErrorMessage,
+				err.Error()))
+			return ctrl.Result{}, err
+		}
+	}
+
 	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
 
 	//
@@ -1033,6 +1064,25 @@ func (r *HeatReconciler) reconcileNormal(ctx context.Context, instance *heatv1be
 			Log.Info(fmt.Sprintf("Deployment %s successfully reconciled - operation: %s", instance.Name, string(op)))
 		}
 	}
+	// Manage the old AC secret's finalizer and status tracking.
+	// On rotation (old != new), only remove the old secret's finalizer after
+	// all sub-services are ready with the new credentials. This prevents
+	// premature revocation during rapid rotations.
+	isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret
+
+	if isRotation {
+		allServicesReady := instance.Status.Conditions.AllSubConditionIsTrue()
+		if allServicesReady {
+			if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+				instance.Status.ApplicationCredentialSecret, heat.ACConsumerFinalizer); err != nil {
+				return ctrl.Result{}, err
+			}
+			instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
+		}
+	} else {
+		instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
+	}
+
 	// We reached the end of the Reconcile, update the Ready condition based on
 	// the sub conditions
 	if instance.Status.Conditions.AllSubConditionIsTrue() {

internal/heat/const.go

@@ -92,6 +92,9 @@ const (
 
 	// HeatManage is the base command used for DBPurge
 	HeatManage = "/usr/bin/heat-manage"
+
+	// ACConsumerFinalizer is added to AC secrets that heat is actively consuming
+	ACConsumerFinalizer = "openstack.org/heat-ac-consumer"
 )
 
 // DbsyncPropagation keeps track of the DBSync Service Propagation Type

test/functional/api_fixture.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package functional_test contains functional tests for the heat operator.
+package functional_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	"github.com/gophercloud/gophercloud/v2/openstack/identity/v3/roles"
+	keystone_helpers "github.com/openstack-k8s-operators/keystone-operator/api/test/helpers"
+	api "github.com/openstack-k8s-operators/lib-common/modules/test/apis"
+)
+
+func heatKeystoneTokenResponse(keystoneURL string) string {
+	return fmt.Sprintf(
+		`{"token":{"catalog":[{"endpoints":[{"interface":"internal","region_id":"RegionOne","url":"%s","region":"RegionOne"},{"interface":"public","region_id":"RegionOne","url":"%s","region":"RegionOne"}],"type":"identity","name":"keystone"}]}}`,
+		keystoneURL, keystoneURL)
+}
+
+func setupHeatKeystoneFixture(logger logr.Logger) *keystone_helpers.KeystoneAPIFixture {
+	fixture := keystone_helpers.NewKeystoneAPIFixtureWithServer(logger)
+	roleStore := map[string]roles.Role{
+		"admin": {
+			Name: "admin",
+			ID:   uuid.NewString(),
+		},
+	}
+
+	fixture.Setup(
+		api.Handler{Pattern: "/", Func: fixture.HandleVersion},
+		api.Handler{Pattern: "/v3/users", Func: fixture.HandleUsers},
+		api.Handler{Pattern: "/v3/domains", Func: fixture.HandleDomains},
+		api.Handler{Pattern: "/v3/roles", Func: func(w http.ResponseWriter, r *http.Request) {
+			fixture.LogRequest(r)
+			switch r.Method {
+			case "GET":
+				nameFilter := r.URL.Query().Get("name")
+				var rs []roles.Role
+				for name, role := range roleStore {
+					if nameFilter == "" || name == nameFilter {
+						rs = append(rs, role)
+					}
+				}
+				resp := struct {
+					Roles []roles.Role `json:"roles"`
+				}{Roles: rs}
+				bytes, err := json.Marshal(&resp)
+				if err != nil {
+					fixture.InternalError(err, "Error during marshalling response", w, r)
+					return
+				}
+				w.Header().Add("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				_, _ = fmt.Fprint(w, string(bytes))
+			case "POST":
+				bytes, err := io.ReadAll(r.Body)
+				if err != nil {
+					fixture.InternalError(err, "Error reading request body", w, r)
+					return
+				}
+				var s struct {
+					Role roles.Role `json:"role"`
+				}
+				if err := json.Unmarshal(bytes, &s); err != nil {
+					fixture.InternalError(err, "Error during unmarshalling request", w, r)
+					return
+				}
+				if s.Role.ID == "" {
+					s.Role.ID = uuid.NewString()
+				}
+				roleStore[s.Role.Name] = s.Role
+				respBytes, err := json.Marshal(&s)
+				if err != nil {
+					fixture.InternalError(err, "Error during marshalling response", w, r)
+					return
+				}
+				w.Header().Add("Content-Type", "application/json")
+				w.WriteHeader(http.StatusCreated)
+				_, _ = fmt.Fprint(w, string(respBytes))
+			default:
+				fixture.UnexpectedRequest(w, r)
+			}
+		}},
+		api.Handler{Pattern: "/v3/role_assignments", Func: func(w http.ResponseWriter, r *http.Request) {
+			fixture.LogRequest(r)
+			if r.Method != "GET" {
+				fixture.UnexpectedRequest(w, r)
+				return
+			}
+			w.Header().Add("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			_, _ = fmt.Fprint(w, `{"role_assignments":[]}`)
+		}},
+		api.Handler{Pattern: "/v3/domains/", Func: func(w http.ResponseWriter, r *http.Request) {
+			fixture.LogRequest(r)
+			if r.Method == "PUT" {
+				w.WriteHeader(http.StatusNoContent)
+				return
+			}
+			fixture.UnexpectedRequest(w, r)
+		}},
+		api.Handler{Pattern: "/v3/auth/tokens", Func: func(w http.ResponseWriter, r *http.Request) {
+			fixture.LogRequest(r)
+			if r.Method != "POST" {
+				fixture.UnexpectedRequest(w, r)
+				return
+			}
+			w.Header().Add("Content-Type", "application/json")
+			w.WriteHeader(http.StatusAccepted)
+			_, _ = fmt.Fprint(w, heatKeystoneTokenResponse(fixture.Endpoint()))
+		}},
+	)
+	return fixture
+}

test/functional/base_test.go

@@ -19,13 +19,15 @@ package functional_test
 import (
 	. "github.com/onsi/gomega" //revive:disable:dot-imports
 
+	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1"
+	"github.com/openstack-k8s-operators/heat-operator/internal/heat"
 	rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 )
@@ -155,6 +157,81 @@ func GetHeatSpecWithNotificationsBus(notificationsCluster *string, notifications
 	return spec
 }
 
+func simulateHeatSubCRReady(g Gomega, name types.NamespacedName) {
+	heatAPI := &heatv1.HeatAPI{}
+	g.Expect(k8sClient.Get(ctx, types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      name.Name + "-api",
+	}, heatAPI)).To(Succeed())
+	heatAPI.Status.ObservedGeneration = heatAPI.Generation
+	heatAPI.Status.ReadyCount = 1
+	heatAPI.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
+	g.Expect(k8sClient.Status().Update(ctx, heatAPI)).To(Succeed())
+
+	heatCfnAPI := &heatv1.HeatCfnAPI{}
+	g.Expect(k8sClient.Get(ctx, types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      name.Name + "-cfnapi",
+	}, heatCfnAPI)).To(Succeed())
+	heatCfnAPI.Status.ObservedGeneration = heatCfnAPI.Generation
+	heatCfnAPI.Status.ReadyCount = 1
+	heatCfnAPI.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
+	g.Expect(k8sClient.Status().Update(ctx, heatCfnAPI)).To(Succeed())
+
+	heatEngine := &heatv1.HeatEngine{}
+	g.Expect(k8sClient.Get(ctx, types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      name.Name + "-engine",
+	}, heatEngine)).To(Succeed())
+	heatEngine.Status.ObservedGeneration = heatEngine.Generation
+	heatEngine.Status.ReadyCount = 1
+	heatEngine.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
+	g.Expect(k8sClient.Status().Update(ctx, heatEngine)).To(Succeed())
+}
+
+func simulateHeatSubServicesReady(name types.NamespacedName) {
+	Eventually(func(g Gomega) {
+		for _, depName := range []string{"heat-api", "heat-cfnapi", "heat-engine"} {
+			deployment := &appsv1.Deployment{}
+			g.Expect(k8sClient.Get(ctx, types.NamespacedName{
+				Namespace: name.Namespace,
+				Name:      depName,
+			}, deployment)).To(Succeed())
+		}
+	}, timeout, interval).Should(Succeed())
+	th.SimulateDeploymentReplicaReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      "heat-api",
+	})
+	th.SimulateDeploymentReplicaReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      "heat-cfnapi",
+	})
+	th.SimulateDeploymentReplicaReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      "heat-engine",
+	})
+	Eventually(func(g Gomega) {
+		simulateHeatSubCRReady(g, name)
+	}, timeout, interval).Should(Succeed())
+	keystone.SimulateKeystoneServiceReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      heat.ServiceName,
+	})
+	keystone.SimulateKeystoneEndpointReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      heat.ServiceName,
+	})
+	keystone.SimulateKeystoneServiceReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      heat.CfnServiceName,
+	})
+	keystone.SimulateKeystoneEndpointReady(types.NamespacedName{
+		Namespace: name.Namespace,
+		Name:      heat.CfnServiceName,
+	})
+}
+
 func GetTransportURL(name types.NamespacedName) *rabbitmqv1.TransportURL {
 	instance := &rabbitmqv1.TransportURL{}
 	Eventually(func(g Gomega) {