InstanceHA production readiness improvements

Webhook validation:
- Add 11 webhook test cases covering Default(), ValidateCreate(), and
  ValidateUpdate() (cipher suite rejection, duplicate cloud detection,
  default values)
- Fix webhook test suite: correct relative paths and scheme registration
  so envtest can find CRDs and decode InstanceHa objects
- Add openStackCloud immutability check in ValidateUpdate to prevent
  changing the target cloud after creation

Container hardening:
- Add ReadOnlyRootFilesystem to the container SecurityContext
- Add /tmp emptyDir volume so the Python process has a writable temp dir
- Add unit test verifying the security context and /tmp volume

Config template:
- Add HEARTBEAT_CLIFF_THRESHOLD and K8S_API_CHECK_INTERVAL to the config
  template so operators can discover and tune these parameters

Controller error-path tests:
- Add tests for CR deletion and finalizer cleanup
- Add test verifying deployment-not-ready blocks ReadyCondition
- Add tests for missing openstack-config ConfigMap, openstack-config-secret,
  and fencing-secret (each verifies InputReadyCondition stays false)

Stale Nova connection fix:
- Refresh the Nova connection at the start of each poll cycle instead of
  reusing the one created at startup. Prevents token expiry issues where
  auth errors inside _process_stale_services were silently caught by inner
  exception handlers and never surfaced to the reconnection logic.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: lmiccini

apis/instanceha/v1beta1/instanceha_webhook.go

@@ -121,6 +121,18 @@ func (r *InstanceHa) ValidateUpdate(ctx context.Context, c client.Client, old ru
 	var allWarn []string
 	basePath := field.NewPath("spec")
 
+	oldInstance, ok := old.(*InstanceHa)
+	if !ok {
+		return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object"))
+	}
+
+	if r.Spec.OpenStackCloud != oldInstance.Spec.OpenStackCloud {
+		allErrs = append(allErrs, field.Forbidden(
+			basePath.Child("openStackCloud"),
+			"openStackCloud cannot be changed after creation",
+		))
+	}
+
 	allErrs = append(allErrs, r.Spec.ValidateTopology(basePath, r.Namespace)...)
 	allErrs = append(allErrs, r.validateUniqueOpenStackCloud(ctx, c, basePath)...)
 	allErrs = append(allErrs, r.validateCipherSuites(basePath)...)

internal/instanceha/funcs.go

@@ -186,6 +186,7 @@ func Deployment(
 							RunAsUser:                ptr.To(instanceHaUID),
 							RunAsGroup:               ptr.To(instanceHaUID),
 							RunAsNonRoot:             ptr.To(true),
+							ReadOnlyRootFilesystem:   ptr.To(true),
 							AllowPrivilegeEscalation: ptr.To(false),
 							Capabilities: &corev1.Capabilities{
 								Drop: []corev1.Capability{
@@ -279,6 +280,10 @@ func instancehaPodVolumeMounts() []corev1.VolumeMount {
 			SubPath:   "config.yaml",
 			ReadOnly:  true,
 		},
+		{
+			Name:      "tmp",
+			MountPath: "/tmp",
+		},
 	}
 }
 
@@ -336,5 +341,11 @@ func instancehaPodVolumes(
 				},
 			},
 		},
+		{
+			Name: "tmp",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
 	}
 }

internal/instanceha/funcs_test.go

@@ -70,3 +70,55 @@ func TestDeploymentDisabledEnvVar(t *testing.T) {
 		})
 	}
 }
+
+func TestDeploymentSecurityContext(t *testing.T) {
+	g := NewWithT(t)
+
+	instance := &instancehav1.InstanceHa{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-instanceha",
+			Namespace: "test-namespace",
+		},
+		Spec: instancehav1.InstanceHaSpec{
+			ContainerImage:        "test-image:latest",
+			OpenStackCloud:        "default",
+			OpenStackConfigMap:    "openstack-config",
+			OpenStackConfigSecret: "openstack-config-secret",
+			FencingSecret:         "fencing-secret",
+			InstanceHaConfigMap:   "instanceha-config",
+			InstanceHaKdumpPort:   7410,
+			Disabled:              "False",
+		},
+	}
+
+	labels := map[string]string{"app": "instanceha"}
+	annotations := map[string]string{}
+
+	dep := Deployment(instance, labels, annotations, "default", "hash123", "test-image:latest", nil, "", "", nil)
+
+	container := dep.Spec.Template.Spec.Containers[0]
+
+	g.Expect(container.SecurityContext.ReadOnlyRootFilesystem).NotTo(BeNil())
+	g.Expect(*container.SecurityContext.ReadOnlyRootFilesystem).To(BeTrue())
+
+	// Verify /tmp emptyDir volume exists
+	var tmpVolumeFound bool
+	for _, vol := range dep.Spec.Template.Spec.Volumes {
+		if vol.Name == "tmp" {
+			tmpVolumeFound = true
+			g.Expect(vol.VolumeSource.EmptyDir).NotTo(BeNil())
+			break
+		}
+	}
+	g.Expect(tmpVolumeFound).To(BeTrue(), "/tmp emptyDir volume should exist")
+
+	// Verify /tmp mount exists
+	var tmpMountFound bool
+	for _, mount := range container.VolumeMounts {
+		if mount.Name == "tmp" && mount.MountPath == "/tmp" {
+			tmpMountFound = true
+			break
+		}
+	}
+	g.Expect(tmpMountFound).To(BeTrue(), "/tmp volume mount should exist")
+}

internal/webhook/instanceha/v1beta1/instanceha_webhook_test.go

@@ -0,0 +1,279 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	. "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports
+	. "github.com/onsi/gomega"    //revive:disable:dot-imports
+	instancehav1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/instanceha/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("InstanceHa webhook", func() {
+	Context("Default method", func() {
+		It("should set all defaults on an empty spec", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-instanceha-defaults",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{},
+			}
+
+			instanceha.Default()
+
+			Expect(instanceha.Spec.OpenStackCloud).To(Equal(instancehav1beta1.OpenStackCloud))
+			Expect(instanceha.Spec.OpenStackConfigMap).To(Equal("openstack-config"))
+			Expect(instanceha.Spec.OpenStackConfigSecret).To(Equal("openstack-config-secret"))
+			Expect(instanceha.Spec.FencingSecret).To(Equal("fencing-secret"))
+			Expect(instanceha.Spec.InstanceHaConfigMap).To(Equal("instanceha-config"))
+			Expect(instanceha.Spec.InstanceHaKdumpPort).To(Equal(int32(7410)))
+			Expect(instanceha.Spec.InstanceHaHeartbeatPort).To(Equal(int32(7411)))
+			Expect(instanceha.Spec.MetricsTLS.MinTLSVersion).To(Equal("1.2"))
+			Expect(instanceha.Spec.MetricsTLS.CipherSuites).To(Equal("HIGH:!aNULL:!MD5:!RC4:!3DES:!kRSA"))
+		})
+
+		It("should not override explicitly set values", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-instanceha-explicit",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud:          "my-cloud",
+					OpenStackConfigMap:      "my-config",
+					OpenStackConfigSecret:   "my-secret",
+					FencingSecret:           "my-fencing",
+					InstanceHaConfigMap:     "my-instanceha-config",
+					InstanceHaKdumpPort:     8000,
+					InstanceHaHeartbeatPort: 9000,
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						MinTLSVersion: "1.3",
+						CipherSuites:  "HIGH",
+					},
+				},
+			}
+
+			instanceha.Default()
+
+			Expect(instanceha.Spec.OpenStackCloud).To(Equal("my-cloud"))
+			Expect(instanceha.Spec.OpenStackConfigMap).To(Equal("my-config"))
+			Expect(instanceha.Spec.OpenStackConfigSecret).To(Equal("my-secret"))
+			Expect(instanceha.Spec.FencingSecret).To(Equal("my-fencing"))
+			Expect(instanceha.Spec.InstanceHaConfigMap).To(Equal("my-instanceha-config"))
+			Expect(instanceha.Spec.InstanceHaKdumpPort).To(Equal(int32(8000)))
+			Expect(instanceha.Spec.InstanceHaHeartbeatPort).To(Equal(int32(9000)))
+			Expect(instanceha.Spec.MetricsTLS.MinTLSVersion).To(Equal("1.3"))
+			Expect(instanceha.Spec.MetricsTLS.CipherSuites).To(Equal("HIGH"))
+		})
+	})
+
+	Context("ValidateCreate", func() {
+		It("should accept a valid InstanceHa", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-valid",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "default",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:!aNULL:!MD5",
+					},
+				},
+			}
+
+			warnings, err := instanceha.ValidateCreate(ctx, k8sClient)
+			Expect(warnings).To(BeEmpty())
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should reject NULL cipher suite without ! prefix", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-null-cipher",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "default",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:NULL",
+					},
+				},
+			}
+
+			_, err := instanceha.ValidateCreate(ctx, k8sClient)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("NULL"))
+		})
+
+		It("should reject ALL cipher suite", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-all-cipher",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "default",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "ALL",
+					},
+				},
+			}
+
+			_, err := instanceha.ValidateCreate(ctx, k8sClient)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("ALL"))
+		})
+
+		It("should reject @SECLEVEL=0 cipher suite", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-seclevel0",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "default",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:@SECLEVEL=0",
+					},
+				},
+			}
+
+			_, err := instanceha.ValidateCreate(ctx, k8sClient)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("SECLEVEL=0"))
+		})
+
+		It("should accept cipher suite with !NULL (negated)", func() {
+			instanceha := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-negated-null",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "default",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:!NULL:!aNULL",
+					},
+				},
+			}
+
+			warnings, err := instanceha.ValidateCreate(ctx, k8sClient)
+			Expect(warnings).To(BeEmpty())
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should reject duplicate OpenStackCloud in the same namespace", func() {
+			existing := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-existing",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					ContainerImage: "test:latest",
+					OpenStackCloud: "shared-cloud",
+					Disabled:       "False",
+				},
+			}
+			Expect(k8sClient.Create(ctx, existing)).To(Succeed())
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(ctx, existing)).To(Succeed())
+			})
+
+			duplicate := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-duplicate",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "shared-cloud",
+				},
+			}
+
+			_, err := duplicate.ValidateCreate(ctx, k8sClient)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("already manages OpenStackCloud"))
+		})
+	})
+
+	Context("ValidateUpdate", func() {
+		It("should reject changing openStackCloud", func() {
+			oldInstance := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-update-immutable",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "original-cloud",
+				},
+			}
+
+			newInstance := oldInstance.DeepCopy()
+			newInstance.Spec.OpenStackCloud = "different-cloud"
+
+			_, err := newInstance.ValidateUpdate(ctx, k8sClient, oldInstance)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("cannot be changed after creation"))
+		})
+
+		It("should accept update when openStackCloud is unchanged", func() {
+			oldInstance := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-update-valid",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "my-cloud",
+					Disabled:       "False",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:!aNULL",
+					},
+				},
+			}
+
+			newInstance := oldInstance.DeepCopy()
+			newInstance.Spec.Disabled = "True"
+
+			warnings, err := newInstance.ValidateUpdate(ctx, k8sClient, oldInstance)
+			Expect(warnings).To(BeEmpty())
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should reject unsafe cipher suites on update", func() {
+			oldInstance := &instancehav1beta1.InstanceHa{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-update-cipher",
+					Namespace: "default",
+				},
+				Spec: instancehav1beta1.InstanceHaSpec{
+					OpenStackCloud: "my-cloud",
+					MetricsTLS: instancehav1beta1.InstanceHaMetricsTLS{
+						CipherSuites: "HIGH:!aNULL",
+					},
+				},
+			}
+
+			newInstance := oldInstance.DeepCopy()
+			newInstance.Spec.MetricsTLS.CipherSuites = "ALL"
+
+			_, err := newInstance.ValidateUpdate(ctx, k8sClient, oldInstance)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("ALL"))
+		})
+	})
+})