e2e: add fencing credentials update and validation tests for dual-replica etcd

Add an e2e test that rotates BMC fencing credentials via the Redfish API,
runs update-fencing-credentials.sh with the new password, and validates
that fencing and cluster recovery still work with the updated credentials.
Asserts PacemakerCluster CR health at each stage (baseline, post-update,
post-recovery) to ensure CEO's view stays consistent with the live
pacemaker state.

Introduces Redfish API helpers, PacemakerCluster CR helpers, and fencing
credentials discovery in the apis package.

Author: fracappa

test/extended/edge_topologies/tnf_recovery.go

@@ -21,6 +21,7 @@ import (
 	exutil "github.com/openshift/origin/test/extended/util"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8srand "k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -414,6 +415,230 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 			memberPromotedVotingTimeout, utils.FiveSecondPollInterval)
 	})
 
+	g.It("should update fencing credentials and validate fencing with updated credentials", func() {
+		bmcNode := targetNode
+		survivedNode := peerNode
+
+		g.By(fmt.Sprintf("Reading current fencing credentials for node %s", bmcNode.Name))
+		creds, err := apis.FindFencingCredentialsByNodeName(oc, bmcNode.Name)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected to find fencing credentials secret")
+		framework.Logf("Found fencing credentials secret %s (address: %s, username: %s)",
+			creds.SecretName, creds.Address, creds.Username)
+
+		g.By("Parsing Redfish address from fencing credentials")
+		redfishHost, redfishPort, redfishPath, err := apis.ParseRedfishAddress(creds.Address)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected to parse Redfish address")
+		framework.Logf("Redfish endpoint: host=%s port=%s path=%s", redfishHost, redfishPort, redfishPath)
+
+		isSushy := apis.IsSushyEmulator(redfishPath)
+		var hypervisorSSH *core.SSHConfig
+		var hypervisorKnownHosts string
+		if isSushy {
+			if !exutil.HasHypervisorConfig() {
+				g.Skip("sushy-tools detected but no hypervisor SSH config available")
+			}
+			sshCfg := exutil.GetHypervisorConfig()
+			o.Expect(sshCfg).ToNot(o.BeNil(), "expected hypervisor config to parse")
+			hypervisorSSH = &core.SSHConfig{
+				IP:             sshCfg.HypervisorIP,
+				User:           sshCfg.SSHUser,
+				PrivateKeyPath: sshCfg.PrivateKeyPath,
+			}
+			var khErr error
+			hypervisorKnownHosts, khErr = core.PrepareLocalKnownHostsFile(hypervisorSSH)
+			o.Expect(khErr).ToNot(o.HaveOccurred(), "expected to prepare hypervisor known_hosts")
+			framework.Logf("Using sushy-tools password change via hypervisor SSH (%s)", hypervisorSSH.IP)
+		}
+
+		changeBMCPassword := func(currentPw, newPw string) error {
+			if isSushy {
+				return apis.ChangeSushyToolsPassword(creds.Username, newPw, hypervisorSSH, hypervisorKnownHosts)
+			}
+			return apis.ChangeBMCPasswordViaRedfish(oc, bmcNode.Name, redfishHost, redfishPort,
+				creds.Username, currentPw, newPw)
+		}
+
+		hasPacemakerCR := apis.IsPacemakerClusterAvailable(oc)
+		if hasPacemakerCR {
+			g.By("Verifying PacemakerCluster CR is healthy before credential change")
+			pc, pcErr := apis.GetPacemakerCluster(oc)
+			o.Expect(pcErr).ToNot(o.HaveOccurred(), "expected to get PacemakerCluster CR")
+			o.Expect(apis.ExpectClusterHealthy(pc)).ToNot(o.HaveOccurred(), "expected PacemakerCluster to be healthy before credential change")
+			o.Expect(apis.ExpectNodeFencingHealthy(pc, bmcNode.Name)).ToNot(o.HaveOccurred(),
+				"expected fencing to be healthy for %s before credential change", bmcNode.Name)
+		} else {
+			framework.Logf("PacemakerCluster CRD not available, skipping CR health checks")
+		}
+
+		sslInsecure := creds.CertificateVerification == "Disabled"
+		originalPassword := creds.Password
+		newPassword := k8srand.String(32)
+		nodeIdentifier := strings.TrimPrefix(creds.SecretName, "fencing-credentials-")
+
+		scriptPath := "/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-scripts/update-fencing-credentials.sh"
+		bashCmd := scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"`
+		if sslInsecure {
+			bashCmd += " --ssl-insecure"
+		}
+
+		// On sushy-tools, a single htpasswd file serves all BMC endpoints, so changing
+		// the password affects all nodes. Fetch the survived node's credentials so we
+		// can update its stonith device and secret in lockstep.
+		var survivedNodeCreds *apis.FencingCredentials
+		var survivedNodeIdentifier string
+		var survivedBashCmd string
+		if isSushy {
+			survivedNodeCreds, err = apis.FindFencingCredentialsByNodeName(oc, survivedNode.Name)
+			o.Expect(err).ToNot(o.HaveOccurred(), "expected to find survived node fencing credentials")
+			survivedNodeIdentifier = strings.TrimPrefix(survivedNodeCreds.SecretName, "fencing-credentials-")
+			survivedBashCmd = scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"`
+			if survivedNodeCreds.CertificateVerification == "Disabled" {
+				survivedBashCmd += " --ssl-insecure"
+			}
+			framework.Logf("sushy-tools: will also update survived node %s credentials (secret: %s)",
+				survivedNode.Name, survivedNodeCreds.SecretName)
+		}
+
+		g.DeferCleanup(func() {
+			var cleanupFailed bool
+
+			framework.Logf("Restoring original BMC password")
+			if restoreErr := changeBMCPassword(newPassword, originalPassword); restoreErr != nil {
+				fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore BMC password: %v\n", restoreErr)
+				cleanupFailed = true
+			}
+
+			scriptPassword := originalPassword
+			if cleanupFailed {
+				scriptPassword = newPassword
+			}
+
+			framework.Logf("Re-running update-fencing-credentials.sh with original credentials")
+			output, restoreErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd",
+				"bash", "-c", bashCmd, "update-fencing-credentials",
+				nodeIdentifier, creds.Username, scriptPassword, creds.Address)
+			if restoreErr != nil {
+				fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore fencing credentials via script: %v\noutput: %s\n",
+					restoreErr, output)
+			}
+
+			if isSushy {
+				framework.Logf("Restoring survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name)
+				survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd",
+					"bash", "-c", survivedBashCmd, "update-fencing-credentials",
+					survivedNodeIdentifier, survivedNodeCreds.Username, scriptPassword, survivedNodeCreds.Address)
+				if survivedErr != nil {
+					fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore survived node fencing credentials: %v\noutput: %s\n",
+						survivedErr, survivedOutput)
+				}
+			}
+		})
+
+		g.By(fmt.Sprintf("Changing BMC password on %s", bmcNode.Name))
+		err = changeBMCPassword(originalPassword, newPassword)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected to change BMC password")
+
+		g.By(fmt.Sprintf("Validating new BMC credentials via fence_redfish on %s", bmcNode.Name))
+		err = apis.ValidateBMCCredentials(oc, bmcNode.Name, redfishHost, redfishPort, redfishPath,
+			creds.Username, newPassword, sslInsecure)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected new BMC credentials to be valid")
+
+		g.By(fmt.Sprintf("Running update-fencing-credentials.sh on %s with new credentials", bmcNode.Name))
+		output, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd",
+			"bash", "-c", bashCmd, "update-fencing-credentials",
+			nodeIdentifier, creds.Username, newPassword, creds.Address)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected update-fencing-credentials.sh to succeed")
+		framework.Logf("update-fencing-credentials.sh output:\n%s", output)
+
+		if isSushy {
+			g.By(fmt.Sprintf("Updating survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name))
+			survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd",
+				"bash", "-c", survivedBashCmd, "update-fencing-credentials",
+				survivedNodeIdentifier, survivedNodeCreds.Username, newPassword, survivedNodeCreds.Address)
+			o.Expect(survivedErr).ToNot(o.HaveOccurred(),
+				"expected update-fencing-credentials.sh for survived node to succeed")
+			framework.Logf("update-fencing-credentials.sh output for survived node:\n%s", survivedOutput)
+		}
+
+		g.By("Validating pacemaker health after credential update")
+		ctx, cancel := context.WithTimeout(context.Background(), nodeIsHealthyTimeout)
+		defer cancel()
+		pcsOutput, err := services.PcsStatusViaDebug(ctx, oc, bmcNode.Name)
+		o.Expect(err).ToNot(o.HaveOccurred(), "expected pcs status to succeed")
+		failedActions := services.ExtractPcsFailedActions(pcsOutput)
+		o.Expect(failedActions).To(o.BeEmpty(), "expected no failed pacemaker resource actions after credential update")
+
+		g.By("Ensuring etcd members remain healthy after fencing credentials update")
+		o.Eventually(func() error {
+			if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, survivedNode.Name); err != nil {
+				return err
+			}
+			if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, bmcNode.Name); err != nil {
+				return err
+			}
+			return nil
+		}, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(),
+			"etcd members should be healthy after fencing credentials update")
+
+		if hasPacemakerCR {
+			g.By("Verifying PacemakerCluster CR remains healthy after credential update")
+			o.Eventually(func() error {
+				pc, pcErr := apis.GetPacemakerCluster(oc)
+				if pcErr != nil {
+					return pcErr
+				}
+				if pcErr = apis.ExpectClusterHealthy(pc); pcErr != nil {
+					return pcErr
+				}
+				return apis.ExpectNodeFencingHealthy(pc, bmcNode.Name)
+			}, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(),
+				"expected PacemakerCluster to remain healthy after credential update")
+		}
+
+		g.By(fmt.Sprintf("Triggering fencing-style network disruption between %s and %s", bmcNode.Name, survivedNode.Name))
+		command, err := exutil.TriggerNetworkDisruption(oc.KubeClient(), &bmcNode, &survivedNode, networkDisruptionDuration)
+		o.Expect(err).To(o.BeNil(), "Expected to disrupt network without errors")
+		framework.Logf("network disruption command: %q", command)
+
+		g.By(fmt.Sprintf("Ensuring cluster recovery after network disruption (timeout: %v)", memberIsLeaderTimeout))
+		leaderNode, learnerNode, learnerStarted := validateEtcdRecoveryStateWithoutAssumingLeader(oc, etcdClientFactory,
+			&survivedNode, &bmcNode, memberIsLeaderTimeout, utils.FiveSecondPollInterval)
+
+		if learnerStarted {
+			framework.Logf("Learner node %q already started as learner after disruption", learnerNode.Name)
+		} else {
+			g.By(fmt.Sprintf("Ensuring '%s' rejoins as learner (timeout: %v)", learnerNode.Name, memberRejoinedLearnerTimeout))
+			validateEtcdRecoveryState(oc, etcdClientFactory,
+				leaderNode,
+				learnerNode, true, true,
+				memberRejoinedLearnerTimeout, utils.FiveSecondPollInterval)
+		}
+
+		g.By(fmt.Sprintf("Ensuring learner node '%s' is promoted back as voting member (timeout: %v)", learnerNode.Name, memberPromotedVotingTimeout))
+		validateEtcdRecoveryState(oc, etcdClientFactory,
+			leaderNode,
+			learnerNode, true, false,
+			memberPromotedVotingTimeout, utils.FiveSecondPollInterval)
+
+		if hasPacemakerCR {
+			g.By("Verifying PacemakerCluster CR is healthy after full recovery")
+			o.Eventually(func() error {
+				pc, pcErr := apis.GetPacemakerCluster(oc)
+				if pcErr != nil {
+					return pcErr
+				}
+				if pcErr = apis.ExpectClusterHealthy(pc); pcErr != nil {
+					return pcErr
+				}
+				if pcErr = apis.ExpectNodeFencingHealthy(pc, bmcNode.Name); pcErr != nil {
+					return pcErr
+				}
+				return apis.ExpectNodeFencingHealthy(pc, survivedNode.Name)
+			}, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(),
+				"expected PacemakerCluster to be fully healthy after recovery")
+		}
+	})
+
 	g.It("should compute etcd revision bump and preserve backup container after kernel panic recovery", func() {
 		// Note: This test triggers a kernel panic on one node via sysrq trigger, then verifies
 		// the surviving node computes the etcd revision bump as floor(maxRaftIndex * 0.2) per

test/extended/edge_topologies/utils/apis/baremetalhost.go

@@ -20,10 +20,72 @@ import (
 )
 
 const (
-	BMCSecretNamespace     = "openshift-machine-api"
-	secretsDataPasswordKey = "password"
+	BMCSecretNamespace          = "openshift-machine-api"
+	FencingCredentialsNamespace = "openshift-etcd"
+	fencingCredentialsPrefix    = "fencing-credentials-"
+	secretsDataPasswordKey      = "password"
 )
 
+// FencingCredentials holds the fields from a fencing-credentials secret in openshift-etcd.
+type FencingCredentials struct {
+	SecretName              string
+	Address                 string
+	Username                string
+	Password                string
+	CertificateVerification string
+}
+
+// FindFencingCredentialsByNodeName discovers the fencing-credentials secret for a node
+// by listing secrets in openshift-etcd and matching against the node's short name.
+func FindFencingCredentialsByNodeName(oc *exutil.CLI, nodeName string) (*FencingCredentials, error) {
+	shortName := strings.Split(nodeName, ".")[0]
+
+	ctx := context.Background()
+	list, err := oc.AdminKubeClient().CoreV1().Secrets(FencingCredentialsNamespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("list secrets in %s: %w", FencingCredentialsNamespace, err)
+	}
+
+	expected := map[string]struct{}{
+		fencingCredentialsPrefix + shortName: {},
+		fencingCredentialsPrefix + nodeName:  {},
+	}
+
+	for _, secret := range list.Items {
+		if _, ok := expected[secret.Name]; ok {
+			getRequired := func(key string) (string, error) {
+				v, exists := secret.Data[key]
+				if !exists || len(v) == 0 {
+					return "", fmt.Errorf("secret %s missing required key %q", secret.Name, key)
+				}
+				return string(v), nil
+			}
+			address, err := getRequired("address")
+			if err != nil {
+				return nil, err
+			}
+			username, err := getRequired("username")
+			if err != nil {
+				return nil, err
+			}
+			password, err := getRequired("password")
+			if err != nil {
+				return nil, err
+			}
+			return &FencingCredentials{
+				SecretName:              secret.Name,
+				Address:                 address,
+				Username:                username,
+				Password:                password,
+				CertificateVerification: string(secret.Data["certificateVerification"]),
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no fencing-credentials secret found matching node %q (prefix: %s, contains: %s) in %s",
+		nodeName, fencingCredentialsPrefix, shortName, FencingCredentialsNamespace)
+}
+
 // BMHGVR is the GroupVersionResource for BareMetalHost (metal3.io/v1alpha1). Use for API-based get/delete/patch.
 var BMHGVR = schema.GroupVersionResource{
 	Group: "metal3.io", Version: "v1alpha1", Resource: "baremetalhosts",

test/extended/edge_topologies/utils/apis/pacemakercluster.go

@@ -0,0 +1,79 @@
+package apis
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	etcdv1alpha1 "github.com/openshift/api/etcd/v1alpha1"
+	exutil "github.com/openshift/origin/test/extended/util"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var PacemakerClusterGVR = schema.GroupVersionResource{
+	Group: etcdv1alpha1.GroupName, Version: "v1alpha1", Resource: "pacemakerclusters",
+}
+
+func IsPacemakerClusterAvailable(oc *exutil.CLI) bool {
+	_, err := oc.AdminDynamicClient().Resource(PacemakerClusterGVR).List(
+		context.Background(), metav1.ListOptions{Limit: 1})
+	return err == nil
+}
+
+func GetPacemakerCluster(oc *exutil.CLI) (*etcdv1alpha1.PacemakerCluster, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	u, err := oc.AdminDynamicClient().Resource(PacemakerClusterGVR).Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("get PacemakerCluster: %w", err)
+	}
+	var pc etcdv1alpha1.PacemakerCluster
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &pc); err != nil {
+		return nil, fmt.Errorf("convert PacemakerCluster: %w", err)
+	}
+	return &pc, nil
+}
+
+func findCondition(conditions []metav1.Condition, condType string) *metav1.Condition {
+	for i := range conditions {
+		if conditions[i].Type == condType {
+			return &conditions[i]
+		}
+	}
+	return nil
+}
+
+func ExpectClusterHealthy(pc *etcdv1alpha1.PacemakerCluster) error {
+	c := findCondition(pc.Status.Conditions, etcdv1alpha1.ClusterHealthyConditionType)
+	if c == nil {
+		return fmt.Errorf("PacemakerCluster missing %s condition", etcdv1alpha1.ClusterHealthyConditionType)
+	}
+	if c.Status != metav1.ConditionTrue {
+		return fmt.Errorf("PacemakerCluster %s=%s (reason: %s, message: %s)",
+			etcdv1alpha1.ClusterHealthyConditionType, c.Status, c.Reason, c.Message)
+	}
+	return nil
+}
+
+func ExpectNodeFencingHealthy(pc *etcdv1alpha1.PacemakerCluster, nodeName string) error {
+	if pc.Status.Nodes == nil {
+		return fmt.Errorf("PacemakerCluster has no nodes in status")
+	}
+	for _, node := range *pc.Status.Nodes {
+		if node.NodeName != nodeName {
+			continue
+		}
+		c := findCondition(node.Conditions, etcdv1alpha1.NodeFencingHealthyConditionType)
+		if c == nil {
+			return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1alpha1.NodeFencingHealthyConditionType)
+		}
+		if c.Status != metav1.ConditionTrue {
+			return fmt.Errorf("node %s %s=%s (reason: %s, message: %s)",
+				nodeName, etcdv1alpha1.NodeFencingHealthyConditionType, c.Status, c.Reason, c.Message)
+		}
+		return nil
+	}
+	return fmt.Errorf("node %s not found in PacemakerCluster status", nodeName)
+}