fix(etcd-recovery): OCPBUGS-84577: handle transient errors when checking etcd statefulset health

Return error instead of silently falling through to the failure
condition when r.Get() for the etcd StatefulSet fails with a
non-NotFound error. This prevents transient API errors from
incorrectly marking a healthy cluster as needing manual
intervention.

Signed-off-by: Vimal Solanki <vsolanki@redhat.com>

Author: vsolanki12

hypershift-operator/controllers/hostedcluster/etcd_recovery.go

@@ -58,23 +58,24 @@ func (r *HostedClusterReconciler) reconcileETCDMemberRecovery(ctx context.Contex
 		if !jobStatus.successful {
 			// Check if etcd has recovered despite the job failing
 			etcdStatefulSet := etcdrecoverymanifests.EtcdStatefulSet(hcpNS)
-			if err := r.Get(ctx, crclient.ObjectKeyFromObject(etcdStatefulSet), etcdStatefulSet); err == nil {
-				if etcdStatefulSet.Status.ReadyReplicas == 3 && etcdStatefulSet.Status.AvailableReplicas == 3 {
-					log.Info("etcd recovered despite failed recovery job, cleaning up")
-					if err := r.cleanupEtcdRecoveryObjects(ctx, hcluster); err != nil {
-						return nil, fmt.Errorf("failed to cleanup etcd recovery job: %w", err)
-					}
-					etcdRecoveryActiveCondition.Status = metav1.ConditionFalse
-					etcdRecoveryActiveCondition.Reason = hyperv1.AsExpectedReason
-					etcdRecoveryActiveCondition.LastTransitionTime = r.now()
-					meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
-					if err := r.Client.Status().Update(ctx, hcluster); err != nil {
-						return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
-					}
-					return nil, nil
+			if err := r.Get(ctx, crclient.ObjectKeyFromObject(etcdStatefulSet), etcdStatefulSet); err != nil {
+				if !apierrors.IsNotFound(err) {
+					return nil, fmt.Errorf("failed to get etcd statefulset: %w", err)
 				}
+			} else if etcdStatefulSet.Status.ReadyReplicas == 3 && etcdStatefulSet.Status.AvailableReplicas == 3 {
+				log.Info("etcd recovered despite failed recovery job, cleaning up")
+				if err := r.cleanupEtcdRecoveryObjects(ctx, hcluster); err != nil {
+					return nil, fmt.Errorf("failed to cleanup etcd recovery job: %w", err)
+				}
+				etcdRecoveryActiveCondition.Status = metav1.ConditionFalse
+				etcdRecoveryActiveCondition.Reason = hyperv1.AsExpectedReason
+				etcdRecoveryActiveCondition.LastTransitionTime = r.now()
+				meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
+				if err := r.Client.Status().Update(ctx, hcluster); err != nil {
+					return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
+				}
+				return nil, nil
 			}
-
 			etcdRecoveryActiveCondition.Status = metav1.ConditionFalse
 			etcdRecoveryActiveCondition.Reason = hyperv1.EtcdRecoveryJobFailedReason
 			etcdRecoveryActiveCondition.Message = "Error in Etcd Recovery job: the Etcd cluster requires manual intervention."