requeue logic for pending NAD

The operator currently returns a misleading reconcile error when Pod interfaces are not yet ready with assigned IPs.
This change introduces a requeue mechanism to allow time for NetworkAttachmentDefinitions (NAD) to be fully provisioned.

Author: posikoya

OWNERS_ALIASES

@@ -7,10 +7,12 @@ aliases:
   - stuggi
   test-approvers:
   - lpiwowar
+  - adrianfusco
   - arxcruz
   - kstrenkova
   test-reviewers:
   - evallesp
-  - frenzyfriday
-  - sdatko
-  - adrianfusco
+  - rebtoor
+  - Valkyrie00
+  - imatza-rh
+  - posikoya

internal/controller/common.go

@@ -598,17 +598,21 @@ func (r *Reconciler) ReleaseLock(ctx context.Context, instance client.Object) (b
 	return false, ErrFailedToDeleteLock
 }
 
-// PodExists checks if a pod exists for the given instance and workflow step
-func (r *Reconciler) PodExists(ctx context.Context, instance client.Object, workflowStepIndex int) bool {
-	pod := &corev1.Pod{}
+// GetPodIfExists returns the pod for the given instance and workflow step if it exists
+func (r *Reconciler) GetPodIfExists(
+	ctx context.Context,
+	instance client.Object,
+	workflowStepIndex int,
+) (*corev1.Pod, error) {
 	podName := r.GetPodName(instance, workflowStepIndex)
-	objectKey := client.ObjectKey{Namespace: instance.GetNamespace(), Name: podName}
-	err := r.Client.Get(ctx, objectKey, pod)
-	if err != nil && k8s_errors.IsNotFound(err) {
-		return false
+	pod, err := r.GetPod(ctx, podName, instance.GetNamespace())
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
 	}
-
-	return true
+	return pod, nil
 }
 
 // GetCommonRbacRules returns the common RBAC rules for test operations, with optional privileged permissions
@@ -690,8 +694,9 @@ func (r *Reconciler) VerifyNetworkAttachments(
 	conditions *condition.Conditions,
 	networkAttachmentStatus *map[string][]string,
 ) (ctrl.Result, error) {
-	if !r.PodExists(ctx, instance, workflowStepIndex) {
-		return ctrl.Result{}, nil
+	pod, err := r.GetPodIfExists(ctx, instance, workflowStepIndex)
+	if pod == nil {
+		return ctrl.Result{}, err
 	}
 
 	networkReady, status, err := nad.VerifyNetworkStatusFromAnnotation(
@@ -713,14 +718,33 @@ func (r *Reconciler) VerifyNetworkAttachments(
 			condition.NetworkAttachmentsReadyMessage)
 	} else {
 		err := fmt.Errorf("%w: %s", ErrNetworkAttachmentsMismatch, networkAttachments)
+
+		// maxWaitTime to wait for NAD that should escalate to a hard error
+		const maxWaitTime = 5 * time.Minute
+		elaspedTime := time.Since(pod.GetCreationTimestamp().Time)
+		if elaspedTime > maxWaitTime {
+			conditions.Set(condition.FalseCondition(
+				condition.NetworkAttachmentsReadyCondition,
+				condition.ErrorReason,
+				condition.SeverityError,
+				condition.NetworkAttachmentsReadyErrorMessage,
+				fmt.Errorf("timed out waiting for network attachments: %w", err).Error()))
+
+			return ctrl.Result{}, err
+		}
+
 		conditions.Set(condition.FalseCondition(
 			condition.NetworkAttachmentsReadyCondition,
-			condition.ErrorReason,
+			"NetworkAttachmentsWaiting",
 			condition.SeverityWarning,
 			condition.NetworkAttachmentsReadyErrorMessage,
 			err.Error()))
 
-		return ctrl.Result{}, err
+		Log.Info("Waiting for network attachments to become  ready",
+			"elaspedTime", elaspedTime,
+			"maxWaitTime", maxWaitTime)
+
+		return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 	}
 
 	return ctrl.Result{}, nil