controller: nrop: switch to vertical per-tree processing

Change reconcileResource from horizontal orchestration (all trees
complete machine config step, then all trees complete daemonset step)
to vertical processing (each tree progresses through all steps
independently). This means a single slow nodegroup no longer blocks
progress for the others.

Move tree normalization from the pre-processing loop in Reconcile
into the per-tree loop in reconcileResource, where it naturally
belongs alongside the rest of per-tree processing.

Add a test verifying that a tree whose MCP is ready creates its
DaemonSet even while another tree's MCP is still pending.

AI-attribution: AIA PAI CeNc Hin R claude-4.6-opus-1M v1.0
Signed-off-by: Francesco Romani <fromani@redhat.com>

Author: ffromani

internal/controller/numaresourcesoperator_controller.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -179,11 +180,6 @@ func (r *NUMAResourcesOperatorReconciler) Reconcile(ctx context.Context, req ctr
 		return r.degradeStatus(ctx, initialInstance, instance, validation.NodeGroupsError, err)
 	}
 
-	for idx := range trees {
-		conf := trees[idx].NodeGroup.NormalizeConfig()
-		trees[idx].NodeGroup.Config = &conf
-	}
-
 	step := r.reconcileResource(ctx, instance, trees)
 	instance.Status.Conditions, _ = status.ComputeConditions(instance.Status.Conditions, step.ConditionInfo.ToMetav1Condition(instance.Generation), time.Now())
 
@@ -259,39 +255,39 @@ func (r *NUMAResourcesOperatorReconciler) reconcileResource(ctx context.Context,
 		}
 	}
 
-	// horizontal processing: all trees complete each step before moving to the next
+	var results []perTreeResult
+	for _, tree := range trees {
+		tree.NodeGroup.Config = ptr.To(tree.NodeGroup.NormalizeConfig())
 
-	// step 1: machine configs for all trees
-	var mcResults []perTreeResult
-	if r.Platform == platform.OpenShift {
-		for _, tree := range trees {
-			treeExisting := existing.PerTree(ctx, r.Client, tree)
-			mcResults = append(mcResults, r.reconcilePerTreeMachineConfig(ctx, instance, treeExisting, tree))
+		treeExisting := existing.PerTree(ctx, r.Client, tree)
+
+		mcResult := perTreeResult{}
+		if r.Platform == platform.OpenShift {
+			mcResult = r.reconcilePerTreeMachineConfig(ctx, instance, treeExisting, tree)
+			if mcResult.step.EarlyStop() {
+				results = append(results, mcResult)
+				continue
+			}
 		}
-	}
-	mcOverall := reducePerTreeResults(mcResults)
-	instance.Status.MachineConfigPools = mcOverall.mcpStatuses
-	instance.Status.Conditions = updateMachineConfigPoolPausedCondition(instance.Status.Conditions, instance.Generation, mcOverall.pausedMCPNames)
-	if mcOverall.step.EarlyStop() {
-		return mcOverall.step
-	}
 
-	// step 2: daemonsets for all trees
-	var dsResults []perTreeResult
-	for _, tree := range trees {
-		treeExisting := existing.PerTree(ctx, r.Client, tree)
-		dsResults = append(dsResults, r.reconcilePerTreeDaemonSet(ctx, instance, treeExisting, tree))
+		dsResult := r.reconcilePerTreeDaemonSet(ctx, instance, treeExisting, tree)
+		dsResult.mcpStatuses = mcResult.mcpStatuses
+		dsResult.pausedMCPNames = mcResult.pausedMCPNames
+		results = append(results, dsResult)
 	}
-	dsOverall := reducePerTreeResults(dsResults)
-	dssReady := collectDaemonSets(dsOverall.dsInfo)
+
+	overall := reducePerTreeResults(results)
+	dssReady := collectDaemonSets(overall.dsInfo)
+	instance.Status.MachineConfigPools = overall.mcpStatuses
+	instance.Status.Conditions = updateMachineConfigPoolPausedCondition(instance.Status.Conditions, instance.Generation, overall.pausedMCPNames)
 	instance.Status.DaemonSets = dssReady
 	instance.Status.RelatedObjects = relatedobjects.ResourceTopologyExporter(r.Namespace, dssReady)
-	if !dsOverall.step.Done() {
-		return dsOverall.step
+
+	if overall.step.Done() {
+		instance.Status.NodeGroups = syncNodeGroupsStatus(instance, overall.dsInfo)
 	}
 
-	instance.Status.NodeGroups = syncNodeGroupsStatus(instance, dsOverall.dsInfo)
-	return intreconcile.StepSuccess()
+	return overall.step
 }
 
 func syncNodeGroupsStatus(instance *nropv1.NUMAResourcesOperator, dsPerPool []poolDaemonSet) []nropv1.NodeGroupStatus {

internal/controller/numaresourcesoperator_controller_test.go

@@ -2268,6 +2268,53 @@ var _ = Describe("Test NUMAResourcesOperator Reconcile", func() {
 				Expect(pausedCond.Status).To(Equal(metav1.ConditionTrue))
 			})
 
+			It("should create DaemonSet for ready tree while another tree is still pending", func(ctx context.Context) {
+				// pool1 has custom policy -> MC created, needs MCO rollout
+				// pool2 has no custom annotation -> default policy (no MC to wait for)
+				nro.Spec.NodeGroups[0].Annotations[annotations.SELinuxPolicyConfigAnnotation] = annotations.SELinuxPolicyCustom
+
+				var err error
+				reconciler, err = NewFakeNUMAResourcesOperatorReconciler(platform.OpenShift, defaultOCPVersion, nro, mcp1, mcp2)
+				Expect(err).ToNot(HaveOccurred())
+
+				key := client.ObjectKeyFromObject(nro)
+
+				By("First reconcile: pool1 MC pending, pool2 ready immediately")
+				Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key})).To(CauseRequeue())
+
+				By("Verify pool2 DaemonSet exists even though pool1 is still pending")
+				ds := &appsv1.DaemonSet{}
+				mcp2DSKey := client.ObjectKey{
+					Name:      objectnames.GetComponentName(nro.Name, mcp2.Name),
+					Namespace: testNamespace,
+				}
+				Expect(reconciler.Client.Get(ctx, mcp2DSKey, ds)).To(Succeed())
+
+				By("Verify global status is Progressing, not Available")
+				Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(nro), nro)).To(Succeed())
+				progressingCond := getConditionByType(nro.Status.Conditions, status.ConditionProgressing)
+				Expect(progressingCond).ToNot(BeNil())
+				Expect(progressingCond.Status).To(Equal(metav1.ConditionTrue))
+
+				By("Make pool1 ready and reconcile again")
+				Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(mcp1), mcp1)).To(Succeed())
+				ensureMCPIsReady(mcp1, nro.Name)
+				Expect(reconciler.Client.Update(ctx, mcp1)).To(Succeed())
+
+				Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key})).ToNot(CauseRequeue())
+
+				By("Now both DaemonSets exist and status is Available")
+				mcp1DSKey := client.ObjectKey{
+					Name:      objectnames.GetComponentName(nro.Name, mcp1.Name),
+					Namespace: testNamespace,
+				}
+				Expect(reconciler.Client.Get(ctx, mcp1DSKey, ds)).To(Succeed())
+				Expect(reconciler.Client.Get(ctx, mcp2DSKey, ds)).To(Succeed())
+
+				Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(nro), nro)).To(Succeed())
+				Expect(nro).To(BeInCondition(status.ConditionAvailable))
+			})
+
 			It("should report paused condition while Progressing", func(ctx context.Context) {
 				// pool1 has custom policy -> MC created, needs MCO rollout
 				// pool2 is paused -> skipped