add support for default nvidiadriver and migration from clusterpolicy to nvidiadriver

changes include:
* added default nvidiadriver which doesn't conflict with other user-specified nvidiadrivers
* added migration support from clusterpolicy to nvidiadriver
* added/updated e2e tests for nvidiadriver workflow
* use new field to mark a nvidiadriver as default instead of using a label
* don't allow adding nodeselector to default nvidiadriver, it should cover entire cluster
* move default NVIDIADriver ownership helpers into internal/nvidiadriver and re-use
* requeue after node owner label changes so later reconcile steps read updated cache state
* use patch instead of update for node label updates

Signed-off-by: Rahul Sharma <rahulsharm@nvidia.com>

Author: rahulait

api/nvidia/v1alpha1/nvidiadriver_types.go

@@ -36,11 +36,19 @@ const (
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
-// NVIDIADriverSpec defines the desired state of NVIDIADriver
+// NVIDIADriverSpec defines the desired state of NVIDIADriver.
+// The CEL validation allows non-default drivers to use nodeSelector, but requires
+// default drivers to leave nodeSelector unset or empty.
+// +kubebuilder:validation:XValidation:rule="has(self.default) && self.default ? !has(self.nodeSelector) || size(self.nodeSelector) == 0 : true",message="default NVIDIADriver must not use nodeSelector"
 type NVIDIADriverSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
+	// Default indicates that this NVIDIADriver acts as the fallback driver for GPU nodes
+	// that do not match any non-default NVIDIADriver nodeSelector.
+	// +kubebuilder:default=false
+	Default bool `json:"default"`
+
 	// +kubebuilder:validation:Enum=gpu;vgpu;vgpu-host-manager
 	// +kubebuilder:default=gpu
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="driverType is an immutable field. Please create a new NvidiaDriver resource instead when you want to change this setting."
@@ -504,6 +512,7 @@ type NVIDIADriverStatus struct {
 //+kubebuilder:subresource:status
 //+kubebuilder:resource:scope=Cluster,shortName={"nvd","nvdriver","nvdrivers"}
 //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0
+//+kubebuilder:printcolumn:name="Default",type=boolean,JSONPath=`.spec.default`,priority=0
 //+kubebuilder:printcolumn:name="Age",type=string,JSONPath=`.metadata.creationTimestamp`,priority=0
 
 // NVIDIADriver is the Schema for the nvidiadrivers API

bundle/manifests/nvidia.com_nvidiadrivers.yaml

@@ -22,6 +22,9 @@ spec:
     - jsonPath: .status.state
       name: Status
       type: string
+    - jsonPath: .spec.default
+      name: Default
+      type: boolean
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: string
@@ -48,7 +51,10 @@ spec:
           metadata:
             type: object
           spec:
-            description: NVIDIADriverSpec defines the desired state of NVIDIADriver
+            description: |-
+              NVIDIADriverSpec defines the desired state of NVIDIADriver.
+              The CEL validation allows non-default drivers to use nodeSelector, but requires
+              default drivers to leave nodeSelector unset or empty.
             properties:
               annotations:
                 additionalProperties:
@@ -70,6 +76,12 @@ spec:
                   name:
                     type: string
                 type: object
+              default:
+                default: false
+                description: |-
+                  Default indicates that this NVIDIADriver acts as the fallback driver for GPU nodes
+                  that do not match any non-default NVIDIADriver nodeSelector.
+                type: boolean
               driverType:
                 default: gpu
                 description: DriverType defines NVIDIA driver type
@@ -968,9 +980,14 @@ spec:
                     type: string
                 type: object
             required:
+            - default
             - driverType
             - image
             type: object
+            x-kubernetes-validations:
+            - message: default NVIDIADriver must not use nodeSelector
+              rule: 'has(self.default) && self.default ? !has(self.nodeSelector) ||
+                size(self.nodeSelector) == 0 : true'
           status:
             description: NVIDIADriverStatus defines the observed state of NVIDIADriver
             properties:

config/crd/bases/nvidia.com_nvidiadrivers.yaml

@@ -22,6 +22,9 @@ spec:
     - jsonPath: .status.state
       name: Status
       type: string
+    - jsonPath: .spec.default
+      name: Default
+      type: boolean
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: string
@@ -48,7 +51,10 @@ spec:
           metadata:
             type: object
           spec:
-            description: NVIDIADriverSpec defines the desired state of NVIDIADriver
+            description: |-
+              NVIDIADriverSpec defines the desired state of NVIDIADriver.
+              The CEL validation allows non-default drivers to use nodeSelector, but requires
+              default drivers to leave nodeSelector unset or empty.
             properties:
               annotations:
                 additionalProperties:
@@ -70,6 +76,12 @@ spec:
                   name:
                     type: string
                 type: object
+              default:
+                default: false
+                description: |-
+                  Default indicates that this NVIDIADriver acts as the fallback driver for GPU nodes
+                  that do not match any non-default NVIDIADriver nodeSelector.
+                type: boolean
               driverType:
                 default: gpu
                 description: DriverType defines NVIDIA driver type
@@ -968,9 +980,14 @@ spec:
                     type: string
                 type: object
             required:
+            - default
             - driverType
             - image
             type: object
+            x-kubernetes-validations:
+            - message: default NVIDIADriver must not use nodeSelector
+              rule: 'has(self.default) && self.default ? !has(self.nodeSelector) ||
+                size(self.nodeSelector) == 0 : true'
           status:
             description: NVIDIADriverStatus defines the observed state of NVIDIADriver
             properties:

controllers/clusterpolicy_controller.go

@@ -42,7 +42,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
+	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
 	"github.com/NVIDIA/gpu-operator/internal/conditions"
+	nvidiadriverutil "github.com/NVIDIA/gpu-operator/internal/nvidiadriver"
 )
 
 const (
@@ -143,6 +145,24 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	}
 
 	clusterPolicyCtrl.operatorMetrics.reconciliationTotal.Inc()
+
+	nvidiaDriverEnabled := nvidiadriverutil.CRDEnabled(instance)
+	if nvidiaDriverEnabled {
+		changed, err := nvidiadriverutil.AssignOwners(ctx, r.Client)
+		if err != nil {
+			clusterPolicyCtrl.operatorMetrics.reconciliationStatus.Set(reconciliationStatusNotReady)
+			clusterPolicyCtrl.operatorMetrics.reconciliationFailed.Inc()
+			updateCRState(ctx, r, req.NamespacedName, gpuv1.NotReady)
+			if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
+				r.Log.Error(condErr, "failed to set condition")
+			}
+			return ctrl.Result{}, err
+		}
+		if changed {
+			return ctrl.Result{RequeueAfter: time.Second}, nil
+		}
+	}
+
 	overallStatus := gpuv1.Ready
 	statesNotReady := []string{}
 	for {
@@ -170,6 +190,12 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		}
 	}
 
+	if nvidiaDriverEnabled {
+		if err := clusterPolicyCtrl.labelNodesWithOrphanedDriverPods(ctx); err != nil {
+			r.Log.Error(err, "failed to label nodes with orphaned NVIDIA driver pods")
+		}
+	}
+
 	// if any state is not ready, requeue for reconcile after 5 seconds
 	if overallStatus != gpuv1.Ready {
 		clusterPolicyCtrl.operatorMetrics.reconciliationStatus.Set(reconciliationStatusNotReady)
@@ -371,6 +397,29 @@ func (r *ClusterPolicyReconciler) SetupWithManager(ctx context.Context, mgr ctrl
 		return err
 	}
 
+	err = c.Watch(
+		source.Kind(mgr.GetCache(),
+			&nvidiav1alpha1.NVIDIADriver{},
+			handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ *nvidiav1alpha1.NVIDIADriver) []reconcile.Request {
+				list := &gpuv1.ClusterPolicyList{}
+				if err := mgr.GetClient().List(ctx, list); err != nil {
+					return []reconcile.Request{}
+				}
+				requests := make([]reconcile.Request, 0, len(list.Items))
+				for _, item := range list.Items {
+					requests = append(requests, reconcile.Request{
+						NamespacedName: types.NamespacedName{Name: item.GetName()},
+					})
+				}
+				return requests
+			}),
+			nvidiaDriverGenerationChangedPredicate(),
+		),
+	)
+	if err != nil {
+		return err
+	}
+
 	// TODO(user): Modify this to be the types you create that are owned by the primary resource
 	// Watch for changes to secondary resource Daemonsets and requeue the owner ClusterPolicy
 	err = c.Watch(

controllers/nvidiadriver_controller.go

@@ -45,6 +45,7 @@ import (
 	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
 	"github.com/NVIDIA/gpu-operator/internal/conditions"
 	"github.com/NVIDIA/gpu-operator/internal/consts"
+	nvidiadriverutil "github.com/NVIDIA/gpu-operator/internal/nvidiadriver"
 	"github.com/NVIDIA/gpu-operator/internal/state"
 	"github.com/NVIDIA/gpu-operator/internal/validator"
 )
@@ -96,6 +97,10 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		// Error reading the object - requeue the request.
 		return reconcile.Result{}, wrappedErr
 	}
+	if !instance.GetDeletionTimestamp().IsZero() {
+		logger.Info("NVIDIADriver delete requested; skipping reconciliation")
+		return reconcile.Result{}, nil
+	}
 
 	// Get the singleton NVIDIA ClusterPolicy object in the cluster.
 	clusterPolicyList := &gpuv1.ClusterPolicyList{}
@@ -121,7 +126,7 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	clusterPolicyInstance := clusterPolicyList.Items[0]
 
 	// Ensure that ClusterPolicy is configured to use NVIDIADriver CRD
-	if !clusterPolicyInstance.Spec.Driver.UseNvidiaDriverCRDType() {
+	if !nvidiadriverutil.CRDEnabled(&clusterPolicyInstance) {
 		msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy"
 		logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)
 		instance.Status.State = nvidiav1alpha1.Disabled
@@ -152,6 +157,19 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return reconcile.Result{}, nil
 	}
 
+	changed, err := nvidiadriverutil.AssignOwners(ctx, r.Client)
+	if err != nil {
+		logger.Error(err, "failed to assign NVIDIADriver owners to nodes")
+		instance.Status.State = nvidiav1alpha1.NotReady
+		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
+			logger.Error(condErr, "failed to set condition")
+		}
+		return reconcile.Result{}, err
+	}
+	if changed {
+		return reconcile.Result{RequeueAfter: time.Second}, nil
+	}
+
 	if instance.Spec.UsePrecompiledDrivers() && (instance.Spec.IsGDSEnabled() || instance.Spec.IsGDRCopyEnabled()) {
 		err := errors.New("GPUDirect Storage driver (nvidia-fs) and/or GDRCopy driver is not supported along with pre-compiled NVIDIA drivers")
 		logger.Error(err, "unsupported driver combination detected")
@@ -316,7 +334,7 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 		mgr.GetCache(),
 		&nvidiav1alpha1.NVIDIADriver{},
 		handler.TypedEnqueueRequestsFromMapFunc(nvidiaDriverMapFn),
-		predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.NVIDIADriver]{},
+		nvidiaDriverGenerationChangedPredicate(),
 	),
 	)
 	if err != nil {
@@ -413,3 +431,27 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 
 	return nil
 }
+
+// nvidiaDriverGenerationChangedPredicate reconciles NVIDIADrivers on create, delete, and spec changes.
+func nvidiaDriverGenerationChangedPredicate() predicate.TypedPredicate[*nvidiav1alpha1.NVIDIADriver] {
+	return predicate.TypedFuncs[*nvidiav1alpha1.NVIDIADriver]{
+		CreateFunc: func(event.TypedCreateEvent[*nvidiav1alpha1.NVIDIADriver]) bool {
+			return true
+		},
+		DeleteFunc: func(event.TypedDeleteEvent[*nvidiav1alpha1.NVIDIADriver]) bool {
+			return true
+		},
+		UpdateFunc: func(e event.TypedUpdateEvent[*nvidiav1alpha1.NVIDIADriver]) bool {
+			if e.ObjectOld == nil || e.ObjectNew == nil {
+				return true
+			}
+			if e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() {
+				return true
+			}
+			return false
+		},
+		GenericFunc: func(event.TypedGenericEvent[*nvidiav1alpha1.NVIDIADriver]) bool {
+			return false
+		},
+	}
+}