Add GPUClusterConfig Helm install with ClusterPolicy mutual exclusion

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Author: karthikvetrivel

config/rbac/role.yaml

@@ -142,6 +142,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - nvidia.com
+  resources:
+  - clusterpolicies
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - nvidia.com
   resources:

config/samples/nvidia_v1alpha1_gpuclusterconfig.yaml

@@ -4,9 +4,9 @@ metadata:
   name: gpuclusterconfig-sample
 spec:
   draDriver:
-    repository: nvcr.io/nvidia
-    image: k8s-dra-driver-gpu
-    version: v0.1.0
+    repository: registry.k8s.io/dra-driver-nvidia
+    image: dra-driver-nvidia-gpu
+    version: v0.4.0
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     # NOTE: a driver-validation init container config block (e.g. `validator:`) is

controllers/gpuclusterconfig_controller.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
 	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
 	"github.com/NVIDIA/gpu-operator/internal/conditions"
@@ -59,6 +60,7 @@ type GPUClusterConfigReconciler struct {
 //+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/finalizers,verbs=update
+//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
 
 func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
@@ -79,6 +81,24 @@ func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, wrappedErr
 	}
 
+	// GPUClusterConfig (DRA path) is mutually exclusive with ClusterPolicy: if one
+	// exists, yield to it rather than deploying the DRA stack alongside it.
+	clusterPolicies := &gpuv1.ClusterPolicyList{}
+	if err := r.List(ctx, clusterPolicies); err != nil {
+		return ctrl.Result{}, fmt.Errorf("error listing ClusterPolicies: %w", err)
+	}
+	if len(clusterPolicies.Items) > 0 {
+		logger.V(consts.LogLevelWarning).Info("ClusterPolicy present, skipping mutually exclusive GPUClusterConfig")
+		if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Disabled); err != nil {
+			return ctrl.Result{}, err
+		}
+		msg := "GPUClusterConfig is mutually exclusive with ClusterPolicy; remove the ClusterPolicy or disable GPUClusterConfig"
+		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, msg); condErr != nil {
+			logger.Error(condErr, "failed to set condition")
+		}
+		return ctrl.Result{}, nil
+	}
+
 	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
 	// claims ownership; any other instance is marked Ignored and skipped. The owner is
 	// held in memory, so the choice resets on operator restart.

controllers/gpuclusterconfig_controller_test.go

@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
 	"github.com/NVIDIA/gpu-operator/internal/conditions"
 	"github.com/NVIDIA/gpu-operator/internal/state"
@@ -42,6 +43,7 @@ func newGPUClusterConfigReconciler(t *testing.T, objs ...client.Object) (*GPUClu
 
 	scheme := runtime.NewScheme()
 	require.NoError(t, nvidiav1alpha1.AddToScheme(scheme))
+	require.NoError(t, gpuv1.AddToScheme(scheme))
 
 	c := fake.NewClientBuilder().
 		WithScheme(scheme).
@@ -110,6 +112,18 @@ func TestGPUClusterConfigReconcileNotFound(t *testing.T) {
 	require.Zero(t, res)
 }
 
+// A ClusterPolicy in the cluster disables the GPUClusterConfig: the two paths are
+// mutually exclusive, so the DRA stack is not deployed alongside ClusterPolicy.
+func TestGPUClusterConfigDisabledByClusterPolicy(t *testing.T) {
+	cfg := &nvidiav1alpha1.GPUClusterConfig{ObjectMeta: metav1.ObjectMeta{Name: "config"}}
+	cp := &gpuv1.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "cluster-policy"}}
+	r, c := newGPUClusterConfigReconciler(t, cfg, cp)
+
+	gccReconcile(t, r, cfg.Name)
+
+	require.Equal(t, nvidiav1alpha1.Disabled, gccState(t, c, cfg.Name))
+}
+
 // First-reconciled wins (mirroring ClusterPolicy): whichever instance reconciles first
 // claims ownership, regardless of name or creationTimestamp.
 func TestGPUClusterConfigSingleton(t *testing.T) {

deployments/gpu-operator/templates/cleanup_crd.yaml

@@ -40,6 +40,7 @@ spec:
             - delete
             - --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
             - --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
+            - --filepath=/opt/gpu-operator/nvidia.com_gpuclusterconfigs.yaml
         {{- if .Values.nfd.enabled }}
             - --filepath=/opt/gpu-operator/nfd-api-crds.yaml
         {{- end }}

deployments/gpu-operator/templates/clusterpolicy.yaml

@@ -1,3 +1,5 @@
+{{- /* ClusterPolicy is mutually exclusive with the DRA path; skip it when gpuClusterConfig is enabled. */ -}}
+{{- if not .Values.gpuClusterConfig.enabled }}
 apiVersion: nvidia.com/v1
 kind: ClusterPolicy
 metadata:
@@ -815,3 +817,4 @@ spec:
     {{- if .Values.kataSandboxDevicePlugin.hostNetwork }}
     hostNetwork: {{ .Values.kataSandboxDevicePlugin.hostNetwork }}
     {{- end }}
+{{- end }}

deployments/gpu-operator/templates/clusterrole.yaml

@@ -263,3 +263,16 @@ rules:
   - list
   - watch
   - patch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingadmissionpolicies
+  - validatingadmissionpolicybindings
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete

deployments/gpu-operator/templates/gpuclusterconfig.yaml

@@ -0,0 +1,56 @@
+{{- if .Values.gpuClusterConfig.enabled }}
+{{- $dra := .Values.gpuClusterConfig.draDriver }}
+apiVersion: nvidia.com/v1alpha1
+kind: GPUClusterConfig
+metadata:
+  name: gpu-cluster-config
+  labels:
+    {{- include "gpu-operator.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "gpu-operator"
+  {{- if .Values.operator.cleanupCRD }}
+  # CR cleanup is handled during pre-delete hook
+  # Add below annotation so that helm doesn't attempt to cleanup CR twice
+  annotations:
+    "helm.sh/resource-policy": keep
+  {{- end }}
+spec:
+  draDriver:
+    repository: {{ $dra.repository }}
+    image: {{ $dra.image }}
+    version: {{ $dra.version }}
+    {{- if $dra.imagePullPolicy }}
+    imagePullPolicy: {{ $dra.imagePullPolicy }}
+    {{- end }}
+    {{- with $dra.imagePullSecrets }}
+    imagePullSecrets:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with $dra.featureGates }}
+    featureGates:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    gpus:
+      enabled: {{ $dra.gpus.enabled }}
+      {{- with $dra.gpus.kubeletPlugin }}
+      kubeletPlugin:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    computeDomains:
+      enabled: {{ $dra.computeDomains.enabled }}
+      {{- with $dra.computeDomains.controller }}
+      controller:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $dra.computeDomains.kubeletPlugin }}
+      kubeletPlugin:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  hostPaths:
+    rootFS: {{ .Values.hostPaths.rootFS }}
+    driverInstallDir: {{ .Values.hostPaths.driverInstallDir }}
+    {{- if .Values.hostPaths.kubeletRootDir }}
+    kubeletRootDir: {{ .Values.hostPaths.kubeletRootDir }}
+    {{- end }}
+  daemonsets:
+    {{- toYaml .Values.daemonsets | nindent 4 }}
+{{- end }}

deployments/gpu-operator/values.yaml

@@ -557,6 +557,59 @@ ccManager:
   resources: {}
   hostNetwork: false
 
+# gpuClusterConfig deploys the DRA-based GPU enablement stack (the NVIDIA DRA
+# driver for GPUs) via a GPUClusterConfig CR, as an alternative to the classic
+# device-plugin path. This is an alpha feature (nvidia.com/v1alpha1) and is
+# disabled by default.
+#
+# NOTE: when enabling this, the classic NVIDIA device plugin should be disabled
+# (devicePlugin.enabled=false) to avoid conflicts, and an NVIDIA driver (>= 580)
+# with CDI must be available (host-installed or via driver.enabled / an
+# NVIDIADriver CR). GPUClusterConfig does not manage the driver or device plugin
+# itself; it waits for driver readiness before deploying the DRA driver.
+gpuClusterConfig:
+  enabled: false
+  draDriver:
+    repository: registry.k8s.io/dra-driver-nvidia
+    image: dra-driver-nvidia-gpu
+    version: v0.4.0
+    imagePullPolicy: IfNotPresent
+    imagePullSecrets: []
+    # featureGates toggles DRA driver feature gates; rendered as FEATURE_GATES.
+    # e.g. featureGates: {MPSSupport: true, NVMLDeviceHealthCheck: true}
+    featureGates: {}
+    # gpus configures the gpu.nvidia.com / mig.nvidia.com / vfio.gpu.nvidia.com
+    # capability (the gpus container of the kubelet-plugin DaemonSet).
+    gpus:
+      enabled: true
+      # kubeletPlugin overrides scheduling/resources for the gpus container.
+      # All fields are optional:
+      kubeletPlugin: {}
+      #   env: []                          # list of {name, value}
+      #   resources: {}                    # requests/limits
+      #   healthcheckPort: 51000           # negative value disables the health probe
+      #   priorityClassName: system-node-critical
+      #   nodeSelector: {}
+      #   tolerations: []
+      #   affinity: {}
+    # computeDomains configures the Multi-Node NVLink (MNNVL) compute-domain
+    # capability: a controller Deployment plus the compute-domains kubelet-plugin
+    # container.
+    computeDomains:
+      enabled: true
+      # controller overrides scheduling/resources for the compute-domain
+      # controller Deployment. All fields are optional:
+      controller: {}
+      #   env: []
+      #   resources: {}
+      #   priorityClassName: system-node-critical
+      #   nodeSelector: {}
+      #   tolerations: []
+      #   affinity: {}
+      # kubeletPlugin overrides the compute-domains container (same fields as
+      # gpus.kubeletPlugin above).
+      kubeletPlugin: {}
+
 # Array of extra K8s manifests to deploy
 # Supports use of custom Helm templates
 extraObjects: []

docker/Dockerfile

@@ -121,6 +121,7 @@ COPY validator/manifests /opt/validator/manifests
 # Add CRD resource into the image for helm upgrades
 COPY deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml /opt/gpu-operator/nvidia.com_clusterpolicies.yaml
 COPY deployments/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
+COPY deployments/gpu-operator/crds/nvidia.com_gpuclusterconfigs.yaml /opt/gpu-operator/nvidia.com_gpuclusterconfigs.yaml
 COPY deployments/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml /opt/gpu-operator/nfd-api-crds.yaml
 
 USER 65532:65532