Add GPUClusterConfig controller with singleton status handling

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Author: karthikvetrivel

api/nvidia/v1alpha1/gpuclusterconfig_types.go

@@ -27,6 +27,11 @@ const (
 	GPUClusterConfigCRDName = "GPUClusterConfig"
 )
 
+const (
+	// Ignored marks a duplicate GPUClusterConfig that the singleton controller does not reconcile.
+	Ignored State = "ignored"
+)
+
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 // GPUClusterConfigSpec defines the desired state of GPUClusterConfig, the DRA-based

bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml

@@ -320,6 +320,37 @@ spec:
           path: state
           x-descriptors:
             - 'urn:alm:descriptor:text'
+    - name: gpuclusterconfigs.nvidia.com
+      kind: GPUClusterConfig
+      version: v1alpha1
+      group: nvidia.com
+      displayName: GPUClusterConfig
+      description: GPUClusterConfig allows you to configure the DRA-based GPU software stack
+      resources:
+        - kind: ServiceAccount
+          name: ''
+          version: v1
+        - kind: DaemonSet
+          name: ''
+          version: apps/v1
+        - kind: Deployment
+          name: ''
+          version: apps/v1
+        - kind: ConfigMap
+          name: ''
+          version: v1
+        - kind: Pod
+          name: ''
+          version: v1
+        - kind: status
+          name: ''
+          version: v1
+      statusDescriptors:
+        - description: The current state of the GPUClusterConfig.
+          displayName: State
+          path: state
+          x-descriptors:
+            - 'urn:alm:descriptor:text'
     - name: clusterpolicies.nvidia.com
       kind: ClusterPolicy
       version: v1
@@ -635,6 +666,9 @@ spec:
           - clusterpolicies
           - clusterpolicies/finalizers
           - clusterpolicies/status
+          - gpuclusterconfigs
+          - gpuclusterconfigs/finalizers
+          - gpuclusterconfigs/status
           - nvidiadrivers
           - nvidiadrivers/finalizers
           - nvidiadrivers/status

cmd/gpu-operator/main.go

@@ -210,6 +210,16 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "NVIDIADriver")
 		os.Exit(1)
 	}
+
+	if err = (&controllers.GPUClusterConfigReconciler{
+		Namespace:   operatorNamespace,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		ClusterInfo: clusterInfo,
+	}).SetupWithManager(ctx, mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "GPUClusterConfig")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")

config/rbac/role.yaml

@@ -132,6 +132,7 @@ rules:
   - nvidia.com
   resources:
   - '*'
+  - gpuclusterconfigs
   - nvidiadrivers
   verbs:
   - create
@@ -144,12 +145,14 @@ rules:
 - apiGroups:
   - nvidia.com
   resources:
+  - gpuclusterconfigs/finalizers
   - nvidiadrivers/finalizers
   verbs:
   - update
 - apiGroups:
   - nvidia.com
   resources:
+  - gpuclusterconfigs/status
   - nvidiadrivers/status
   verbs:
   - get

controllers/gpuclusterconfig_controller.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
+	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
+	"github.com/NVIDIA/gpu-operator/internal/conditions"
+	"github.com/NVIDIA/gpu-operator/internal/consts"
+	"github.com/NVIDIA/gpu-operator/internal/state"
+)
+
+// GPUClusterConfigReconciler reconciles a GPUClusterConfig object
+type GPUClusterConfigReconciler struct {
+	client.Client
+	Scheme      *runtime.Scheme
+	ClusterInfo clusterinfo.Interface
+	Namespace   string
+
+	stateManager     state.Manager
+	conditionUpdater conditions.Updater
+
+	// singleton is the GPUClusterConfig that owns reconciliation; the first instance to
+	// reconcile claims it (first-wins), mirroring ClusterPolicy.
+	singleton *nvidiav1alpha1.GPUClusterConfig
+}
+
+//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/finalizers,verbs=update
+
+func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	logger.V(consts.LogLevelInfo).Info("Reconciling GPUClusterConfig")
+
+	instance := &nvidiav1alpha1.GPUClusterConfig{}
+	if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
+		if apierrors.IsNotFound(err) {
+			// Deleted; owned objects are garbage-collected, so there is nothing to clean up.
+			return reconcile.Result{}, nil
+		}
+		wrappedErr := fmt.Errorf("error getting GPUClusterConfig object: %w", err)
+		logger.Error(err, "error getting GPUClusterConfig object")
+		instance.Status.State = nvidiav1alpha1.NotReady
+		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, wrappedErr.Error()); condErr != nil {
+			logger.Error(condErr, "failed to set condition")
+		}
+		return reconcile.Result{}, wrappedErr
+	}
+
+	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
+	// claims ownership; any other instance is marked Ignored and skipped. The owner is
+	// held in memory, so the choice resets on operator restart.
+	if r.singleton != nil && r.singleton.Name != instance.Name {
+		logger.V(consts.LogLevelWarning).Info("Multiple GPUClusterConfig instances found, ignoring this one",
+			"name", instance.Name, "owner", r.singleton.Name)
+		if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+	r.singleton = instance
+
+	infoCatalog := state.NewInfoCatalog()
+	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)
+
+	managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)
+
+	if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if managerStatus.Status != state.SyncStateReady {
+		logger.Info("GPUClusterConfig instance is not ready")
+		var errorInfo error
+		for _, result := range managerStatus.StatesStatus {
+			if result.Status != state.SyncStateReady && result.ErrInfo != nil {
+				errorInfo = result.ErrInfo
+				if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, errorInfo.Error())); condErr != nil {
+					logger.Error(condErr, "failed to set condition")
+				}
+				break
+			}
+		}
+		// if no errors are reported from any state, then we are waiting on operand pods
+		if errorInfo == nil {
+			if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {
+				logger.Error(condErr, "failed to set condition")
+			}
+		}
+		return reconcile.Result{RequeueAfter: time.Second * 5}, nil
+	}
+
+	if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
+		logger.Error(condErr, "failed to set condition")
+		return ctrl.Result{}, condErr
+	}
+	return reconcile.Result{}, nil
+}
+
+// updateCrStatus writes desired to the CR's status, skipping the write when it is already current.
+func (r *GPUClusterConfigReconciler) updateCrStatus(ctx context.Context, cr *nvidiav1alpha1.GPUClusterConfig, desired nvidiav1alpha1.State) error {
+	reqLogger := log.FromContext(ctx)
+
+	// Refetch to avoid a resourceVersion conflict.
+	instance := &nvidiav1alpha1.GPUClusterConfig{}
+	if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
+		reqLogger.Error(err, "Failed to get GPUClusterConfig instance for status update")
+		return err
+	}
+
+	if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {
+		return nil
+	}
+	instance.Status.State = desired
+	instance.Status.Namespace = r.Namespace
+
+	reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
+	if err := r.Status().Update(ctx, instance); err != nil {
+		reqLogger.Error(err, "Failed to update CR status")
+		return err
+	}
+	cr.Status.State = instance.Status.State
+	cr.Status.Namespace = instance.Status.Namespace
+	return nil
+}
+
+// enqueueAllGPUClusterConfigs enqueues every instance so each is reconciled when any
+// instance or owned resource changes.
+func (r *GPUClusterConfigReconciler) enqueueAllGPUClusterConfigs(ctx context.Context) []reconcile.Request {
+	logger := log.FromContext(ctx)
+	list := &nvidiav1alpha1.GPUClusterConfigList{}
+
+	if err := r.List(ctx, list); err != nil {
+		logger.Error(err, "Unable to list GPUClusterConfig resources")
+		return []reconcile.Request{}
+	}
+
+	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
+	for _, config := range list.Items {
+		reconcileRequests = append(reconcileRequests,
+			reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name: config.GetName(),
+				},
+			})
+	}
+
+	return reconcileRequests
+}
+
+func (r *GPUClusterConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
+	// No operands are rendered yet (empty state set).
+	stateManager, err := state.NewManager(
+		nvidiav1alpha1.GPUClusterConfigCRDName,
+		r.Namespace,
+		mgr.GetClient(),
+		mgr.GetScheme())
+	if err != nil {
+		return fmt.Errorf("error creating state manager: %v", err)
+	}
+	r.stateManager = stateManager
+
+	r.conditionUpdater = conditions.NewGPUClusterConfigUpdater(mgr.GetClient())
+
+	c, err := controller.New("gpu-cluster-config-controller", mgr, controller.Options{
+		Reconciler:              r,
+		MaxConcurrentReconciles: 1,
+		RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
+	})
+	if err != nil {
+		return err
+	}
+
+	gpuClusterConfigMapFn := func(ctx context.Context, _ *nvidiav1alpha1.GPUClusterConfig) []reconcile.Request {
+		return r.enqueueAllGPUClusterConfigs(ctx)
+	}
+
+	err = c.Watch(source.Kind(
+		mgr.GetCache(),
+		&nvidiav1alpha1.GPUClusterConfig{},
+		handler.TypedEnqueueRequestsFromMapFunc(gpuClusterConfigMapFn),
+		predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUClusterConfig]{},
+	),
+	)
+	if err != nil {
+		return err
+	}
+
+	// Watch the secondary resources each state manager owns.
+	watchSources := stateManager.GetWatchSources(mgr)
+	for _, watchSource := range watchSources {
+		err = c.Watch(
+			watchSource,
+		)
+		if err != nil {
+			return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
+		}
+	}
+
+	return nil
+}