require explicit enablement of awaiting csi driver in the untaint controller

Author: adel121

cmd/main.go

@@ -148,6 +148,7 @@ type options struct {
 	datadogGenericResourceEnabled          bool
 	datadogCSIDriverEnabled                bool
 	untaintControllerEnabled               bool
+	untaintControllerWaitForCSIDriver      bool
 
 	// Secret Backend options
 	secretBackendCommand  string
@@ -188,6 +189,8 @@ func (opts *options) Parse() {
 	flag.BoolVar(&opts.datadogGenericResourceEnabled, "datadogGenericResourceEnabled", false, "Enable the DatadogGenericResource controller")
 	flag.BoolVar(&opts.datadogCSIDriverEnabled, "datadogCSIDriverEnabled", false, "Enable the DatadogCSIDriver controller")
 	flag.BoolVar(&opts.untaintControllerEnabled, "untaintControllerEnabled", false, "Enable the Untaint controller")
+	flag.BoolVar(&opts.untaintControllerWaitForCSIDriver, "untaintControllerWaitForCSIDriver", false,
+		"When true (requires --untaintControllerEnabled), the Untaint controller removes the startup taint only after both the node Agent and Datadog CSI node-server pods are Ready. Requires Pod watch coverage of CSI namespaces (DD_CSIDRIVER_WATCH_NAMESPACE).")
 
 	// DatadogAgentInternal
 	flag.BoolVar(&opts.createControllerRevisions, "createControllerRevisions", false, "Enable creation of ControllerRevision snapshots on each DDA spec change")
@@ -235,6 +238,10 @@ func run(opts *options) error {
 	}
 	version.PrintVersionLogs(setupLog)
 
+	if opts.untaintControllerWaitForCSIDriver && !opts.untaintControllerEnabled {
+		return setupErrorf(setupLog, fmt.Errorf("invalid flags"), "--untaintControllerWaitForCSIDriver requires --untaintControllerEnabled=true")
+	}
+
 	// submits the maximum go routine setting as a metric
 	metrics.MaxGoroutines.Set(float64(opts.maximumGoroutines))
 
@@ -287,15 +294,16 @@ func run(opts *options) error {
 		RenewDeadline:              &renewDeadline,
 		RetryPeriod:                &retryPeriod,
 		Cache: config.CacheOptions(setupLog, config.WatchOptions{
-			DatadogAgentEnabled:           opts.datadogAgentEnabled,
-			DatadogMonitorEnabled:         opts.datadogMonitorEnabled,
-			DatadogSLOEnabled:             opts.datadogSLOEnabled,
-			DatadogAgentProfileEnabled:    opts.datadogAgentProfileEnabled,
-			IntrospectionEnabled:          opts.introspectionEnabled,
-			DatadogDashboardEnabled:       opts.datadogDashboardEnabled,
-			DatadogGenericResourceEnabled: opts.datadogGenericResourceEnabled,
-			DatadogCSIDriverEnabled:       opts.datadogCSIDriverEnabled,
-			UntaintControllerEnabled:      opts.untaintControllerEnabled,
+			DatadogAgentEnabled:               opts.datadogAgentEnabled,
+			DatadogMonitorEnabled:             opts.datadogMonitorEnabled,
+			DatadogSLOEnabled:                 opts.datadogSLOEnabled,
+			DatadogAgentProfileEnabled:        opts.datadogAgentProfileEnabled,
+			IntrospectionEnabled:              opts.introspectionEnabled,
+			DatadogDashboardEnabled:           opts.datadogDashboardEnabled,
+			DatadogGenericResourceEnabled:     opts.datadogGenericResourceEnabled,
+			DatadogCSIDriverEnabled:           opts.datadogCSIDriverEnabled,
+			UntaintControllerEnabled:          opts.untaintControllerEnabled,
+			UntaintControllerWaitForCSIDriver: opts.untaintControllerWaitForCSIDriver,
 		}),
 		// UsePriorityQueue makes all controllers use the priority queue, which
 		// directly registers workqueue metrics into controller-runtime's metrics
@@ -366,20 +374,21 @@ func run(opts *options) error {
 			CanaryAutoPauseMaxSlowStartDuration: opts.edsCanaryAutoPauseMaxSlowStartDuration,
 			MaxPodSchedulerFailure:              opts.edsMaxPodSchedulerFailure,
 		},
-		SupportCilium:                 opts.supportCilium,
-		CredsManager:                  credsManager,
-		DatadogAgentEnabled:           opts.datadogAgentEnabled,
-		CreateControllerRevisions:     opts.createControllerRevisions && opts.datadogAgentEnabled,
-		DatadogMonitorEnabled:         opts.datadogMonitorEnabled,
-		DatadogSLOEnabled:             opts.datadogSLOEnabled,
-		OperatorMetricsEnabled:        opts.operatorMetricsEnabled,
-		V2APIEnabled:                  true,
-		IntrospectionEnabled:          opts.introspectionEnabled,
-		DatadogAgentProfileEnabled:    opts.datadogAgentProfileEnabled,
-		DatadogDashboardEnabled:       opts.datadogDashboardEnabled,
-		DatadogGenericResourceEnabled: opts.datadogGenericResourceEnabled,
-		DatadogCSIDriverEnabled:       opts.datadogCSIDriverEnabled,
-		UntaintControllerEnabled:      opts.untaintControllerEnabled,
+		SupportCilium:                     opts.supportCilium,
+		CredsManager:                      credsManager,
+		DatadogAgentEnabled:               opts.datadogAgentEnabled,
+		CreateControllerRevisions:         opts.createControllerRevisions && opts.datadogAgentEnabled,
+		DatadogMonitorEnabled:             opts.datadogMonitorEnabled,
+		DatadogSLOEnabled:                 opts.datadogSLOEnabled,
+		OperatorMetricsEnabled:            opts.operatorMetricsEnabled,
+		V2APIEnabled:                      true,
+		IntrospectionEnabled:              opts.introspectionEnabled,
+		DatadogAgentProfileEnabled:        opts.datadogAgentProfileEnabled,
+		DatadogDashboardEnabled:           opts.datadogDashboardEnabled,
+		DatadogGenericResourceEnabled:     opts.datadogGenericResourceEnabled,
+		DatadogCSIDriverEnabled:           opts.datadogCSIDriverEnabled,
+		UntaintControllerEnabled:          opts.untaintControllerEnabled,
+		UntaintControllerWaitForCSIDriver: opts.untaintControllerWaitForCSIDriver,
 	}
 
 	versionInfo, platformInfo, err := getVersionAndPlatformInfo(rest.CopyConfig(mgr.GetConfig()))

docs/untaint_controller.md

@@ -10,13 +10,13 @@ readiness criteria are met (see below), or after a configurable timeout. It is
 intended to run alongside a separate mechanism (cluster-autoscaler hook, CCM,
 admission webhook, etc.) that adds the taint to new nodes.
 
-**With `--untaintControllerEnabled` only** (or with `--datadogCSIDriverEnabled=false`):
+**With `--untaintControllerEnabled=true` only** (and without `--untaintControllerWaitForCSIDriver`):
 the controller removes the taint once the **node Agent** pod
 (`agent.datadoghq.com/component=agent`) on that node is `Ready`. Agent pods are
 listed in the operator's agent watch namespaces (`WATCH_NAMESPACE` /
 `DD_AGENT_WATCH_NAMESPACE`).
 
-**With both `--untaintControllerEnabled=true` and `--datadogCSIDriverEnabled=true`:**
+**With `--untaintControllerEnabled=true` and `--untaintControllerWaitForCSIDriver=true`:**
 the controller waits until **both** the node Agent and **CSI
 node-server** pod (`app=datadog-csi-driver-node-server`) on the node are
 `Ready` before removing the taint. The taint stays until both are
@@ -25,19 +25,26 @@ satisfied or a timeout fires. The operator's Pod informer then watches the
 pods in those namespaces—keep namespaces tight). Ensure CSI namespaces are
 covered so the controller can list CSI pod status.
 
+**`--datadogCSIDriverEnabled`** only controls whether the **DatadogCSIDriver**
+controller runs; it does **not** by itself turn on dual-readiness untaint.
+Enable `--untaintControllerWaitForCSIDriver` only when you actually deploy CSI
+node-server pods on tainted nodes (for example via a `DatadogCSIDriver` CR with
+the operator's CSI controller enabled, or another install path that produces
+the same pod labels).
+
 If a required pod never reaches Ready on a tainted node, a configurable timeout
 policy ensures the node is never permanently unschedulable. Two clocks cover
 the main failure modes:
 
 - **Readiness timeout** — at least one Agent pod is on the node but the Agent
-  is not Ready yet, **or** (with CSI enabled) the Agent is Ready but a CSI
+  is not Ready yet, **or** (with `--untaintControllerWaitForCSIDriver`) the Agent is Ready but a CSI
   node-server pod exists on the node and is not Ready. Clock: latest
   `pod.Status.StartTime` among **Agent** pods in the first case, and among **CSI
   node-server** pods only in the second (the Agent’s age does not shorten the
   wait for CSI). Pod recreation restarts the window; container restarts inside the
   same pod do not.
-- **Scheduling timeout** — no Agent pod is on the node, **or** (with CSI
-  enabled) the Agent is Ready but **no** CSI node-server pod is on the node
+- **Scheduling timeout** — no Agent pod is on the node, **or** (with wait-for-CSI)
+  the Agent is Ready but **no** CSI node-server pod is on the node
   yet. Clock: `node.metadata.creationTimestamp`. Covers DaemonSets that never
   schedule onto the node (taint not tolerated, missing labels, CSI still pulling,
   etc.).
@@ -62,18 +69,23 @@ manager:
 ```yaml
 args:
   - --untaintControllerEnabled=true
+  # Optional: require CSI node-server Ready before untainting (see Overview).
+  - --untaintControllerWaitForCSIDriver=true
 ```
 
-| `--untaintControllerEnabled` | `--datadogCSIDriverEnabled` | Behavior |
-| ----------------------------- | --------------------------- | -------- |
-| `false` | any | Untaint controller off; no startup toleration injection for this feature on Agent or CSI. |
-| `true` | `false` | Agent-only readiness and Agent DaemonSet toleration (default historical behavior). |
-| `true` | `true` | Wait for Agent **and** CSI node-server Ready; widened Pod cache (agent + `DD_CSIDRIVER_WATCH_NAMESPACE` namespaces); toleration on Agent and CSI DaemonSets. |
+| `--untaintControllerEnabled` | `--untaintControllerWaitForCSIDriver` | Behavior |
+| ----------------------------- | ------------------------------------- | -------- |
+| `false` | any | Untaint controller off; no Agent startup toleration for this feature. |
+| `true` | `false` | Agent-only readiness; Agent DaemonSet startup toleration injected. |
+| `true` | `true` | Wait for Agent **and** CSI node-server Ready; widened Pod cache (agent + `DD_CSIDRIVER_WATCH_NAMESPACE` namespaces); startup toleration on Agent and, when the DatadogCSIDriver controller is enabled, on the CSI node DaemonSet. |
+
+`--untaintControllerWaitForCSIDriver` requires `--untaintControllerEnabled=true` (the operator exits on invalid combinations).
 
-When this flag is enabled, the operator injects a toleration for
+When `--untaintControllerEnabled` is enabled, the operator injects a toleration for
 `agent.datadoghq.com/not-ready=presence:NoSchedule` into the node Agent
 DaemonSet (or ExtendedDaemonSet) pod template, unless an equivalent toleration
-is already present. When **`--datadogCSIDriverEnabled`** is also true, the same
+is already present. When **`--untaintControllerWaitForCSIDriver`** is also true **and**
+the DatadogCSIDriver controller is running (`--datadogCSIDriverEnabled=true`), the same
 toleration is injected into the **Datadog CSI node-server** DaemonSet pod
 template so the CSI workload can schedule on tainted nodes before the taint is
 removed.
@@ -104,7 +116,7 @@ Metrics, under the `untaint` Prometheus subsystem:
 Kubernetes Events (gated by `DD_UNTAINT_CONTROLLER_EVENTS_ENABLED=true`):
 
 - `TaintRemoved` (Normal) — taint removed after the Agent became Ready, or (when
-  the Datadog CSI driver controller is also enabled) after both the Agent and
+  `--untaintControllerWaitForCSIDriver` is enabled) after both the Agent and
   CSI node-server pods became Ready.
 - `UntaintTimeout` — a timeout fired. Normal under `remove`, Warning under `keep`. Message carries the reason, elapsed time, and policy.
 

internal/controller/datadogcsidriver/controller.go

@@ -36,19 +36,19 @@ const (
 
 // Reconciler reconciles a DatadogCSIDriver object
 type Reconciler struct {
-	client                   client.Client
-	scheme                   *runtime.Scheme
-	recorder                 record.EventRecorder
-	untaintControllerEnabled bool
+	client                            client.Client
+	scheme                            *runtime.Scheme
+	recorder                          record.EventRecorder
+	untaintInjectCSIStartupToleration bool
 }
 
 // NewReconciler creates a new DatadogCSIDriver reconciler
-func NewReconciler(client client.Client, scheme *runtime.Scheme, recorder record.EventRecorder, untaintControllerEnabled bool) *Reconciler {
+func NewReconciler(client client.Client, scheme *runtime.Scheme, recorder record.EventRecorder, untaintInjectCSIStartupToleration bool) *Reconciler {
 	return &Reconciler{
-		client:                   client,
-		scheme:                   scheme,
-		recorder:                 recorder,
-		untaintControllerEnabled: untaintControllerEnabled,
+		client:                            client,
+		scheme:                            scheme,
+		recorder:                          recorder,
+		untaintInjectCSIStartupToleration: untaintInjectCSIStartupToleration,
 	}
 }
 
@@ -203,7 +203,7 @@ func (r *Reconciler) reconcileCSIDriver(ctx context.Context, instance *v1alpha1.
 func (r *Reconciler) reconcileDaemonSet(ctx context.Context, instance *v1alpha1.DatadogCSIDriver) error {
 	logger := ctrl.LoggerFrom(ctx)
 	desired := buildDaemonSet(instance)
-	if r.untaintControllerEnabled {
+	if r.untaintInjectCSIStartupToleration {
 		componentagent.EnsureAgentNotReadyStartupToleration(logger, &desired.Spec.Template.Spec)
 	}
 

internal/controller/datadogcsidriver/controller_test.go

@@ -37,7 +37,7 @@ const (
 	testName      = "datadog-csi"
 )
 
-func newTestReconciler(t *testing.T, untaintControllerEnabled bool, objects ...client.Object) (*Reconciler, client.Client) {
+func newTestReconciler(t *testing.T, injectCSIStartupToleration bool, objects ...client.Object) (*Reconciler, client.Client) {
 	t.Helper()
 	s := scheme.Scheme
 	s.AddKnownTypes(v1alpha1.GroupVersion,
@@ -54,7 +54,7 @@ func newTestReconciler(t *testing.T, untaintControllerEnabled bool, objects ...c
 	// Set the default controller-runtime logger so ctrl.LoggerFrom(ctx) works in tests
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 	recorder := record.NewFakeRecorder(10)
-	r := NewReconciler(c, s, recorder, untaintControllerEnabled)
+	r := NewReconciler(c, s, recorder, injectCSIStartupToleration)
 
 	return r, c
 }
@@ -489,7 +489,7 @@ func TestReconcile_CSIDriverSpecDriftIsReconciled(t *testing.T) {
 	assert.Contains(t, csiDriver.Spec.VolumeLifecycleModes, storagev1.VolumeLifecycleEphemeral)
 }
 
-func TestReconcile_DaemonSetIncludesStartupTolerationWhenUntaintEnabled(t *testing.T) {
+func TestReconcile_DaemonSetIncludesStartupTolerationWhenUntaintWaitForCSI(t *testing.T) {
 	instance := defaultCSIDriverCR()
 	r, c := newTestReconciler(t, true, instance)
 	ctx := context.Background()
@@ -508,7 +508,7 @@ func TestReconcile_DaemonSetIncludesStartupTolerationWhenUntaintEnabled(t *testi
 		"expected %+v in %+v", want, ds.Spec.Template.Spec.Tolerations)
 }
 
-func TestReconcile_DaemonSetOmitsStartupTolerationWhenUntaintDisabled(t *testing.T) {
+func TestReconcile_DaemonSetOmitsStartupTolerationWhenUntaintCoordinationOff(t *testing.T) {
 	instance := defaultCSIDriverCR()
 	r, c := newTestReconciler(t, false, instance)
 	ctx := context.Background()

internal/controller/datadogcsidriver_controller.go

@@ -27,11 +27,11 @@ import (
 
 // DatadogCSIDriverReconciler reconciles a DatadogCSIDriver object.
 type DatadogCSIDriverReconciler struct {
-	Client                   client.Client
-	Scheme                   *runtime.Scheme
-	Recorder                 record.EventRecorder
-	UntaintControllerEnabled bool
-	internal                 *datadogcsidriver.Reconciler
+	Client                            client.Client
+	Scheme                            *runtime.Scheme
+	Recorder                          record.EventRecorder
+	UntaintInjectCSIStartupToleration bool
+	internal                          *datadogcsidriver.Reconciler
 }
 
 // RBACs for DatadogCSIDriver objects
@@ -61,7 +61,7 @@ func (r *DatadogCSIDriverReconciler) Reconcile(ctx context.Context, instance *da
 
 // SetupWithManager creates a new DatadogCSIDriver controller.
 func (r *DatadogCSIDriverReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.internal = datadogcsidriver.NewReconciler(r.Client, r.Scheme, r.Recorder, r.UntaintControllerEnabled)
+	r.internal = datadogcsidriver.NewReconciler(r.Client, r.Scheme, r.Recorder, r.UntaintInjectCSIStartupToleration)
 
 	or := reconcile.AsReconciler[*datadoghqv1alpha1.DatadogCSIDriver](r.Client, r)
 	return ctrl.NewControllerManagedBy(mgr).

internal/controller/setup.go

@@ -34,22 +34,23 @@ const (
 
 // SetupOptions defines options for setting up controllers to ease testing
 type SetupOptions struct {
-	SupportExtendedDaemonset      ExtendedDaemonsetOptions
-	SupportCilium                 bool
-	CredsManager                  *config.CredentialManager
-	DatadogAgentEnabled           bool
-	DatadogMonitorEnabled         bool
-	DatadogSLOEnabled             bool
-	OperatorMetricsEnabled        bool
-	V2APIEnabled                  bool
-	IntrospectionEnabled          bool
-	DatadogAgentProfileEnabled    bool
-	OtelAgentEnabled              bool
-	DatadogDashboardEnabled       bool
-	DatadogGenericResourceEnabled bool
-	CreateControllerRevisions     bool
-	DatadogCSIDriverEnabled       bool
-	UntaintControllerEnabled      bool
+	SupportExtendedDaemonset          ExtendedDaemonsetOptions
+	SupportCilium                     bool
+	CredsManager                      *config.CredentialManager
+	DatadogAgentEnabled               bool
+	DatadogMonitorEnabled             bool
+	DatadogSLOEnabled                 bool
+	OperatorMetricsEnabled            bool
+	V2APIEnabled                      bool
+	IntrospectionEnabled              bool
+	DatadogAgentProfileEnabled        bool
+	OtelAgentEnabled                  bool
+	DatadogDashboardEnabled           bool
+	DatadogGenericResourceEnabled     bool
+	CreateControllerRevisions         bool
+	DatadogCSIDriverEnabled           bool
+	UntaintControllerEnabled          bool
+	UntaintControllerWaitForCSIDriver bool
 }
 
 // ExtendedDaemonsetOptions defines ExtendedDaemonset options
@@ -250,7 +251,7 @@ func startUntaint(logger logr.Logger, mgr manager.Manager, _ kubernetes.Platform
 		mgr.GetClient(),
 		ctrl.Log.WithName("controllers").WithName(untaintControllerName),
 		mgr.GetEventRecorderFor(untaintControllerName),
-		options.DatadogCSIDriverEnabled,
+		options.UntaintControllerWaitForCSIDriver,
 	)
 	if err != nil {
 		return fmt.Errorf("untaint controller setup: %w", err)
@@ -279,9 +280,10 @@ func startDatadogCSIDriver(logger logr.Logger, mgr manager.Manager, pInfo kubern
 	}
 
 	return (&DatadogCSIDriverReconciler{
-		Client:                   mgr.GetClient(),
-		Scheme:                   mgr.GetScheme(),
-		Recorder:                 mgr.GetEventRecorderFor(csiDriverControllerName),
-		UntaintControllerEnabled: options.UntaintControllerEnabled,
+		Client:   mgr.GetClient(),
+		Scheme:   mgr.GetScheme(),
+		Recorder: mgr.GetEventRecorderFor(csiDriverControllerName),
+		// Inject startup toleration on CSI node DaemonSet only when untaint coordinates with CSI.
+		UntaintInjectCSIStartupToleration: options.UntaintControllerEnabled && options.UntaintControllerWaitForCSIDriver,
 	}).SetupWithManager(mgr)
 }