sd-agent: add discovery.use_sd_agent config options (#45727)

Yumasi · web-flow · commit 86e0a74d0c7a · 2026-02-09T12:22:05.000Z
### What does this PR do?

This PR adds a "killswitch" for sd-agent to always fallback to system-probe no matter what. Setting `discovery.use_sd_agent` to false, or not setting it all will make it fallback. This ensures there is no accidental usage of sd-agent until we want to.

### Motivation

Control sd-agent &amp; prevent sd-agent accidental usage in the future.

### Describe how you validated your changes

Unit tests. Ran with both Bazel &amp; Cargo.

### Additional Notes


Co-authored-by: guillaume.pagnoux &lt;guillaume.pagnoux@datadoghq.com&gt;
diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go
@@ -347,6 +347,7 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Setup) {
 
 	// Discovery config
 	cfg.BindEnv("discovery.enabled") //nolint:forbidigo // TODO: replace by 'SetDefaultAndBindEnv'
+	cfg.BindEnvAndSetDefault("discovery.use_sd_agent", false)
 	cfg.BindEnvAndSetDefault("discovery.cpu_usage_update_delay", "60s")
 	cfg.BindEnvAndSetDefault("discovery.ignored_command_names", []string{"chronyd", "cilium-agent", "containerd", "dhclient", "dockerd", "kubelet", "livenessprobe", "local-volume-pr", "sshd", "systemd"})
 	cfg.BindEnvAndSetDefault("discovery.service_collection_interval", "60s")
diff --git a/pkg/config/setup/system_probe_test.go b/pkg/config/setup/system_probe_test.go
@@ -55,3 +55,32 @@ func TestSystemProbeDefaultConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestDiscoveryUseSdAgent(t *testing.T) {
+	t.Run("disabled by default", func(t *testing.T) {
+		cfg := newEmptyMockConf(t)
+		InitSystemProbeConfig(cfg)
+		assert.False(t, cfg.GetBool("discovery.use_sd_agent"))
+	})
+
+	t.Run("enabled from env var", func(t *testing.T) {
+		t.Setenv("DD_DISCOVERY_USE_SD_AGENT", "true")
+		cfg := newEmptyMockConf(t)
+		InitSystemProbeConfig(cfg)
+		assert.True(t, cfg.GetBool("discovery.use_sd_agent"))
+	})
+
+	t.Run("disabled from env var", func(t *testing.T) {
+		t.Setenv("DD_DISCOVERY_USE_SD_AGENT", "false")
+		cfg := newEmptyMockConf(t)
+		InitSystemProbeConfig(cfg)
+		assert.False(t, cfg.GetBool("discovery.use_sd_agent"))
+	})
+
+	t.Run("enabled from config", func(t *testing.T) {
+		cfg := newEmptyMockConf(t)
+		InitSystemProbeConfig(cfg)
+		cfg.SetWithoutSource("discovery.use_sd_agent", true)
+		assert.True(t, cfg.GetBool("discovery.use_sd_agent"))
+	})
+}
diff --git a/pkg/discovery/module/impl_linux_test.go b/pkg/discovery/module/impl_linux_test.go
@@ -443,6 +443,7 @@ func TestRustBinary(t *testing.T) {
 	}
 
 	env := os.Environ()
+	env = append(env, "DD_DISCOVERY_USE_SD_AGENT=true")
 	env = append(env, "DD_DISCOVERY_ENABLED=false")
 	// Fake system-probe binary with empty configuration file
 	cmd := exec.Command(binaryPath, "--", truePath, "-c", "/dev/null")
diff --git a/pkg/discovery/module/rust/BUILD.bazel b/pkg/discovery/module/rust/BUILD.bazel
@@ -201,6 +201,7 @@ rust_test(
         "MOCK_SYSTEM_PROBE": "$(rootpath :mock_system_probe)",
     },
     deps = [
+        "@sdagent_crates//:nix",
         "@sdagent_crates//:tempfile",
     ],
 )
diff --git a/pkg/discovery/module/rust/src/config.rs b/pkg/discovery/module/rust/src/config.rs
@@ -242,25 +242,24 @@ pub fn get_log_level(config: &Result<Option<Yaml>>) -> log::Level {
 
 /// Determines whether to run sd-agent, fallback to system-probe, or exit cleanly
 pub fn determine_action(config: &Result<Option<Yaml>>) -> FallbackDecision {
-    if has_non_discovery_env_vars() {
+    let Some(yaml_doc) = config.as_ref().ok() else {
+        warn!("Failed to load YAML config. Falling back to system-probe.");
         return FallbackDecision::FallbackToSystemProbe;
-    }
-
-    // Use the pre-loaded config
-    let yaml_doc = match config {
-        Ok(Some(doc)) => Some(doc.clone()),
-        Ok(None) => None,
-        Err(_) => {
-            warn!("Failed to load YAML config. Falling back to system-probe.");
-            return FallbackDecision::FallbackToSystemProbe;
-        }
     };
-
-    if has_non_discovery_yaml_keys(&yaml_doc) {
+    let use_sd_agent = is_config_enabled(
+        "DD_DISCOVERY_USE_SD_AGENT",
+        "discovery.use_sd_agent",
+        yaml_doc,
+    );
+
+    if use_sd_agent.is_none_or(|enabled| !enabled)
+        || has_non_discovery_env_vars()
+        || has_non_discovery_yaml_keys(yaml_doc)
+    {
         return FallbackDecision::FallbackToSystemProbe;
     }
 
-    if let Some(enabled) = is_config_enabled("DD_DISCOVERY_ENABLED", "discovery.enabled", &yaml_doc)
+    if let Some(enabled) = is_config_enabled("DD_DISCOVERY_ENABLED", "discovery.enabled", yaml_doc)
         && !enabled
     {
         return FallbackDecision::ExitCleanly;
@@ -295,10 +294,16 @@ mod tests {
 
     #[test]
     fn test_discovery_only_no_fallback() {
-        temp_env::with_var("DD_DISCOVERY_ENABLED", Some("true"), || {
-            let decision = determine_action_no_config();
-            assert_eq!(decision, FallbackDecision::RunSdAgent);
-        });
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_ENABLED", Some("true")),
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+            ],
+            || {
+                let decision = determine_action_no_config();
+                assert_eq!(decision, FallbackDecision::RunSdAgent);
+            },
+        );
     }
 
     #[test]
@@ -383,7 +388,7 @@ discovery:
 
     #[test]
     fn test_no_modules_enabled() {
-        temp_env::with_vars(Vec::<(String, Option<String>)>::new(), || {
+        temp_env::with_vars([("DD_DISCOVERY_USE_SD_AGENT", Some("true"))], || {
             // Use an empty config file to avoid picking up system config at /etc/datadog-agent/system-probe.yaml
             let decision = determine_action_no_config();
             assert_eq!(decision, FallbackDecision::RunSdAgent);
@@ -566,7 +571,7 @@ system_probe_config:
 
     #[test]
     fn test_determine_action_discovery_only_yaml() {
-        temp_env::with_vars(Vec::<(String, Option<String>)>::new(), || {
+        temp_env::with_vars([("DD_DISCOVERY_USE_SD_AGENT", Some("true"))], || {
             let yaml = r#"
 discovery:
   enabled: true
@@ -580,15 +585,21 @@ discovery:
 
     #[test]
     fn test_determine_action_discovery_only_env() {
-        temp_env::with_var("DD_DISCOVERY_ENABLED", Some("true"), || {
-            let decision = determine_action_no_config();
-            assert_eq!(decision, FallbackDecision::RunSdAgent);
-        });
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_ENABLED", Some("true")),
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+            ],
+            || {
+                let decision = determine_action_no_config();
+                assert_eq!(decision, FallbackDecision::RunSdAgent);
+            },
+        );
     }
 
     #[test]
     fn test_determine_action_no_config_runs_sd_agent() {
-        temp_env::with_vars(Vec::<(String, Option<String>)>::new(), || {
+        temp_env::with_vars([("DD_DISCOVERY_USE_SD_AGENT", Some("true"))], || {
             let decision = determine_action_no_config();
             assert!(decision == FallbackDecision::RunSdAgent);
         });
@@ -646,10 +657,16 @@ discovery:
 
     #[test]
     fn test_determine_action_exit_cleanly_disabled_env() {
-        temp_env::with_var("DD_DISCOVERY_ENABLED", Some("false"), || {
-            let decision = determine_action_no_config();
-            assert_eq!(decision, FallbackDecision::ExitCleanly);
-        });
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_ENABLED", Some("false")),
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+            ],
+            || {
+                let decision = determine_action_no_config();
+                assert_eq!(decision, FallbackDecision::ExitCleanly);
+            },
+        );
     }
 
     #[test]
@@ -947,4 +964,90 @@ log_level: 12345
             assert_eq!(level, log::Level::Error, "Should map 'off' to Error level");
         });
     }
+
+    // Killswitch tests
+    #[test]
+    fn test_killswitch_disabled_forces_fallback() {
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("false")),
+                ("DD_DISCOVERY_ENABLED", Some("true")),
+            ],
+            || {
+                let decision = determine_action(&Ok(None));
+                assert_eq!(
+                    decision,
+                    FallbackDecision::FallbackToSystemProbe,
+                    "Should fallback when killswitch is false via env var"
+                );
+            },
+        );
+    }
+
+    #[test]
+    fn test_killswitch_not_set_defaults_to_fallback() {
+        temp_env::with_var("DD_DISCOVERY_ENABLED", Some("true"), || {
+            let decision = determine_action(&Ok(None));
+            assert_eq!(
+                decision,
+                FallbackDecision::FallbackToSystemProbe,
+                "Should fallback when killswitch is not set (safe default)"
+            );
+        });
+    }
+
+    #[test]
+    fn test_killswitch_enabled_allows_sd_agent() {
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+                ("DD_DISCOVERY_ENABLED", Some("true")),
+            ],
+            || {
+                let decision = determine_action(&Ok(None));
+                assert_eq!(
+                    decision,
+                    FallbackDecision::RunSdAgent,
+                    "Should run sd-agent when killswitch is true via env var"
+                );
+            },
+        );
+    }
+
+    #[test]
+    fn test_killswitch_enabled_respects_other_fallback_logic() {
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+                ("DD_DISCOVERY_ENABLED", Some("true")),
+                ("DD_NETWORK_CONFIG_ENABLED", Some("true")), // Non-discovery module
+            ],
+            || {
+                let decision = determine_action(&Ok(None));
+                assert_eq!(
+                    decision,
+                    FallbackDecision::FallbackToSystemProbe,
+                    "Should still fallback for non-discovery modules even when killswitch is true"
+                );
+            },
+        );
+    }
+
+    #[test]
+    fn test_killswitch_enabled_with_discovery_disabled_exits_cleanly() {
+        temp_env::with_vars(
+            [
+                ("DD_DISCOVERY_USE_SD_AGENT", Some("true")),
+                ("DD_DISCOVERY_ENABLED", Some("false")),
+            ],
+            || {
+                let decision = determine_action(&Ok(None));
+                assert_eq!(
+                    decision,
+                    FallbackDecision::ExitCleanly,
+                    "Should exit cleanly when discovery is disabled, even if killswitch is true"
+                );
+            },
+        );
+    }
 }
diff --git a/pkg/discovery/module/rust/tests/fallback_integration_test.rs b/pkg/discovery/module/rust/tests/fallback_integration_test.rs
diff --git a/pkg/discovery/module/rust/tests/pid_integration_test.rs b/pkg/discovery/module/rust/tests/pid_integration_test.rs

Original file line number	Diff line number	Diff line change
`@@ -443,6 +443,7 @@ func TestRustBinary(t *testing.T) {`
`443`	`443`	`}`
`444`	`444`
`445`	`445`	`env := os.Environ()`
	`446`	`+ env = append(env, "DD_DISCOVERY_USE_SD_AGENT=true")`
`446`	`447`	`env = append(env, "DD_DISCOVERY_ENABLED=false")`
`447`	`448`	`// Fake system-probe binary with empty configuration file`
`448`	`449`	`cmd := exec.Command(binaryPath, "--", truePath, "-c", "/dev/null")`
Original file line number	Diff line number	Diff line change
`@@ -201,6 +201,7 @@ rust_test(`
`201`	`201`	`"MOCK_SYSTEM_PROBE": "$(rootpath :mock_system_probe)",`
`202`	`202`	`},`
`203`	`203`	`deps = [`
	`204`	`+ "@sdagent_crates//:nix",`
`204`	`205`	`"@sdagent_crates//:tempfile",`
`205`	`206`	`],`
`206`	`207`	`)`