test(ffe): cover billed source precedence

Author: leoromanovsky

tests/parametric/test_dynamic_configuration.py

@@ -41,6 +41,8 @@
     # Decrease the heartbeat/poll intervals to speed up the tests
     "DD_TELEMETRY_HEARTBEAT_INTERVAL": "0.2",
     "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2",
+    # This suite validates RC capabilities; Feature Flagging RC is explicit opt-in.
+    "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config",
     # Disable CSS which is enabled by default on Go
     "DD_TRACE_STATS_COMPUTATION_ENABLED": "false",
 }

tests/parametric/test_ffe/test_configuration_sources.py

@@ -1,8 +1,9 @@
 """Parametric FFE configuration-source coverage for a mocked FFE agentless backend.
 
-Feature under test: server SDKs can select where UFC flag definitions come from.
-The agentless path fetches from an HTTP backend, while explicit ``remote_config``
-keeps the existing Agent RC path.
+Feature under test: server SDKs preserve legacy Remote Configuration adopters,
+load UFC flag definitions from the agentless HTTP backend by default only after
+application provider access, and honor explicit source selection and the stable
+provider kill switch.
 
 Test strategy: drive SDKs through public configuration-source env vars and
 OpenFeature evaluation endpoints, then use the mock FFE agentless backend for
@@ -22,7 +23,7 @@
 from tests.parametric.conftest import APMLibrary
 from tests.parametric.test_ffe.test_dynamic_evaluation import _set_and_wait_ffe_rc, _ffe_evaluate_with_rc_retry
 from utils import features, scenarios
-from utils.dd_constants import RemoteConfigApplyState
+from utils.dd_constants import Capabilities, RemoteConfigApplyState
 from utils.docker_fixtures import TestAgentAPI
 from utils.docker_fixtures._mock_ffe_agentless_backend import (
     CONFIG_PATH,
@@ -42,13 +43,12 @@
 AGENTLESS_BASE_URL = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"
 
 BASE_ENVVARS = {
-    "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true",
-    "DD_TELEMETRY_HEARTBEAT_INTERVAL": "0.2",
+    "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "false",
     "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2",
 }
 
 AGENTLESS_ENVVARS = {
-    "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2",
+    "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "1",
     "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "1",
 }
 
@@ -87,6 +87,11 @@ def library_env(
     responses = params.get("responses")
     api_key = params.get("api_key", TEST_API_KEY)
 
+    if "provider_enabled" in params:
+        env["DD_FEATURE_FLAGS_ENABLED"] = str(params["provider_enabled"]).lower()
+    if "legacy_provider_enabled" in params:
+        env["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] = str(params["legacy_provider_enabled"]).lower()
+
     if responses is not None:
         mock_ffe_agentless_backend.set_responses(responses)
     elif response is not None:
@@ -141,6 +146,27 @@ def _assert_no_mock_requests(mock_ffe_agentless_backend: MockFFEAgentlessBackend
         time.sleep(MOCK_STATUS_INTERVAL_SECONDS)
 
 
+def _remote_config_products(test_agent: TestAgentAPI) -> set[str]:
+    products: set[str] = set()
+    for request in test_agent.rc_requests(post_only=True):
+        client = request["body"].get("client", {})
+        products.update(client.get("products", []))
+    return products
+
+
+def _assert_ffe_remote_config_activation(test_agent: TestAgentAPI) -> None:
+    test_agent.assert_rc_capabilities({Capabilities.FFE_FLAG_CONFIGURATION_RULES})
+    assert RC_PRODUCT in _remote_config_products(test_agent)
+
+
+def _assert_no_ffe_remote_config_activation(test_agent: TestAgentAPI) -> None:
+    for _ in range(NO_MOCK_REQUEST_ATTEMPTS):
+        capabilities = test_agent.wait_for_rc_capabilities()
+        assert Capabilities.FFE_FLAG_CONFIGURATION_RULES not in capabilities
+        assert RC_PRODUCT not in _remote_config_products(test_agent)
+        time.sleep(MOCK_STATUS_INTERVAL_SECONDS)
+
+
 def _evaluate(test_library: APMLibrary) -> dict[str, Any]:
     return _ffe_evaluate_with_rc_retry(
         test_library,
@@ -199,25 +225,51 @@ def test_remote_config_positive_ignores_agentless_env(
         apply_state = _set_and_wait_ffe_rc(test_agent, UFC_VALID_DATA)
         assert apply_state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value
         assert apply_state["product"] == RC_PRODUCT
+        _assert_ffe_remote_config_activation(test_agent)
 
         assert test_library.ffe_start(), "failed to start FFE provider in remote_config mode"
         _assert_expected_value(_evaluate(test_library))
 
         _assert_no_mock_requests(mock_ffe_agentless_backend)
 
+    @parametrize(
+        "library_env",
+        [{"configuration_source": "remote_config", "provider_enabled": False, "response": "valid"}],
+        indirect=True,
+    )
+    def test_provider_kill_switch_stops_remote_config_subscription(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
+    ) -> None:
+        test_library.ffe_start()
+        _assert_no_ffe_remote_config_activation(test_agent)
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+
     @parametrize("library_env", [{"configuration_source": "remote_config", "response": "valid"}], indirect=True)
     def test_remote_config_without_rc_does_not_fallback_to_agentless(
-        self, test_library: APMLibrary, mock_ffe_agentless_backend: MockFFEAgentlessBackendServer
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
     ) -> None:
-        _assert_cold_not_ready(test_library, started=test_library.ffe_start())
+        del test_library  # fixture starts the tracer; no RC payload is delivered
 
+        _assert_ffe_remote_config_activation(test_agent)
         _assert_no_mock_requests(mock_ffe_agentless_backend)
 
-    @parametrize("library_env", [{"configuration_source": "agentless", "response": "valid"}], indirect=True)
-    def test_explicit_agentless_positive(
-        self, test_library: APMLibrary, mock_ffe_agentless_backend: MockFFEAgentlessBackendServer
+    @parametrize("library_env", [{"configuration_source": None, "response": "valid"}], indirect=True)
+    def test_default_agentless_positive(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
     ) -> None:
-        assert test_library.ffe_start(), "failed to start FFE provider in explicit agentless mode"
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+        _assert_no_ffe_remote_config_activation(test_agent)
+
+        assert test_library.ffe_start(), "failed to start FFE provider in default agentless mode"
         _assert_expected_value(_evaluate(test_library))
 
         status = _wait_for_status(
@@ -227,29 +279,111 @@ def test_explicit_agentless_positive(
         )
         assert status["last_auth_present"] is True
         assert status["last_path"] == CONFIG_PATH
+        _assert_no_ffe_remote_config_activation(test_agent)
 
-    @parametrize("library_env", [{"configuration_source": None, "response": "valid"}], indirect=True)
-    def test_default_agentless_positive(
-        self, test_library: APMLibrary, mock_ffe_agentless_backend: MockFFEAgentlessBackendServer
+    @parametrize(
+        "library_env",
+        [{"configuration_source": None, "legacy_provider_enabled": True, "response": "valid"}],
+        indirect=True,
+    )
+    def test_legacy_true_preserves_remote_config(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
     ) -> None:
-        assert test_library.ffe_start(), "failed to start FFE provider in default agentless mode"
+        apply_state = _set_and_wait_ffe_rc(test_agent, UFC_VALID_DATA)
+        assert apply_state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value
+        assert apply_state["product"] == RC_PRODUCT
+        _assert_ffe_remote_config_activation(test_agent)
+
+        assert test_library.ffe_start(), "failed to start grandfathered Remote Config provider"
         _assert_expected_value(_evaluate(test_library))
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
 
-        status = _wait_for_status(
+    @parametrize(
+        "library_env",
+        [{"configuration_source": None, "legacy_provider_enabled": False, "response": "valid"}],
+        indirect=True,
+    )
+    def test_legacy_false_keeps_provider_disabled(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
+    ) -> None:
+        test_library.ffe_start()
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+        _assert_no_ffe_remote_config_activation(test_agent)
+
+    @parametrize(
+        "library_env",
+        [{"configuration_source": "agentless", "legacy_provider_enabled": True, "response": "valid"}],
+        indirect=True,
+    )
+    def test_explicit_agentless_wins_over_legacy_true(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
+    ) -> None:
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+        _assert_no_ffe_remote_config_activation(test_agent)
+
+        assert test_library.ffe_start(), "failed to start explicit agentless provider"
+        _assert_expected_value(_evaluate(test_library))
+        _wait_for_status(
             mock_ffe_agentless_backend,
             lambda current: current["requests_total"] > 0 and current["last_status_code"] == 200,
-            "default agentless request",
+            "explicit agentless response request",
         )
-        assert status["last_path"] == CONFIG_PATH
+        _assert_no_ffe_remote_config_activation(test_agent)
+
+    @parametrize(
+        "library_env",
+        [{"configuration_source": "remote_config", "legacy_provider_enabled": False, "response": "valid"}],
+        indirect=True,
+    )
+    def test_explicit_remote_config_wins_over_legacy_false(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
+    ) -> None:
+        apply_state = _set_and_wait_ffe_rc(test_agent, UFC_VALID_DATA)
+        assert apply_state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value
+        assert apply_state["product"] == RC_PRODUCT
+        _assert_ffe_remote_config_activation(test_agent)
+
+        assert test_library.ffe_start(), "failed to start explicit Remote Config provider"
+        _assert_expected_value(_evaluate(test_library))
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+
+    @parametrize(
+        "library_env",
+        [{"configuration_source": None, "provider_enabled": False, "response": "valid"}],
+        indirect=True,
+    )
+    def test_provider_kill_switch_stops_agentless_polling(
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
+    ) -> None:
+        test_library.ffe_start()
+        _assert_no_mock_requests(mock_ffe_agentless_backend)
+        _assert_no_ffe_remote_config_activation(test_agent)
 
     @parametrize("library_env", [{"configuration_source": "invalid", "response": "valid"}], indirect=True)
     def test_invalid_configuration_source_fails_closed(
-        self, test_library: APMLibrary, mock_ffe_agentless_backend: MockFFEAgentlessBackendServer
+        self,
+        test_agent: TestAgentAPI,
+        test_library: APMLibrary,
+        mock_ffe_agentless_backend: MockFFEAgentlessBackendServer,
     ) -> None:
-        started = test_library.ffe_start()
-        if started:
-            _assert_default_or_not_ready(_evaluate(test_library))
+        test_library.ffe_start()
         _assert_no_mock_requests(mock_ffe_agentless_backend)
+        _assert_no_ffe_remote_config_activation(test_agent)
 
 
 @scenarios.parametric

tests/test_the_test/test_mock_ffe_agentless_backend.py

@@ -34,6 +34,7 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id
             timeout=5,
         )
         response.raise_for_status()
+        assert response.headers["Content-Length"] == str(len(response.content))
 
         payload = response.json()
         assert payload["data"]["type"] == UFC_RESPONSE_TYPE

utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java

@@ -28,6 +28,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 @RestController
 @RequestMapping("/ffe")
@@ -42,10 +43,14 @@ public static class FeatureFlagEvaluatorConfig {
         @Bean
         public Client client() {
             final OpenFeatureAPI api = OpenFeatureAPI.getInstance();
-            final String envProperty = System.getenv("DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED");
+            final boolean featureFlaggingConfigured =
+                    System.getenv("DD_FEATURE_FLAGS_ENABLED") != null
+                            || System.getenv("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE") != null
+                            || System.getenv("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL") != null
+                            || Boolean.parseBoolean(System.getenv("DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"));
             final FeatureProvider provider;
-            if (Boolean.parseBoolean(envProperty)) {
-                provider = new Provider();
+            if (featureFlaggingConfigured) {
+                provider = new Provider(new Provider.Options().initTimeout(100, TimeUnit.MILLISECONDS));
             } else {
                 provider = new NoOpProvider() {
                     @Override

utils/build/docker/nodejs/parametric/server.js

@@ -509,8 +509,17 @@ app.post("/trace/otel/otel_set_baggage", (req, res) => {
 
 // Feature Flag & Experimentation endpoints
 app.post('/ffe/start', async (req, res) => {
-  const { openfeature } = tracer
-  await OpenFeature.setProviderAndWait(openfeature)
+  const hasFeatureFlaggingConfiguration = [
+    'DD_FEATURE_FLAGS_ENABLED',
+    'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE',
+    'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL',
+    'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'
+  ].some(name => process.env[name] !== undefined)
+
+  if (hasFeatureFlaggingConfiguration) {
+    const { openfeature } = tracer
+    await OpenFeature.setProviderAndWait(openfeature)
+  }
   openFeatureClient = OpenFeature.getClient()
   res.json({})
 })

utils/docker_fixtures/_mock_ffe_agentless_backend.py

@@ -194,6 +194,7 @@ def _handle_config(self) -> None:
                 self.send_response(status_code)
                 for key, value in headers.items():
                     self.send_header(key, value)
+                self.send_header("Content-Length", str(len(body)))
                 self.end_headers()
                 if body:
                     self.wfile.write(body)
@@ -228,11 +229,13 @@ def _handle_responses_control(self) -> None:
         self._write_json(HTTPStatus.OK, self.server.state.status())
 
     def _write_json(self, status_code: HTTPStatus, payload: dict[str, Any] | MockFFEAgentlessBackendStatus) -> None:
+        body = json.dumps(payload).encode("utf-8")
         with contextlib.suppress(BrokenPipeError, ConnectionResetError):
             self.send_response(status_code)
             self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
             self.end_headers()
-            self.wfile.write(json.dumps(payload).encode("utf-8"))
+            self.wfile.write(body)
 
 
 def _has_auth(headers: Mapping[str, str]) -> bool: