Add agentless Feature Flagging configuration source (#11892)

Define Feature Flagging configuration source contract

Add Datadog-managed agentless UFC polling

Support custom agentless UFC endpoints

Select the Feature Flagging configuration source

Warn on agentless authentication failures

Merge remote-tracking branch 'origin/master' into rewrite/java-agentless-narrative

# Conflicts:
#	dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java
#	products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java

fix(ffe): handle nullable agentless response bodies

Merge branch 'master' into leo.romanovsky/ffl-2693-java-agentless-configuration-source

Address agentless poller review feedback

Fix feature flag config test imports

Address additional agentless source review feedback

Support the UFC CDN response contract

Harden agentless configuration responses

Rely on OkHttp gzip negotiation and cover truncated response cache preservation. Require JSON:API envelopes for custom endpoints.

Separate UFC transport parsers

Keep Remote Configuration on raw UFC parsing and give agentless JSON:API its own streaming envelope parser. Share UFC adapters without materializing an intermediate response map.

Fix feature flagging test formatting

Fix feature flagging parser coverage

Address final feature flagging review feedback

Merge branch 'master' into leo.romanovsky/ffl-2693-java-agentless-configuration-source

Merge branch 'master' into leo.romanovsky/ffl-2693-java-agentless-configuration-source

fix(feature-flags): align configuration source semantics

feat(feature-flags): delay agentless polling until provider use

feat(communication): support mapped retried HTTP calls

fix(feature-flags): align agentless HTTP behavior

fix(feature-flags): complete initial poll during activation

chore(feature-flags): remove unused agent feature

Merge branch 'master' into leo.romanovsky/ffl-2693-java-agentless-configuration-source

Fix race in agentless config source listener test

scheduledPollContinuesAfterListenerRuntimeException waited on
FakeClient.calls, which is incremented when a request starts rather than
when it completes. The barrier therefore released as soon as the second
poll began, letting the assertion race the poll thread that applies the
configuration and notifies the listener.

Wait on a CountDownLatch counted down by the listener itself, so the
second notification is guaranteed to have happened before the
assertions run.

Environment: Datadog workspace

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Co-authored-by: vickie.fridge <vickie.fridge@datadoghq.com>

test(feature-flags): cover agentless lifecycle branches

chore(agent): raise jar size budget to 34 MiB

fix(feature-flags): document retry policy confinement

Revert "chore(agent): raise jar size budget to 34 MiB"

This reverts commit 29c1fd4.

Merge remote-tracking branch 'origin/master' into agent/pr11892-cross-sdk-contract

Co-authored-by: vjfridge <vickie.fridge@datadoghq.com>
Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: 3 people

communication/src/main/java/datadog/communication/http/HttpRetryPolicy.java

@@ -54,7 +54,13 @@ public class HttpRetryPolicy implements AutoCloseable {
   private final double delayFactor;
   private final boolean suppressInterrupts;
 
-  private HttpRetryPolicy(
+  /**
+   * Creates a retry policy.
+   *
+   * <p>Protected so products with a stricter cross-SDK retry contract can reuse the shared HTTP
+   * retry loop while supplying their own response, exception, and backoff rules.
+   */
+  protected HttpRetryPolicy(
       int retriesLeft, long delay, double delayFactor, boolean suppressInterrupts) {
     this.retriesLeft = retriesLeft;
     this.delay = delay;

communication/src/main/java/datadog/communication/http/OkHttpUtils.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
+import okhttp3.Call;
 import okhttp3.ConnectionPool;
 import okhttp3.ConnectionSpec;
 import okhttp3.Credentials;
@@ -405,19 +406,39 @@ public void writeTo(BufferedSink sink) throws IOException {
   public static Response sendWithRetries(
       OkHttpClient httpClient, HttpRetryPolicy.Factory retryPolicyFactory, Request request)
       throws IOException {
+    return sendWithRetries((Call.Factory) httpClient, retryPolicyFactory, request);
+  }
+
+  public static Response sendWithRetries(
+      Call.Factory callFactory, HttpRetryPolicy.Factory retryPolicyFactory, Request request)
+      throws IOException {
+    return sendWithRetries(callFactory, retryPolicyFactory, request, response -> response);
+  }
+
+  public static <T> T sendWithRetries(
+      Call.Factory callFactory,
+      HttpRetryPolicy.Factory retryPolicyFactory,
+      Request request,
+      ResponseMapper<T> responseMapper)
+      throws IOException {
     try (HttpRetryPolicy retryPolicy = retryPolicyFactory.create()) {
       while (true) {
+        Response response = null;
         try {
-          Response response = httpClient.newCall(request).execute();
+          response = callFactory.newCall(request).execute();
           if (response.isSuccessful()) {
-            return response;
+            return responseMapper.map(response);
           }
           if (!retryPolicy.shouldRetry(response)) {
-            return response;
+            return responseMapper.map(response);
           } else {
             closeQuietly(response);
+            response = null;
           }
         } catch (Exception ex) {
+          if (response != null) {
+            closeQuietly(response);
+          }
           if (!retryPolicy.shouldRetry(ex)) {
             throw ex;
           }
@@ -428,6 +449,11 @@ public static Response sendWithRetries(
     }
   }
 
+  @FunctionalInterface
+  public interface ResponseMapper<T> {
+    T map(Response response) throws IOException;
+  }
+
   private static void closeQuietly(Response response) {
     try {
       response.close();

communication/src/test/java/datadog/communication/http/OkHttpUtilsRetryTest.java

@@ -0,0 +1,53 @@
+package datadog.communication.http;
+
+import static datadog.communication.http.OkHttpUtils.sendWithRetries;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.net.ConnectException;
+import java.util.concurrent.atomic.AtomicInteger;
+import okhttp3.Call;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.ResponseBody;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import org.junit.jupiter.api.Test;
+
+class OkHttpUtilsRetryTest {
+
+  @Test
+  void retriesResponseMappingFailureThroughCallFactory() throws Exception {
+    final MockWebServer server = new MockWebServer();
+    final OkHttpClient client = new OkHttpClient();
+    final AtomicInteger mappingAttempts = new AtomicInteger();
+    server.enqueue(new MockResponse().setBody("first"));
+    server.enqueue(new MockResponse().setBody("second"));
+    server.start();
+
+    try {
+      final Request request = new Request.Builder().url(server.url("/configuration")).build();
+
+      final String body =
+          sendWithRetries(
+              (Call.Factory) client,
+              new HttpRetryPolicy.Factory(1, 0, 1),
+              request,
+              response -> {
+                try (ResponseBody responseBody = response.body()) {
+                  if (mappingAttempts.getAndIncrement() == 0) {
+                    throw new ConnectException("response body could not be mapped");
+                  }
+                  return responseBody.string();
+                }
+              });
+
+      assertEquals("second", body);
+      assertEquals(2, mappingAttempts.get());
+      assertEquals(2, server.getRequestCount());
+    } finally {
+      client.dispatcher().executorService().shutdownNow();
+      client.connectionPool().evictAll();
+      server.shutdown();
+    }
+  }
+}

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -135,8 +135,7 @@ private enum AgentFeature {
     AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false),
     APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false),
     LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false),
-    LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false),
-    FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false);
+    LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false);
 
     private final String configKey;
     private final String systemProp;
@@ -285,7 +284,7 @@ public static void start(
     agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION);
     appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION);
     llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS);
-    featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING);
+    featureFlaggingEnabled = isFeatureFlaggingEnabled();
 
     // setup writers when llmobs is enabled to accomodate apm and llmobs
     if (llmObsEnabled) {
@@ -531,6 +530,9 @@ public static void shutdown(final boolean sync) {
     if (flareEnabled) {
       stopFlarePoller();
     }
+    if (featureFlaggingEnabled) {
+      shutdownFeatureFlagging(AGENT_CLASSLOADER);
+    }
 
     if (agentlessLogSubmissionEnabled) {
       shutdownLogsIntake();
@@ -1287,6 +1289,20 @@ private static void maybeStartFeatureFlagging(final Class<?> scoClass, final Obj
     }
   }
 
+  static void shutdownFeatureFlagging(final ClassLoader agentClassLoader) {
+    if (agentClassLoader == null) {
+      return;
+    }
+    try {
+      final Class<?> ffSysClass =
+          agentClassLoader.loadClass("com.datadog.featureflag.FeatureFlaggingSystem");
+      final Method stopMethod = ffSysClass.getMethod("stop");
+      stopMethod.invoke(null);
+    } catch (final Throwable e) {
+      log.warn("Unable to stop Feature Flagging subsystem", e);
+    }
+  }
+
   private static void maybeInstallLogsIntake(Class<?> scoClass, Object sco) {
     if (agentlessLogSubmissionEnabled || appLogsCollectionEnabled) {
       StaticEventLogger.begin("Logs Intake");
@@ -1739,6 +1755,45 @@ private static boolean isFeatureEnabled(AgentFeature feature) {
     }
   }
 
+  private static boolean isFeatureFlaggingEnabled() {
+    final Boolean providerEnabled =
+        featureFlaggingBooleanSetting(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED);
+    final String configurationSource =
+        featureFlaggingSetting(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE);
+    final Boolean legacyProviderEnabled =
+        featureFlaggingBooleanSetting(FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED);
+
+    return FeatureFlaggingConfig.resolveConfiguration(
+            providerEnabled, configurationSource, legacyProviderEnabled)
+        .isEnabled();
+  }
+
+  @SuppressFBWarnings(
+      value = "NP_BOOLEAN_RETURN_NULL",
+      justification = "A null value preserves the distinction between absent and explicitly false")
+  private static Boolean featureFlaggingBooleanSetting(final String configKey) {
+    final String value = featureFlaggingSetting(configKey);
+    if (value == null) {
+      return null;
+    }
+    return Boolean.parseBoolean(value) || "1".equals(value);
+  }
+
+  private static String featureFlaggingSetting(final String configKey) {
+    final String systemProperty = propertyNameToSystemPropertyName(configKey);
+    String value = SystemProperties.get(systemProperty);
+    if (value == null) {
+      value = getStableConfig(FLEET, configKey);
+    }
+    if (value == null) {
+      value = ddGetEnv(systemProperty);
+    }
+    if (value == null) {
+      value = getStableConfig(LOCAL, configKey);
+    }
+    return value;
+  }
+
   /**
    * @see datadog.trace.api.ProductActivation#fromString(String)
    */

dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java

@@ -0,0 +1,48 @@
+package datadog.trace.bootstrap;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class AgentFeatureFlaggingLifecycleTest {
+
+  @BeforeEach
+  void reset() {
+    FakeFeatureFlaggingSystem.stopCalls.set(0);
+  }
+
+  @Test
+  void shutdownInvokesFeatureFlaggingSystemStopThroughAgentClassLoader() {
+    final ClassLoader classLoader =
+        new ClassLoader(null) {
+          @Override
+          public Class<?> loadClass(final String name) throws ClassNotFoundException {
+            if ("com.datadog.featureflag.FeatureFlaggingSystem".equals(name)) {
+              return FakeFeatureFlaggingSystem.class;
+            }
+            return super.loadClass(name);
+          }
+        };
+
+    Agent.shutdownFeatureFlagging(classLoader);
+
+    assertEquals(1, FakeFeatureFlaggingSystem.stopCalls.get());
+  }
+
+  @Test
+  void shutdownIsNoopBeforeAgentClassLoaderExists() {
+    Agent.shutdownFeatureFlagging(null);
+
+    assertEquals(0, FakeFeatureFlaggingSystem.stopCalls.get());
+  }
+
+  public static final class FakeFeatureFlaggingSystem {
+    private static final AtomicInteger stopCalls = new AtomicInteger();
+
+    public static void stop() {
+      stopCalls.incrementAndGet();
+    }
+  }
+}

dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy

@@ -42,6 +42,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
     command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()])
     final builder = new ProcessBuilder(command).directory(new File(buildDirectory))
     builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true')
+    builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config')
     return builder
   }
 

dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

@@ -48,6 +48,9 @@ public final class ConfigDefaults {
 
   static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true;
   static final String DEFAULT_SITE = "datadoghq.com";
+  static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless";
+  static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30;
+  static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 5;
 
   static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false;
   static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8;

internal-api/build.gradle.kts

@@ -269,6 +269,7 @@ dependencies {
   api(project(":components:context"))
   api(project(":components:environment"))
   api(project(":components:json"))
+  implementation(project(":products:feature-flagging:feature-flagging-config"))
   api(project(":utils:config-utils"))
   api(project(":utils:time-utils"))