feat(ffe): defer agentless delivery until provider access

Author: leoromanovsky

products/feature-flagging/feature-flagging-agent/build.gradle.kts

@@ -17,6 +17,7 @@ dependencies {
   api(libs.slf4j)
   api(project(":products:feature-flagging:feature-flagging-lib"))
   api(project(":internal-api"))
+  compileOnly(project(":products:feature-flagging:feature-flagging-config"))
 
   testImplementation(libs.bundles.junit5)
   testImplementation(libs.bundles.mockito)

products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java

@@ -2,6 +2,8 @@
 
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.trace.api.Config;
+import datadog.trace.api.featureflag.FeatureFlaggingGateway;
+import datadog.trace.api.featureflag.config.FeatureFlaggingConfigurationSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -12,16 +14,61 @@ public class FeatureFlaggingSystem {
   private static volatile ConfigurationSourceService CONFIG_SERVICE;
   private static volatile ExposureWriter EXPOSURE_WRITER;
   private static volatile SpanEnrichmentWriter SPAN_ENRICHMENT_WRITER;
+  private static volatile FeatureFlaggingGateway.ActivationListener ACTIVATION_LISTENER;
+  private static volatile boolean STARTED;
 
   private FeatureFlaggingSystem() {}
 
   public static synchronized void start(final SharedCommunicationObjects sco) {
-    if (CONFIG_SERVICE != null || EXPOSURE_WRITER != null) {
+    if (STARTED) {
       LOGGER.debug("Feature Flagging system already started");
       return;
     }
     LOGGER.debug("Feature Flagging system starting");
     final Config config = Config.get();
+    final FeatureFlaggingConfigurationSource source =
+        FeatureFlaggingConfigurationSource.fromResolved(
+            config.getFeatureFlaggingConfigurationSource());
+    STARTED = true;
+
+    if (source == FeatureFlaggingConfigurationSource.OFFLINE) {
+      LOGGER.debug("Feature Flagging system disabled");
+      return;
+    }
+    if (source == FeatureFlaggingConfigurationSource.AGENTLESS) {
+      final FeatureFlaggingGateway.ActivationListener activationListener =
+          () -> activateAgentless(sco, config);
+      ACTIVATION_LISTENER = activationListener;
+      FeatureFlaggingGateway.addActivationListener(activationListener);
+      LOGGER.debug("Feature Flagging system awaiting application provider activation");
+      return;
+    }
+
+    try {
+      initializeSystem(sco, config);
+    } catch (final RuntimeException | Error e) {
+      STARTED = false;
+      throw e;
+    }
+  }
+
+  private static synchronized void activateAgentless(
+      final SharedCommunicationObjects sco, final Config config) {
+    final FeatureFlaggingGateway.ActivationListener activationListener = ACTIVATION_LISTENER;
+    if (!STARTED || activationListener == null) {
+      return;
+    }
+    ACTIVATION_LISTENER = null;
+    FeatureFlaggingGateway.removeActivationListener(activationListener);
+    try {
+      initializeSystem(sco, config);
+    } catch (final RuntimeException | Error e) {
+      STARTED = false;
+      throw e;
+    }
+  }
+
+  private static void initializeSystem(final SharedCommunicationObjects sco, final Config config) {
     final ConfigurationSourceService configService = createConfigurationSourceService(sco, config);
     final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config);
     initialize(configService, exposureWriter);
@@ -60,16 +107,17 @@ static void initialize(
 
   static ConfigurationSourceService createConfigurationSourceService(
       final SharedCommunicationObjects sco, final Config config) {
-    final ConfigurationSource configurationSource =
-        ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource());
+    final FeatureFlaggingConfigurationSource configurationSource =
+        FeatureFlaggingConfigurationSource.fromResolved(
+            config.getFeatureFlaggingConfigurationSource());
 
-    if (configurationSource == ConfigurationSource.REMOTE_CONFIG) {
+    if (configurationSource == FeatureFlaggingConfigurationSource.REMOTE_CONFIG) {
       if (!config.isRemoteConfigEnabled()) {
         throw new IllegalStateException("Feature Flagging system started without RC");
       }
       return new RemoteConfigServiceImpl(sco, config);
     }
-    if (configurationSource == ConfigurationSource.AGENTLESS) {
+    if (configurationSource == FeatureFlaggingConfigurationSource.AGENTLESS) {
       return new AgentlessConfigurationSource(config);
     }
     LOGGER.debug(
@@ -78,12 +126,18 @@ static ConfigurationSourceService createConfigurationSourceService(
   }
 
   public static synchronized void stop() {
+    final FeatureFlaggingGateway.ActivationListener activationListener = ACTIVATION_LISTENER;
     final SpanEnrichmentWriter spanEnrichmentWriter = SPAN_ENRICHMENT_WRITER;
     final ExposureWriter exposureWriter = EXPOSURE_WRITER;
     final ConfigurationSourceService configService = CONFIG_SERVICE;
+    STARTED = false;
+    ACTIVATION_LISTENER = null;
     SPAN_ENRICHMENT_WRITER = null;
     EXPOSURE_WRITER = null;
     CONFIG_SERVICE = null;
+    if (activationListener != null) {
+      FeatureFlaggingGateway.removeActivationListener(activationListener);
+    }
     try {
       if (spanEnrichmentWriter != null) {
         spanEnrichmentWriter.close();
@@ -102,25 +156,7 @@ public static synchronized void stop() {
     LOGGER.debug("Feature Flagging system stopped");
   }
 
-  private enum ConfigurationSource {
-    AGENTLESS("agentless"),
-    REMOTE_CONFIG("remote_config"),
-    OFFLINE("offline");
-
-    private final String value;
-
-    ConfigurationSource(final String value) {
-      this.value = value;
-    }
-
-    private static ConfigurationSource from(final String value) {
-      for (final ConfigurationSource source : values()) {
-        if (source.value.equals(value)) {
-          return source;
-        }
-      }
-      throw new IllegalArgumentException(
-          "Unsupported Feature Flagging configuration source: " + value);
-    }
+  static boolean isAwaitingApplicationActivation() {
+    return ACTIVATION_LISTENER != null;
   }
 }

products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java

@@ -3,9 +3,11 @@
 import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED;
 import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doThrow;
@@ -27,6 +29,20 @@
 
 class FeatureFlaggingSystemTest {
 
+  @Test
+  @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless")
+  void agentlessStartWaitsForApplicationProviderActivation() {
+    try {
+      FeatureFlaggingSystem.start(sharedCommunicationObjects());
+
+      assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation());
+    } finally {
+      FeatureFlaggingSystem.stop();
+    }
+
+    assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation());
+  }
+
   @Test
   @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config")
   @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true")
@@ -97,9 +113,8 @@ void explicitRemoteConfigUsesRemoteConfigService() {
 
   @Test
   @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid")
-  void invalidConfigurationSourceUsesAgentlessDefault() {
-    assertInstanceOf(
-        AgentlessConfigurationSource.class,
+  void invalidConfigurationSourceFailsClosed() {
+    assertNull(
         FeatureFlaggingSystem.createConfigurationSourceService(
             sharedCommunicationObjects(), Config.get()));
   }

products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java

@@ -67,6 +67,7 @@ public DDEvaluator(final Runnable configCallback) {
   @Override
   public boolean initialize(
       final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception {
+    FeatureFlaggingGateway.activate();
     FeatureFlaggingGateway.addConfigListener(this);
     return initializationLatch.await(timeout, unit) || hasConfiguration();
   }

products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java

@@ -23,6 +23,7 @@
 import com.squareup.moshi.JsonWriter;
 import com.squareup.moshi.Moshi;
 import com.squareup.moshi.Types;
+import datadog.trace.api.featureflag.FeatureFlaggingGateway;
 import datadog.trace.api.featureflag.ufc.v1.Flag;
 import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
 import dev.openfeature.sdk.ErrorCode;
@@ -145,6 +146,22 @@ public void testInitializeTimesOutWithoutConfig() throws Exception {
     }
   }
 
+  @Test
+  public void testInitializeSignalsApplicationProviderActivation() throws Exception {
+    final FeatureFlaggingGateway.ActivationListener listener =
+        mock(FeatureFlaggingGateway.ActivationListener.class);
+    final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class));
+    FeatureFlaggingGateway.addActivationListener(listener);
+    try {
+      evaluator.initialize(1, MILLISECONDS, mock(EvaluationContext.class));
+
+      verify(listener).activate();
+    } finally {
+      evaluator.shutdown();
+      FeatureFlaggingGateway.removeActivationListener(listener);
+    }
+  }
+
   @Test
   public void testInitializeWaitsForNonNullConfig() throws Exception {
     final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class));

products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java

@@ -11,11 +11,16 @@ public abstract class FeatureFlaggingGateway {
 
   public interface ConfigListener extends Consumer<ServerConfiguration> {}
 
+  public interface ActivationListener {
+    void activate();
+  }
+
   public interface ExposureListener extends Consumer<ExposureEvent> {}
 
   public interface SpanEnrichmentListener extends Consumer<SpanEnrichmentEvent> {}
 
   private static final List<ConfigListener> CONFIG_LISTENERS = new CopyOnWriteArrayList<>();
+  private static final List<ActivationListener> ACTIVATION_LISTENERS = new CopyOnWriteArrayList<>();
   private static final List<ExposureListener> EXPOSURE_LISTENERS = new CopyOnWriteArrayList<>();
   private static final List<SpanEnrichmentListener> SPAN_ENRICHMENT_LISTENERS =
       new CopyOnWriteArrayList<>();
@@ -42,6 +47,19 @@ public static void dispatch(final ServerConfiguration config) {
     CONFIG_LISTENERS.forEach(listener -> listener.accept(config));
   }
 
+  public static void addActivationListener(final ActivationListener listener) {
+    ACTIVATION_LISTENERS.add(listener);
+  }
+
+  public static void removeActivationListener(final ActivationListener listener) {
+    ACTIVATION_LISTENERS.remove(listener);
+  }
+
+  /** Signals that application code initialized the Datadog OpenFeature provider. */
+  public static void activate() {
+    ACTIVATION_LISTENERS.forEach(ActivationListener::activate);
+  }
+
   public static void addExposureListener(final ExposureListener listener) {
     EXPOSURE_LISTENERS.add(listener);
   }

products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java

@@ -13,6 +13,7 @@
 class FeatureFlaggingGatewayTest {
 
   private FeatureFlaggingGateway.ConfigListener configListener;
+  private FeatureFlaggingGateway.ActivationListener activationListener;
   private FeatureFlaggingGateway.ExposureListener exposureListener;
   private FeatureFlaggingGateway.SpanEnrichmentListener spanEnrichmentListener;
   private ServerConfiguration firstConfiguration;
@@ -23,6 +24,7 @@ class FeatureFlaggingGatewayTest {
   @BeforeEach
   void setUp() {
     configListener = mock(FeatureFlaggingGateway.ConfigListener.class);
+    activationListener = mock(FeatureFlaggingGateway.ActivationListener.class);
     exposureListener = mock(FeatureFlaggingGateway.ExposureListener.class);
     spanEnrichmentListener = mock(FeatureFlaggingGateway.SpanEnrichmentListener.class);
     firstConfiguration = mock(ServerConfiguration.class);
@@ -34,10 +36,21 @@ void setUp() {
   @AfterEach
   void tearDown() {
     FeatureFlaggingGateway.removeConfigListener(configListener);
+    FeatureFlaggingGateway.removeActivationListener(activationListener);
     FeatureFlaggingGateway.removeExposureListener(exposureListener);
     FeatureFlaggingGateway.removeSpanEnrichmentListener(spanEnrichmentListener);
   }
 
+  @Test
+  void testProviderActivationListener() {
+    FeatureFlaggingGateway.addActivationListener(activationListener);
+
+    FeatureFlaggingGateway.activate();
+
+    verify(activationListener).activate();
+    verifyNoMoreInteractions(activationListener);
+  }
+
   @Test
   void testAttachingAConfigListener() {
     clearCurrentServerConfiguration();