Warn on agentless authentication failures

Author: leoromanovsky

products/feature-flagging/feature-flagging-lib/build.gradle.kts

@@ -20,6 +20,7 @@ dependencies {
   api(project(":communication"))
   implementation(project(":internal-api"))
   api(project(":products:feature-flagging:feature-flagging-bootstrap"))
+  implementation(project(":utils:logging-utils"))
   api(project(":utils:queue-utils"))
 
   compileOnly(project(":dd-trace-core")) // shading does not work with this one

products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java

@@ -4,6 +4,7 @@
 import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER;
 
 import datadog.communication.http.OkHttpUtils;
+import datadog.logging.RatelimitedLogger;
 import datadog.trace.api.Config;
 import datadog.trace.api.featureflag.FeatureFlaggingGateway;
 import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
@@ -38,6 +39,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService {
   private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH =
       "/api/v2/feature-flagging/config/server-distribution";
   private static final int MAX_ATTEMPTS = 3;
+  private static final int MINUTES_BETWEEN_AUTH_WARNINGS = 5;
   private static final long FIRST_RETRY_MIN_MILLIS = 2_000;
   private static final long FIRST_RETRY_MAX_MILLIS = 10_000;
   private static final long SECOND_RETRY_MIN_MILLIS = 5_000;
@@ -51,6 +53,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService {
   private final ScheduledExecutorService executor;
   private final RetrySleeper retrySleeper;
   private final DoubleSupplier jitter;
+  private final RatelimitedLogger ratelimitedLogger;
   private final Object lifecycleLock = new Object();
   private final AtomicBoolean polling = new AtomicBoolean();
   private volatile boolean closed;
@@ -73,7 +76,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint
         Executors.newSingleThreadScheduledExecutor(
             new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)),
         TimeUnit.MILLISECONDS::sleep,
-        () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER));
+        () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER),
+        new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES));
   }
 
   AgentlessConfigurationSource(
@@ -89,7 +93,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint
         client,
         executor,
         TimeUnit.MILLISECONDS::sleep,
-        () -> 1.0);
+        () -> 1.0,
+        new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES));
   }
 
   AgentlessConfigurationSource(
@@ -100,13 +105,34 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint
       final ScheduledExecutorService executor,
       final RetrySleeper retrySleeper,
       final DoubleSupplier jitter) {
+    this(
+        endpoint,
+        config,
+        pollIntervalMillis,
+        client,
+        executor,
+        retrySleeper,
+        jitter,
+        new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES));
+  }
+
+  AgentlessConfigurationSource(
+      final HttpUrl endpoint,
+      final Config config,
+      final long pollIntervalMillis,
+      final UfcHttpClient client,
+      final ScheduledExecutorService executor,
+      final RetrySleeper retrySleeper,
+      final DoubleSupplier jitter,
+      final RatelimitedLogger ratelimitedLogger) {
     this.endpoint = endpoint;
     this.config = config;
     this.pollIntervalMillis = pollIntervalMillis;
     this.client = client;
     this.executor = executor;
     this.retrySleeper = retrySleeper;
     this.jitter = jitter;
+    this.ratelimitedLogger = ratelimitedLogger;
   }
 
   @Override
@@ -207,9 +233,13 @@ private boolean apply(final UfcHttpResponse response) {
       return true;
     }
     if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED
-        || response.status == HttpURLConnection.HTTP_FORBIDDEN
-        || response.status != HttpURLConnection.HTTP_OK
-        || response.body == null) {
+        || response.status == HttpURLConnection.HTTP_FORBIDDEN) {
+      ratelimitedLogger.warn(
+          "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid",
+          response.status);
+      return false;
+    }
+    if (response.status != HttpURLConnection.HTTP_OK || response.body == null) {
       return false;
     }
     final ServerConfiguration configuration;

products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java

@@ -15,6 +15,7 @@
 import static org.mockito.Mockito.verifyNoInteractions;
 
 import datadog.communication.http.OkHttpUtils;
+import datadog.logging.RatelimitedLogger;
 import datadog.trace.agent.test.server.http.JavaTestHttpServer;
 import datadog.trace.api.Config;
 import datadog.trace.api.featureflag.FeatureFlaggingGateway;
@@ -413,6 +414,39 @@ void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception {
     verifyNoInteractions(listener);
   }
 
+  @Test
+  void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception {
+    final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class);
+    final FakeClient client = new FakeClient(response(401, null, null), response(403, null, null));
+    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+    final AgentlessConfigurationSource service =
+        new AgentlessConfigurationSource(
+            HttpUrl.get("http://localhost" + CONFIG_PATH),
+            config(),
+            30_000,
+            client,
+            executor,
+            delay -> {},
+            () -> 1.0,
+            ratelimitedLogger);
+
+    try {
+      assertFalse(service.pollOnce());
+      assertFalse(service.pollOnce());
+
+      verify(ratelimitedLogger)
+          .warn(
+              "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid",
+              HttpURLConnection.HTTP_UNAUTHORIZED);
+      verify(ratelimitedLogger)
+          .warn(
+              "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid",
+              HttpURLConnection.HTTP_FORBIDDEN);
+    } finally {
+      service.close();
+    }
+  }
+
   @Test
   void retriesTimeoutBeforeApplyingConfig() throws Exception {
     final FakeClient client =