From 268fbea47772cf0db4f93c8f5704c71a6399e533 Mon Sep 17 00:00:00 2001 From: typotter Date: Thu, 23 Apr 2026 16:16:22 -0600 Subject: [PATCH 1/3] Add PROVIDER_FATAL state propagation for non-retryable RC errors When the RC endpoint returns a non-retryable 4xx (401, 403), the OpenFeature provider now transitions to PROVIDER_FATAL instead of blocking indefinitely or timing out with PROVIDER_NOT_READY. Code path: - DefaultConfigurationPoller.sendRequest() detects 401/403 and notifies registered NonRetryableErrorListeners - RemoteConfigServiceImpl registers such a listener and forwards to FeatureFlaggingGateway.dispatchFatalError() - DDEvaluator implements FeatureFlaggingGateway.FatalErrorListener; on receipt it stores the error message and releases the initialization latch - DDEvaluator.initialize() throws FatalError after the latch is released with a fatal error present - Provider.initialize() re-throws FatalError (it extends OpenFeatureError) causing the OpenFeature SDK to set state to PROVIDER_FATAL --- .../trace/api/openfeature/DDEvaluator.java | 29 +++++++++++++- .../featureflag/FeatureFlaggingGateway.java | 23 +++++++++++ .../featureflag/RemoteConfigServiceImpl.java | 4 ++ .../remoteconfig/ConfigurationPoller.java | 13 ++++++ .../DefaultConfigurationPoller.java | 40 ++++++++++++++++++- 5 files changed, 106 insertions(+), 3 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index a0a85f6a9ab..3a038af46a3 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -23,6 +23,7 @@ import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Structure; import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.exceptions.FatalError; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -41,29 +42,46 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -class DDEvaluator implements Evaluator, FeatureFlaggingGateway.ConfigListener { +class DDEvaluator + implements Evaluator, + FeatureFlaggingGateway.ConfigListener, + FeatureFlaggingGateway.FatalErrorListener { private static final Set> SUPPORTED_RESOLUTION_TYPES = new HashSet<>(asList(String.class, Boolean.class, Integer.class, Double.class, Value.class)); private final Runnable configCallback; + private final Runnable fatalCallback; private final AtomicReference configuration = new AtomicReference<>(); private final CountDownLatch initializationLatch = new CountDownLatch(1); + private volatile String fatalErrorMessage = null; public DDEvaluator(final Runnable configCallback) { + this(configCallback, () -> {}); + } + + DDEvaluator(final Runnable configCallback, final Runnable fatalCallback) { this.configCallback = configCallback; + this.fatalCallback = fatalCallback; } @Override public boolean initialize( final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception { FeatureFlaggingGateway.addConfigListener(this); - return initializationLatch.await(timeout, unit); // await for initialization + FeatureFlaggingGateway.addFatalErrorListener(this); + initializationLatch.await(timeout, unit); // await for initialization or fatal error + final String fatal = fatalErrorMessage; + if (fatal != null) { + throw new FatalError(fatal); + } + return configuration.get() != null; } @Override public void shutdown() { FeatureFlaggingGateway.removeConfigListener(this); + FeatureFlaggingGateway.removeFatalErrorListener(this); } @Override @@ -73,6 +91,13 @@ public void accept(final ServerConfiguration config) { configCallback.run(); } + @Override + public void onFatalError(final int httpStatus, final String message) { + fatalErrorMessage = message != null ? message : "RC fatal error (HTTP " + httpStatus + ")"; + initializationLatch.countDown(); + fatalCallback.run(); + } + @Override public ProviderEvaluation evaluate( final Class target, diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index b9d73ffa7ab..cd5a2c5b63a 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -13,8 +13,19 @@ public interface ConfigListener extends Consumer {} public interface ExposureListener extends Consumer {} + /** + * Listener notified when a non-retryable fatal error is received from the RC endpoint (e.g. HTTP + * 401 Unauthorized). Implementations should transition the OpenFeature provider to + * PROVIDER_FATAL. + */ + public interface FatalErrorListener { + void onFatalError(int httpStatus, String message); + } + private static final List CONFIG_LISTENERS = new CopyOnWriteArrayList<>(); private static final List EXPOSURE_LISTENERS = new CopyOnWriteArrayList<>(); + private static final List FATAL_ERROR_LISTENERS = + new CopyOnWriteArrayList<>(); private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); @@ -49,4 +60,16 @@ public static void removeExposureListener(final ExposureListener listener) { public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + + public static void addFatalErrorListener(final FatalErrorListener listener) { + FATAL_ERROR_LISTENERS.add(listener); + } + + public static void removeFatalErrorListener(final FatalErrorListener listener) { + FATAL_ERROR_LISTENERS.remove(listener); + } + + public static void dispatchFatalError(final int httpStatus, final String message) { + FATAL_ERROR_LISTENERS.forEach(listener -> listener.onFatalError(httpStatus, message)); + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index cf0c400ccad..3534037b48a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -26,6 +26,8 @@ public class RemoteConfigServiceImpl implements RemoteConfigService, ConfigurationChangesTypedListener { private final ConfigurationPoller configurationPoller; + private final ConfigurationPoller.NonRetryableErrorListener nonRetryableErrorListener = + FeatureFlaggingGateway::dispatchFatalError; public RemoteConfigServiceImpl(final SharedCommunicationObjects sco, final Config config) { configurationPoller = sco.configurationPoller(config); @@ -36,6 +38,7 @@ public void init() { configurationPoller.addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); configurationPoller.addListener( Product.FFE_FLAGS, UniversalFlagConfigDeserializer.INSTANCE, this); + configurationPoller.addNonRetryableErrorListener(nonRetryableErrorListener); configurationPoller.start(); } @@ -43,6 +46,7 @@ public void init() { public void close() { configurationPoller.removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); configurationPoller.removeListeners(Product.FFE_FLAGS); + configurationPoller.removeNonRetryableErrorListener(nonRetryableErrorListener); configurationPoller.stop(); } diff --git a/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java b/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java index f358b0e457e..0e0fd58ef15 100644 --- a/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java +++ b/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java @@ -33,4 +33,17 @@ void addListener( void start(); void stop(); + + /** + * Registers a listener that is called when a non-retryable HTTP error (e.g. 401, 403) is + * received from the RC endpoint. The default implementation is a no-op. + */ + default void addNonRetryableErrorListener(NonRetryableErrorListener listener) {} + + default void removeNonRetryableErrorListener(NonRetryableErrorListener listener) {} + + @FunctionalInterface + interface NonRetryableErrorListener { + void onNonRetryableError(int httpStatus, String message); + } } diff --git a/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java b/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java index 61e5fc88d77..cc083a3c9cb 100644 --- a/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java +++ b/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java @@ -76,6 +76,7 @@ public class DefaultConfigurationPoller private final Map productStates = new EnumMap<>(Product.class); private final Map fileListeners = new HashMap<>(); private final List configurationEndListeners = new ArrayList<>(); + private final List nonRetryableErrorListeners = new ArrayList<>(); private final ClientState nextClientState = new ClientState(); private final AtomicInteger startCount = new AtomicInteger(0); @@ -194,6 +195,16 @@ public synchronized void removeConfigurationEndListener(ConfigurationEndListener this.configurationEndListeners.removeIf(l -> l == listener); } + @Override + public synchronized void addNonRetryableErrorListener(NonRetryableErrorListener listener) { + this.nonRetryableErrorListeners.add(listener); + } + + @Override + public synchronized void removeNonRetryableErrorListener(NonRetryableErrorListener listener) { + this.nonRetryableErrorListeners.removeIf(l -> l == listener); + } + @Override public synchronized void addCapabilities(long flags) { capabilities |= flags; @@ -344,13 +355,15 @@ void sendRequest(Consumer responseBodyConsumer) throws IOException return; } // Retrieve body content for detailed error messages + String bodyString = null; if (body != null) { try { + bodyString = body.string(); ratelimitedLogger.warn( "Failed to retrieve remote configuration: unexpected response code {} {} {}", response.message(), response.code(), - body.string()); + bodyString); } catch (IOException ex) { ExceptionHelper.rateLimitedLogException( ratelimitedLogger, log, ex, "Error while getting error message body"); @@ -361,6 +374,31 @@ void sendRequest(Consumer responseBodyConsumer) throws IOException response.message(), response.code()); } + // Non-retryable 4xx responses (e.g. 401 Unauthorized, 403 Forbidden) indicate a permanent + // configuration error. Notify listeners so they can transition to a fatal state. + if (isNonRetryableError(response.code())) { + final String message = + "Remote configuration rejected with HTTP " + + response.code() + + " " + + response.message() + + (bodyString != null ? ": " + bodyString : ""); + notifyNonRetryableErrorListeners(response.code(), message); + } + } + } + + private static boolean isNonRetryableError(int code) { + return code == 401 || code == 403; + } + + private synchronized void notifyNonRetryableErrorListeners(int code, String message) { + for (NonRetryableErrorListener listener : nonRetryableErrorListeners) { + try { + listener.onNonRetryableError(code, message); + } catch (RuntimeException ex) { + log.warn("Error notifying non-retryable error listener", ex); + } } } From 23a7c9442c9d38c29e1b83a054c2bdf12b357298 Mon Sep 17 00:00:00 2001 From: typotter Date: Tue, 28 Apr 2026 12:23:08 -0600 Subject: [PATCH 2/3] feat(ffe): handle post-init RC fatal errors in DDEvaluator and Provider - DDEvaluator.evaluate() now returns PROVIDER_FATAL when fatalErrorMessage is set, taking precedence over PROVIDER_NOT_READY and normal evaluation - Two-arg DDEvaluator constructor (configCallback, fatalCallback) made public - Provider.buildEvaluator() wires onProviderFatal() as fatalCallback via the two-arg constructor; onProviderFatal() emits PROVIDER_ERROR event - DefaultConfigurationPoller.sendRequest() removes special 404 early return; 400 and 404 added to isNonRetryableError() so they trigger fatal listener - Added unit tests for fatal path: onFatalError behavior, evaluate after fatal, fatal takes precedence over existing config, callback invocation --- .../trace/api/openfeature/DDEvaluator.java | 6 +++- .../trace/api/openfeature/Provider.java | 13 ++++++-- .../api/openfeature/DDEvaluatorTest.java | 33 +++++++++++++++++++ .../DefaultConfigurationPoller.java | 6 +--- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 3a038af46a3..b5db3886d38 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -60,7 +60,7 @@ public DDEvaluator(final Runnable configCallback) { this(configCallback, () -> {}); } - DDEvaluator(final Runnable configCallback, final Runnable fatalCallback) { + public DDEvaluator(final Runnable configCallback, final Runnable fatalCallback) { this.configCallback = configCallback; this.fatalCallback = fatalCallback; } @@ -105,6 +105,10 @@ public ProviderEvaluation evaluate( final T defaultValue, final EvaluationContext context) { try { + final String fatal = fatalErrorMessage; + if (fatal != null) { + return error(defaultValue, ErrorCode.PROVIDER_FATAL, fatal); + } final ServerConfiguration config = configuration.get(); if (config == null) { return error(defaultValue, ErrorCode.PROVIDER_NOT_READY); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index f16b6e582a8..6fb62431068 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -88,13 +88,22 @@ private void onConfigurationChange() { } } + private void onProviderFatal() { + emit( + ProviderEvent.PROVIDER_ERROR, + ProviderEventDetails.builder() + .message("Remote configuration permanently rejected") + .build()); + } + private Evaluator buildEvaluator() throws Exception { if (evaluator != null) { return evaluator; } final Class evaluatorClass = loadEvaluatorClass(); - final Constructor ctor = evaluatorClass.getConstructor(Runnable.class); - return (Evaluator) ctor.newInstance((Runnable) this::onConfigurationChange); + final Constructor ctor = evaluatorClass.getConstructor(Runnable.class, Runnable.class); + return (Evaluator) + ctor.newInstance((Runnable) this::onConfigurationChange, (Runnable) this::onProviderFatal); } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index b949dc20956..d04bf11df7d 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -1,6 +1,7 @@ package datadog.trace.api.openfeature; import static dev.openfeature.sdk.ErrorCode.FLAG_NOT_FOUND; +import static dev.openfeature.sdk.ErrorCode.PROVIDER_FATAL; import static dev.openfeature.sdk.ErrorCode.TARGETING_KEY_MISSING; import static dev.openfeature.sdk.Reason.DEFAULT; import static dev.openfeature.sdk.Reason.DISABLED; @@ -147,6 +148,38 @@ public void testEvaluateNoConfig() { assertThat(details.getErrorCode(), equalTo(ErrorCode.PROVIDER_NOT_READY)); } + @Test + public void testEvaluateAfterFatalError() { + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class), mock(Runnable.class)); + evaluator.onFatalError(403, "Unauthorized API key"); + final ProviderEvaluation details = + evaluator.evaluate(Integer.class, "test", 23, mock(EvaluationContext.class)); + assertThat(details.getValue(), equalTo(23)); + assertThat(details.getReason(), equalTo(ERROR.name())); + assertThat(details.getErrorCode(), equalTo(PROVIDER_FATAL)); + } + + @Test + public void testFatalErrorTakesPrecedenceOverConfig() { + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class), mock(Runnable.class)); + // Config received, then a fatal error arrives post-init + evaluator.accept(mock(ServerConfiguration.class)); + evaluator.onFatalError(401, "RC permanently rejected"); + final ProviderEvaluation details = + evaluator.evaluate(Integer.class, "test", 23, mock(EvaluationContext.class)); + assertThat(details.getValue(), equalTo(23)); + assertThat(details.getReason(), equalTo(ERROR.name())); + assertThat(details.getErrorCode(), equalTo(PROVIDER_FATAL)); + } + + @Test + public void testFatalCallbackInvoked() { + final Runnable fatalCallback = mock(Runnable.class); + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class), fatalCallback); + evaluator.onFatalError(403, "Bad credentials"); + verify(fatalCallback, times(1)).run(); + } + @Test public void testEvaluateNoContext() { final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); diff --git a/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java b/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java index cc083a3c9cb..c9db7c13594 100644 --- a/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java +++ b/remote-config/remote-config-core/src/main/java/datadog/remoteconfig/DefaultConfigurationPoller.java @@ -337,10 +337,6 @@ List getConfigState() { void sendRequest(Consumer responseBodyConsumer) throws IOException { try (Response response = fetchConfiguration()) { - if (response.code() == 404) { - log.debug("Remote configuration endpoint is disabled"); - return; - } if (response.code() == 204) { log.debug("No configuration changes (HTTP 204 No Content)"); return; @@ -389,7 +385,7 @@ void sendRequest(Consumer responseBodyConsumer) throws IOException } private static boolean isNonRetryableError(int code) { - return code == 401 || code == 403; + return code == 400 || code == 401 || code == 403 || code == 404; } private synchronized void notifyNonRetryableErrorListeners(int code, String message) { From e485c134dfa39daed9fe89da1087cb242f08b809 Mon Sep 17 00:00:00 2001 From: typotter Date: Tue, 28 Apr 2026 13:27:07 -0600 Subject: [PATCH 3/3] fix(ffe): update ConfigurationPoller Javadoc to include 400/404 error codes --- .../main/java/datadog/remoteconfig/ConfigurationPoller.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java b/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java index 0e0fd58ef15..fda5aa9383c 100644 --- a/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java +++ b/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/ConfigurationPoller.java @@ -35,8 +35,8 @@ void addListener( void stop(); /** - * Registers a listener that is called when a non-retryable HTTP error (e.g. 401, 403) is - * received from the RC endpoint. The default implementation is a no-op. + * Registers a listener that is called when a non-retryable HTTP error (e.g. 400, 401, 403, 404) + * is received from the RC endpoint. The default implementation is a no-op. */ default void addNonRetryableErrorListener(NonRetryableErrorListener listener) {}