fix(flags): retry flag requests on transient network errors

.changeset/quiet-flags-retry.md

@@ -0,0 +1,5 @@
+---
+'PostHog': patch
+---
+
+Retry feature flag requests after transient network errors only. The feature flag request retry count defaults to 1 and can be set to 0 to disable retries.

src/PostHog/Api/PostHogApiClient.cs

@@ -142,9 +142,11 @@ public async Task<ApiResult> SendEventAsync(
 
         PrepareAndMutatePayload(payload);
 
-        return await _httpClient.PostJsonAsync<FlagsApiResult>(
+        return await _httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
             endpointUrl,
             payload,
+            _timeProvider,
+            _options.Value,
             cancellationToken);
     }
 

src/PostHog/Config/PostHogOptions.cs

@@ -173,6 +173,12 @@ public string? ProjectApiKey
     /// </remarks>
     public int MaxRetries { get; set; } = 3;
 
+    /// <summary>
+    /// The maximum number of retries for feature flag requests after transient network errors. (Default: 1)
+    /// Set to 0 to disable feature flag request retries.
+    /// </summary>
+    public int FeatureFlagRequestMaxRetries { get; set; } = 1;
+
     /// <summary>
     /// The initial delay between retries. (Default: 1 second)
     /// </summary>

src/PostHog/Library/HttpClientExtensions.cs

@@ -1,6 +1,7 @@
 using System.IO.Compression;
 using System.Net;
 using System.Net.Http.Json;
+using System.Net.Sockets;
 using System.Text.Json;
 using PostHog.Api;
 using PostHog.Json;
@@ -42,6 +43,84 @@ internal static class HttpClientExtensions
             cancellationToken: cancellationToken);
     }
 
+    /// <summary>
+    /// Sends a POST request with retry logic only for network/transport failures and timeouts.
+    /// Non-successful HTTP responses are not retried.
+    /// </summary>
+    public static async Task<TBody?> PostJsonWithNetworkRetryAsync<TBody>(
+        this HttpClient httpClient,
+        Uri requestUri,
+        object content,
+        TimeProvider timeProvider,
+        PostHogOptions options,
+        CancellationToken cancellationToken)
+    {
+        var maxRetries = options.FeatureFlagRequestMaxRetries;
+        var currentDelay = options.InitialRetryDelay;
+        var maxDelay = options.MaxRetryDelay;
+        var attempt = 0;
+
+        while (true)
+        {
+            attempt++;
+
+            HttpResponseMessage response;
+            try
+            {
+                response = await httpClient.PostAsJsonAsync(
+                    requestUri,
+                    content,
+                    JsonSerializerHelper.Options,
+                    cancellationToken);
+            }
+            catch (HttpRequestException e) when (attempt <= maxRetries && IsRetryableFlagsHttpRequestException(e))
+            {
+                await Delay(timeProvider, currentDelay > maxDelay ? maxDelay : currentDelay, cancellationToken);
+                currentDelay = DoubleWithCap(currentDelay, maxDelay);
+                continue;
+            }
+            catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested && attempt <= maxRetries)
+            {
+                await Delay(timeProvider, currentDelay > maxDelay ? maxDelay : currentDelay, cancellationToken);
+                currentDelay = DoubleWithCap(currentDelay, maxDelay);
+                continue;
+            }
+
+            // Response processing is outside the try-catch so that exceptions from
+            // EnsureSuccessfulApiCall (which may return HttpRequestException for 404s) won't
+            // be caught by the retry logic above.
+            using (response)
+            {
+                await response.EnsureSuccessfulApiCall(cancellationToken);
+
+                var result = await response.Content.ReadAsStreamAsync(cancellationToken);
+                return await JsonSerializerHelper.DeserializeFromCamelCaseJsonAsync<TBody>(
+                    result,
+                    cancellationToken: cancellationToken);
+            }
+        }
+    }
+
+    static bool IsRetryableFlagsHttpRequestException(HttpRequestException exception)
+    {
+        for (Exception? current = exception; current != null; current = current.InnerException)
+        {
+            if (current is SocketException socketException)
+            {
+                return socketException.SocketErrorCode is SocketError.ConnectionReset
+                    or SocketError.NetworkReset
+                    or SocketError.TimedOut;
+            }
+
+            if (current is EndOfStreamException)
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /// <summary>
     /// Sends a POST request with retry logic for transient failures.
     /// Retries on 5xx, 408 (Request Timeout), and 429 (Too Many Requests) status codes.

src/PostHog/PublicAPI.Unshipped.txt

@@ -1 +1,3 @@
 #nullable enable
+PostHog.PostHogOptions.FeatureFlagRequestMaxRetries.get -> int
+PostHog.PostHogOptions.FeatureFlagRequestMaxRetries.set -> void

tests/UnitTests/Features/FeatureFlagsTests.cs

@@ -3978,11 +3978,23 @@ public async Task DoesNotIncludeErrorPropertyWhenNoErrors()
 
     [Fact]
     public async Task IncludesTimeoutErrorWhenRequestTimesOut()
-        => await AssertCapturedFeatureFlagErrorAsync(
-            handler => handler.AddFlagsResponseException(new TaskCanceledException("Request timed out")),
-            featureKey: "some-flag",
-            expectedResult: false,
-            expectedError: "timeout");
+    {
+        var container = new TestContainer(services => services.Configure<PostHogOptions>(options =>
+        {
+            options.FeatureFlagRequestMaxRetries = 0;
+        }));
+        container.FakeHttpMessageHandler.AddFlagsResponseException(new TaskCanceledException("Request timed out"));
+        var captureRequestHandler = container.FakeHttpMessageHandler.AddBatchResponse();
+        var client = container.Activate<PostHogClient>();
+
+        var result = await client.GetFeatureFlagAsync("some-flag", "distinct-id");
+
+        Assert.NotNull(result);
+        Assert.False(result.IsEnabled);
+        await client.FlushAsync();
+        var received = captureRequestHandler.GetReceivedRequestBody(indented: true);
+        Assert.Contains("\"$feature_flag_error\": \"timeout\"", received, StringComparison.Ordinal);
+    }
 
     [Fact]
     public async Task IncludesConnectionErrorWhenNetworkFails()

tests/UnitTests/Library/HttpClientExtensionsTests.cs

@@ -1,6 +1,7 @@
 using System.IO.Compression;
 using System.Net;
 using System.Net.Http.Headers;
+using System.Net.Sockets;
 using System.Text;
 using System.Text.Json;
 using Microsoft.Extensions.Time.Testing;
@@ -549,6 +550,227 @@ await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
 #endif
 }
 
+public class ThePostJsonWithNetworkRetryAsyncMethod
+{
+    static readonly Uri FlagsUrl = new("https://us.i.posthog.com/flags/?v=2");
+
+    static PostHogOptions CreateOptions(int maxRetries = 3) => new()
+    {
+        ProjectToken = "test-api-key",
+        MaxRetries = maxRetries,
+        FeatureFlagRequestMaxRetries = maxRetries,
+        InitialRetryDelay = TimeSpan.FromMilliseconds(1),
+        MaxRetryDelay = TimeSpan.FromSeconds(30)
+    };
+
+    static HttpClient CreateHttpClient(FakeRetryHttpMessageHandler handler)
+        => new(handler) { BaseAddress = new Uri("https://us.i.posthog.com") };
+
+    [Fact]
+    public async Task RetriesOnConnectionResetThenSucceeds()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions();
+        var timeProvider = new FakeTimeProvider();
+
+        var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+            FlagsUrl,
+            new { api_key = "test", distinct_id = "user-1" },
+            timeProvider,
+            options,
+            CancellationToken.None);
+
+        await handler.WaitForRequestCountAsync(1);
+        timeProvider.Advance(TimeSpan.FromSeconds(1));
+        var result = await task;
+
+        Assert.NotNull(result);
+        Assert.Equal(2, handler.RequestCount);
+    }
+
+    [Fact]
+    public async Task RetriesUntilSuccessAfterMultipleConnectionResetErrors()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions(maxRetries: 3);
+        var timeProvider = new FakeTimeProvider();
+
+        var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+            FlagsUrl,
+            new { api_key = "test", distinct_id = "user-1" },
+            timeProvider,
+            options,
+            CancellationToken.None);
+
+        for (var i = 1; i <= 4 && !task.IsCompleted; i++)
+        {
+            await handler.WaitForRequestCountAsync(i);
+            timeProvider.Advance(TimeSpan.FromMilliseconds(50));
+        }
+
+        var result = await task;
+
+        Assert.NotNull(result);
+        Assert.Equal(4, handler.RequestCount);
+    }
+
+    [Fact]
+    public async Task ThrowsAfterMaxRetriesWhenConnectionResetPersists()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions(maxRetries: 3);
+        var timeProvider = new FakeTimeProvider();
+
+        var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+            FlagsUrl,
+            new { api_key = "test", distinct_id = "user-1" },
+            timeProvider,
+            options,
+            CancellationToken.None);
+
+        for (var i = 1; i <= 4 && !task.IsCompleted; i++)
+        {
+            await handler.WaitForRequestCountAsync(i);
+            timeProvider.Advance(TimeSpan.FromMilliseconds(50));
+        }
+
+        await Assert.ThrowsAsync<HttpRequestException>(() => task);
+        Assert.Equal(4, handler.RequestCount);
+    }
+
+    [Fact]
+    public async Task DoesNotRetryWhenFeatureFlagRequestMaxRetriesIsZero()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions(maxRetries: 0);
+        var timeProvider = new FakeTimeProvider();
+
+        await Assert.ThrowsAsync<HttpRequestException>(() =>
+            httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+                FlagsUrl,
+                new { api_key = "test", distinct_id = "user-1" },
+                timeProvider,
+                options,
+                CancellationToken.None));
+
+        Assert.Equal(1, handler.RequestCount);
+    }
+
+#if NET8_0_OR_GREATER
+    [Fact]
+    public async Task DoesNotRetryOnUserCancellation()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        using var cts = new CancellationTokenSource();
+        await cts.CancelAsync();
+        handler.AddException(new TaskCanceledException("Operation was canceled.", null, cts.Token));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions();
+        var timeProvider = new FakeTimeProvider();
+
+        await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
+            httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+                FlagsUrl,
+                new { api_key = "test", distinct_id = "user-1" },
+                timeProvider,
+                options,
+                cts.Token));
+
+        Assert.Equal(1, handler.RequestCount);
+    }
+#endif
+
+    [Fact]
+    public async Task DoesNotRetryConnectionRefused()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new HttpRequestException("Connection refused", new SocketException((int)SocketError.ConnectionRefused)));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions();
+        var timeProvider = new FakeTimeProvider();
+
+        await Assert.ThrowsAsync<HttpRequestException>(() =>
+            httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+                FlagsUrl,
+                new { api_key = "test", distinct_id = "user-1" },
+                timeProvider,
+                options,
+                CancellationToken.None));
+
+        Assert.Equal(1, handler.RequestCount);
+    }
+
+    [Fact]
+    public async Task RetriesOnTaskCanceledExceptionFromTimeoutThenSucceeds()
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddException(new TaskCanceledException("The request timed out."));
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions();
+        var timeProvider = new FakeTimeProvider();
+
+        var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+            FlagsUrl,
+            new { api_key = "test", distinct_id = "user-1" },
+            timeProvider,
+            options,
+            CancellationToken.None);
+
+        await handler.WaitForRequestCountAsync(1);
+        timeProvider.Advance(TimeSpan.FromSeconds(1));
+        var result = await task;
+
+        Assert.NotNull(result);
+        Assert.Equal(2, handler.RequestCount);
+    }
+
+    [Theory]
+    [InlineData(HttpStatusCode.RequestTimeout)] // 408
+    [InlineData(HttpStatusCode.TooManyRequests)] // 429
+    [InlineData(HttpStatusCode.InternalServerError)] // 500
+    [InlineData(HttpStatusCode.BadGateway)] // 502
+    [InlineData(HttpStatusCode.ServiceUnavailable)] // 503
+    [InlineData(HttpStatusCode.GatewayTimeout)] // 504
+    public async Task DoesNotRetryOnHttpErrorStatusCodes(HttpStatusCode statusCode)
+    {
+        var handler = new FakeRetryHttpMessageHandler();
+        handler.AddResponse(statusCode, new { type = "error", detail = "server error" });
+        handler.AddResponse(HttpStatusCode.OK, new { flags = new { } }); // Should never be reached
+        using var httpClient = CreateHttpClient(handler);
+        var options = CreateOptions();
+        var timeProvider = new FakeTimeProvider();
+
+        await Assert.ThrowsAsync<ApiException>(() =>
+            httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
+                FlagsUrl,
+                new { api_key = "test", distinct_id = "user-1" },
+                timeProvider,
+                options,
+                CancellationToken.None));
+
+        Assert.Equal(1, handler.RequestCount);
+    }
+}
+
 public class ThePostCompressedJsonAsyncMethod
 {
     static readonly Uri BatchUrl = new("https://us.i.posthog.com/batch");