Skip to content

Commit 0da29c6

Browse files
marandanetomarandaneto
authored
fix(flags): retry flag requests on transient network errors (#245)
* fix: retry feature flag requests on network errors * fix: align flags retry policy with transient errors * fix: make flags retry count configurable * address pr review feedback * address pr review feedback
1 parent ee2a189 commit 0da29c6

7 files changed

Lines changed: 334 additions & 6 deletions

File tree

‎.changeset/quiet-flags-retry.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'PostHog': patch
3+
---
4+
5+
Retry feature flag requests after transient network errors only. The feature flag request retry count defaults to 1 and can be set to 0 to disable retries.

‎src/PostHog/Api/PostHogApiClient.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,11 @@ public async Task<ApiResult> SendEventAsync(
142142

143143
PrepareAndMutatePayload(payload);
144144

145-
return await _httpClient.PostJsonAsync<FlagsApiResult>(
145+
return await _httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
146146
endpointUrl,
147147
payload,
148+
_timeProvider,
149+
_options.Value,
148150
cancellationToken);
149151
}
150152

‎src/PostHog/Config/PostHogOptions.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,12 @@ public string? ProjectApiKey
173173
/// </remarks>
174174
public int MaxRetries { get; set; } = 3;
175175

176+
/// <summary>
177+
/// The maximum number of retries for feature flag requests after transient network errors. (Default: 1)
178+
/// Set to 0 to disable feature flag request retries.
179+
/// </summary>
180+
public int FeatureFlagRequestMaxRetries { get; set; } = 1;
181+
176182
/// <summary>
177183
/// The initial delay between retries. (Default: 1 second)
178184
/// </summary>

‎src/PostHog/Library/HttpClientExtensions.cs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.IO.Compression;
22
using System.Net;
33
using System.Net.Http.Json;
4+
using System.Net.Sockets;
45
using System.Text.Json;
56
using PostHog.Api;
67
using PostHog.Json;
@@ -42,6 +43,84 @@ internal static class HttpClientExtensions
4243
cancellationToken: cancellationToken);
4344
}
4445

46+
/// <summary>
47+
/// Sends a POST request with retry logic only for network/transport failures and timeouts.
48+
/// Non-successful HTTP responses are not retried.
49+
/// </summary>
50+
public static async Task<TBody?> PostJsonWithNetworkRetryAsync<TBody>(
51+
this HttpClient httpClient,
52+
Uri requestUri,
53+
object content,
54+
TimeProvider timeProvider,
55+
PostHogOptions options,
56+
CancellationToken cancellationToken)
57+
{
58+
var maxRetries = options.FeatureFlagRequestMaxRetries;
59+
var currentDelay = options.InitialRetryDelay;
60+
var maxDelay = options.MaxRetryDelay;
61+
var attempt = 0;
62+
63+
while (true)
64+
{
65+
attempt++;
66+
67+
HttpResponseMessage response;
68+
try
69+
{
70+
response = await httpClient.PostAsJsonAsync(
71+
requestUri,
72+
content,
73+
JsonSerializerHelper.Options,
74+
cancellationToken);
75+
}
76+
catch (HttpRequestException e) when (attempt <= maxRetries && IsRetryableFlagsHttpRequestException(e))
77+
{
78+
await Delay(timeProvider, currentDelay > maxDelay ? maxDelay : currentDelay, cancellationToken);
79+
currentDelay = DoubleWithCap(currentDelay, maxDelay);
80+
continue;
81+
}
82+
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested && attempt <= maxRetries)
83+
{
84+
await Delay(timeProvider, currentDelay > maxDelay ? maxDelay : currentDelay, cancellationToken);
85+
currentDelay = DoubleWithCap(currentDelay, maxDelay);
86+
continue;
87+
}
88+
89+
// Response processing is outside the try-catch so that exceptions from
90+
// EnsureSuccessfulApiCall (which may return HttpRequestException for 404s) won't
91+
// be caught by the retry logic above.
92+
using (response)
93+
{
94+
await response.EnsureSuccessfulApiCall(cancellationToken);
95+
96+
var result = await response.Content.ReadAsStreamAsync(cancellationToken);
97+
return await JsonSerializerHelper.DeserializeFromCamelCaseJsonAsync<TBody>(
98+
result,
99+
cancellationToken: cancellationToken);
100+
}
101+
}
102+
}
103+
104+
static bool IsRetryableFlagsHttpRequestException(HttpRequestException exception)
105+
{
106+
for (Exception? current = exception; current != null; current = current.InnerException)
107+
{
108+
if (current is SocketException socketException)
109+
{
110+
return socketException.SocketErrorCode is SocketError.ConnectionReset
111+
or SocketError.NetworkReset
112+
or SocketError.TimedOut;
113+
}
114+
115+
if (current is EndOfStreamException)
116+
{
117+
return true;
118+
}
119+
}
120+
121+
return false;
122+
}
123+
45124
/// <summary>
46125
/// Sends a POST request with retry logic for transient failures.
47126
/// Retries on 5xx, 408 (Request Timeout), and 429 (Too Many Requests) status codes.

‎src/PostHog/PublicAPI.Unshipped.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
#nullable enable
2+
PostHog.PostHogOptions.FeatureFlagRequestMaxRetries.get -> int
3+
PostHog.PostHogOptions.FeatureFlagRequestMaxRetries.set -> void

‎tests/UnitTests/Features/FeatureFlagsTests.cs‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3978,11 +3978,23 @@ public async Task DoesNotIncludeErrorPropertyWhenNoErrors()
39783978

39793979
[Fact]
39803980
public async Task IncludesTimeoutErrorWhenRequestTimesOut()
3981-
=> await AssertCapturedFeatureFlagErrorAsync(
3982-
handler => handler.AddFlagsResponseException(new TaskCanceledException("Request timed out")),
3983-
featureKey: "some-flag",
3984-
expectedResult: false,
3985-
expectedError: "timeout");
3981+
{
3982+
var container = new TestContainer(services => services.Configure<PostHogOptions>(options =>
3983+
{
3984+
options.FeatureFlagRequestMaxRetries = 0;
3985+
}));
3986+
container.FakeHttpMessageHandler.AddFlagsResponseException(new TaskCanceledException("Request timed out"));
3987+
var captureRequestHandler = container.FakeHttpMessageHandler.AddBatchResponse();
3988+
var client = container.Activate<PostHogClient>();
3989+
3990+
var result = await client.GetFeatureFlagAsync("some-flag", "distinct-id");
3991+
3992+
Assert.NotNull(result);
3993+
Assert.False(result.IsEnabled);
3994+
await client.FlushAsync();
3995+
var received = captureRequestHandler.GetReceivedRequestBody(indented: true);
3996+
Assert.Contains("\"$feature_flag_error\": \"timeout\"", received, StringComparison.Ordinal);
3997+
}
39863998

39873999
[Fact]
39884000
public async Task IncludesConnectionErrorWhenNetworkFails()

‎tests/UnitTests/Library/HttpClientExtensionsTests.cs‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.IO.Compression;
22
using System.Net;
33
using System.Net.Http.Headers;
4+
using System.Net.Sockets;
45
using System.Text;
56
using System.Text.Json;
67
using Microsoft.Extensions.Time.Testing;
@@ -549,6 +550,227 @@ await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
549550
#endif
550551
}
551552

553+
public class ThePostJsonWithNetworkRetryAsyncMethod
554+
{
555+
static readonly Uri FlagsUrl = new("https://us.i.posthog.com/flags/?v=2");
556+
557+
static PostHogOptions CreateOptions(int maxRetries = 3) => new()
558+
{
559+
ProjectToken = "test-api-key",
560+
MaxRetries = maxRetries,
561+
FeatureFlagRequestMaxRetries = maxRetries,
562+
InitialRetryDelay = TimeSpan.FromMilliseconds(1),
563+
MaxRetryDelay = TimeSpan.FromSeconds(30)
564+
};
565+
566+
static HttpClient CreateHttpClient(FakeRetryHttpMessageHandler handler)
567+
=> new(handler) { BaseAddress = new Uri("https://us.i.posthog.com") };
568+
569+
[Fact]
570+
public async Task RetriesOnConnectionResetThenSucceeds()
571+
{
572+
var handler = new FakeRetryHttpMessageHandler();
573+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
574+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
575+
using var httpClient = CreateHttpClient(handler);
576+
var options = CreateOptions();
577+
var timeProvider = new FakeTimeProvider();
578+
579+
var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
580+
FlagsUrl,
581+
new { api_key = "test", distinct_id = "user-1" },
582+
timeProvider,
583+
options,
584+
CancellationToken.None);
585+
586+
await handler.WaitForRequestCountAsync(1);
587+
timeProvider.Advance(TimeSpan.FromSeconds(1));
588+
var result = await task;
589+
590+
Assert.NotNull(result);
591+
Assert.Equal(2, handler.RequestCount);
592+
}
593+
594+
[Fact]
595+
public async Task RetriesUntilSuccessAfterMultipleConnectionResetErrors()
596+
{
597+
var handler = new FakeRetryHttpMessageHandler();
598+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
599+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
600+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
601+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
602+
using var httpClient = CreateHttpClient(handler);
603+
var options = CreateOptions(maxRetries: 3);
604+
var timeProvider = new FakeTimeProvider();
605+
606+
var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
607+
FlagsUrl,
608+
new { api_key = "test", distinct_id = "user-1" },
609+
timeProvider,
610+
options,
611+
CancellationToken.None);
612+
613+
for (var i = 1; i <= 4 && !task.IsCompleted; i++)
614+
{
615+
await handler.WaitForRequestCountAsync(i);
616+
timeProvider.Advance(TimeSpan.FromMilliseconds(50));
617+
}
618+
619+
var result = await task;
620+
621+
Assert.NotNull(result);
622+
Assert.Equal(4, handler.RequestCount);
623+
}
624+
625+
[Fact]
626+
public async Task ThrowsAfterMaxRetriesWhenConnectionResetPersists()
627+
{
628+
var handler = new FakeRetryHttpMessageHandler();
629+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
630+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
631+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
632+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
633+
using var httpClient = CreateHttpClient(handler);
634+
var options = CreateOptions(maxRetries: 3);
635+
var timeProvider = new FakeTimeProvider();
636+
637+
var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
638+
FlagsUrl,
639+
new { api_key = "test", distinct_id = "user-1" },
640+
timeProvider,
641+
options,
642+
CancellationToken.None);
643+
644+
for (var i = 1; i <= 4 && !task.IsCompleted; i++)
645+
{
646+
await handler.WaitForRequestCountAsync(i);
647+
timeProvider.Advance(TimeSpan.FromMilliseconds(50));
648+
}
649+
650+
await Assert.ThrowsAsync<HttpRequestException>(() => task);
651+
Assert.Equal(4, handler.RequestCount);
652+
}
653+
654+
[Fact]
655+
public async Task DoesNotRetryWhenFeatureFlagRequestMaxRetriesIsZero()
656+
{
657+
var handler = new FakeRetryHttpMessageHandler();
658+
handler.AddException(new HttpRequestException("Connection reset", new SocketException((int)SocketError.ConnectionReset)));
659+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
660+
using var httpClient = CreateHttpClient(handler);
661+
var options = CreateOptions(maxRetries: 0);
662+
var timeProvider = new FakeTimeProvider();
663+
664+
await Assert.ThrowsAsync<HttpRequestException>(() =>
665+
httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
666+
FlagsUrl,
667+
new { api_key = "test", distinct_id = "user-1" },
668+
timeProvider,
669+
options,
670+
CancellationToken.None));
671+
672+
Assert.Equal(1, handler.RequestCount);
673+
}
674+
675+
#if NET8_0_OR_GREATER
676+
[Fact]
677+
public async Task DoesNotRetryOnUserCancellation()
678+
{
679+
var handler = new FakeRetryHttpMessageHandler();
680+
using var cts = new CancellationTokenSource();
681+
await cts.CancelAsync();
682+
handler.AddException(new TaskCanceledException("Operation was canceled.", null, cts.Token));
683+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
684+
using var httpClient = CreateHttpClient(handler);
685+
var options = CreateOptions();
686+
var timeProvider = new FakeTimeProvider();
687+
688+
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
689+
httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
690+
FlagsUrl,
691+
new { api_key = "test", distinct_id = "user-1" },
692+
timeProvider,
693+
options,
694+
cts.Token));
695+
696+
Assert.Equal(1, handler.RequestCount);
697+
}
698+
#endif
699+
700+
[Fact]
701+
public async Task DoesNotRetryConnectionRefused()
702+
{
703+
var handler = new FakeRetryHttpMessageHandler();
704+
handler.AddException(new HttpRequestException("Connection refused", new SocketException((int)SocketError.ConnectionRefused)));
705+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
706+
using var httpClient = CreateHttpClient(handler);
707+
var options = CreateOptions();
708+
var timeProvider = new FakeTimeProvider();
709+
710+
await Assert.ThrowsAsync<HttpRequestException>(() =>
711+
httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
712+
FlagsUrl,
713+
new { api_key = "test", distinct_id = "user-1" },
714+
timeProvider,
715+
options,
716+
CancellationToken.None));
717+
718+
Assert.Equal(1, handler.RequestCount);
719+
}
720+
721+
[Fact]
722+
public async Task RetriesOnTaskCanceledExceptionFromTimeoutThenSucceeds()
723+
{
724+
var handler = new FakeRetryHttpMessageHandler();
725+
handler.AddException(new TaskCanceledException("The request timed out."));
726+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } });
727+
using var httpClient = CreateHttpClient(handler);
728+
var options = CreateOptions();
729+
var timeProvider = new FakeTimeProvider();
730+
731+
var task = httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
732+
FlagsUrl,
733+
new { api_key = "test", distinct_id = "user-1" },
734+
timeProvider,
735+
options,
736+
CancellationToken.None);
737+
738+
await handler.WaitForRequestCountAsync(1);
739+
timeProvider.Advance(TimeSpan.FromSeconds(1));
740+
var result = await task;
741+
742+
Assert.NotNull(result);
743+
Assert.Equal(2, handler.RequestCount);
744+
}
745+
746+
[Theory]
747+
[InlineData(HttpStatusCode.RequestTimeout)] // 408
748+
[InlineData(HttpStatusCode.TooManyRequests)] // 429
749+
[InlineData(HttpStatusCode.InternalServerError)] // 500
750+
[InlineData(HttpStatusCode.BadGateway)] // 502
751+
[InlineData(HttpStatusCode.ServiceUnavailable)] // 503
752+
[InlineData(HttpStatusCode.GatewayTimeout)] // 504
753+
public async Task DoesNotRetryOnHttpErrorStatusCodes(HttpStatusCode statusCode)
754+
{
755+
var handler = new FakeRetryHttpMessageHandler();
756+
handler.AddResponse(statusCode, new { type = "error", detail = "server error" });
757+
handler.AddResponse(HttpStatusCode.OK, new { flags = new { } }); // Should never be reached
758+
using var httpClient = CreateHttpClient(handler);
759+
var options = CreateOptions();
760+
var timeProvider = new FakeTimeProvider();
761+
762+
await Assert.ThrowsAsync<ApiException>(() =>
763+
httpClient.PostJsonWithNetworkRetryAsync<FlagsApiResult>(
764+
FlagsUrl,
765+
new { api_key = "test", distinct_id = "user-1" },
766+
timeProvider,
767+
options,
768+
CancellationToken.None));
769+
770+
Assert.Equal(1, handler.RequestCount);
771+
}
772+
}
773+
552774
public class ThePostCompressedJsonAsyncMethod
553775
{
554776
static readonly Uri BatchUrl = new("https://us.i.posthog.com/batch");

0 commit comments

Comments
 (0)