fix(audience-sdk): treat HTTP 429 as retryable with Retry-After backoff

ImmutableJeffrey · ImmutableJeffrey · commit 2e36abf1f5c8 · 2026-04-29T15:46:44.000+10:00
Fixes a real SDK bug surfaced by the SDK-148 live-fire test suite: every
4xx — including 429 (Too Many Requests) — was treated as a permanent
"server rejected your data, retry won't help" failure. Per RFC 6585, 429
is the canonical retryable 4xx and must honor the Retry-After header.

Changes:
- HttpTransport: split 429 out of the 4xx delete-and-error branch. Keep
  the batch on disk, honor Retry-After (delta-seconds or HTTP-date) if
  present, otherwise apply the existing exponential schedule (5/10/20/40/60s).
  Do NOT fire onError — the next flush tick retries automatically once
  the backoff window expires. Persistent rate-limiting manifests as a
  growing on-disk queue, the correct studio-actionable signal.
- ImmutableAudience consent sync: wrap the PUT in a 4-attempt retry loop
  with 1s/2s/4s backoff between attempts (or Retry-After if the server
  supplies one). ConsentSyncFailed only fires after the budget exhausts.
- HttpRetry helper class for shared Retry-After parsing across batch and
  consent-sync paths.

Tests:
- HttpTransport: 429 keeps batch + sets backoff + no onError; Retry-After
  delta-seconds drives NextAttemptAt; HTTP-date variant engages backoff
  window; past Retry-After falls through to exponential; 429-then-200
  delivers the batch and clears backoff.
- ConsentSync: 429-then-2xx is invisible to onError; 429×4 surfaces
  ConsentSyncFailed after the full retry budget.
diff --git a/src/Packages/Audience/Runtime/ImmutableAudience.cs b/src/Packages/Audience/Runtime/ImmutableAudience.cs
@@ -565,17 +565,33 @@ private static void SyncConsentToBackend(AudienceConfig config, ConsentLevel lev
 
             Task.Run(async () =>
             {
+                // 429 retried up to 4 attempts (1s/2s/4s or Retry-After).
+                // Other non-2xx fail fast.
+                const int maxAttempts = 4;
+                var attempt = 0;
                 try
                 {
-                    using var request = new HttpRequestMessage(HttpMethod.Put, url);
-                    request.Headers.Add(Constants.PublishableKeyHeader, publishableKey);
-                    request.Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json");
-                    using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
-
-                    if (!response.IsSuccessStatusCode)
+                    while (true)
                     {
+                        attempt++;
+                        using var request = new HttpRequestMessage(HttpMethod.Put, url);
+                        request.Headers.Add(Constants.PublishableKeyHeader, publishableKey);
+                        request.Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json");
+                        using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
+
+                        if (response.IsSuccessStatusCode) return;
+
+                        if ((int)response.StatusCode == 429 && attempt < maxAttempts)
+                        {
+                            var delay = HttpRetry.ParseRetryAfter(response)
+                                ?? TimeSpan.FromMilliseconds(1_000 * (1 << (attempt - 1)));
+                            await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
+                            continue;
+                        }
+
                         NotifyErrorCallback(onError, AudienceErrorCode.ConsentSyncFailed,
                             $"Consent sync failed with status {(int)response.StatusCode}");
+                        return;
                     }
                 }
                 catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
diff --git a/src/Packages/Audience/Runtime/Transport/HttpRetry.cs b/src/Packages/Audience/Runtime/Transport/HttpRetry.cs
@@ -0,0 +1,24 @@
+#nullable enable
+
+using System;
+using System.Net.Http;
+
+namespace Immutable.Audience
+{
+    internal static class HttpRetry
+    {
+        // Past HTTP-date returns null so callers fall through to their default backoff.
+        internal static TimeSpan? ParseRetryAfter(HttpResponseMessage response)
+        {
+            var ra = response.Headers.RetryAfter;
+            if (ra == null) return null;
+            if (ra.Delta.HasValue) return ra.Delta.Value;
+            if (ra.Date.HasValue)
+            {
+                var d = ra.Date.Value - DateTimeOffset.UtcNow;
+                return d > TimeSpan.Zero ? d : (TimeSpan?)null;
+            }
+            return null;
+        }
+    }
+}
diff --git a/src/Packages/Audience/Runtime/Transport/HttpRetry.cs.meta b/src/Packages/Audience/Runtime/Transport/HttpRetry.cs.meta
diff --git a/src/Packages/Audience/Runtime/Transport/HttpTransport.cs b/src/Packages/Audience/Runtime/Transport/HttpTransport.cs
@@ -114,13 +114,26 @@ internal async Task<bool> SendBatchAsync(CancellationToken ct = default)
                             $"Batch partially rejected: {rejected} of {batch.Count} events dropped");
                     }
                 }
+                else if (statusCode == 429)
+                {
+                    // 429 is retryable (RFC 6585). Keep the batch, honor Retry-After
+                    // if present else use the existing 5xx backoff schedule. No
+                    // onError — next flush tick retries; persistent rate-limits
+                    // surface as a growing on-disk queue.
+                    var retryAfter = HttpRetry.ParseRetryAfter(response);
+                    if (retryAfter.HasValue)
+                        SetBackoffUntil(_getUtcNow() + retryAfter.Value);
+                    else
+                        RecordFailure();
+                }
                 else if (statusCode >= 400 && statusCode < 500)
                 {
-                    // 4xx: server rejected the payload. Drop it (retry won't help) and
-                    // reset backoff — server is healthy, our data was the problem.
-                    // Capture the response body so the caller's OnError surfaces
-                    // the server's reason string ("unknown publishable key",
-                    // "missing field X", etc.) rather than a bare status code.
+                    // 4xx (non-429): server rejected the payload. Drop it (retry
+                    // won't help) and reset backoff — server is healthy, our data
+                    // was the problem. Capture the response body so the caller's
+                    // OnError surfaces the server's reason string ("unknown
+                    // publishable key", "missing field X", etc.) rather than a
+                    // bare status code.
                     var rejectionBody = await ReadBodyForErrorAsync(response).ConfigureAwait(false);
                     _store.Delete(batch);
                     ResetBackoff();
@@ -205,6 +218,16 @@ private void RecordFailure()
             }
         }
 
+        // Server-supplied Retry-After is authoritative; bypasses BackoffMsLocked.
+        private void SetBackoffUntil(DateTime nextAt)
+        {
+            lock (_backoffLock)
+            {
+                _consecutiveFailures++;
+                _nextAttemptAt = nextAt;
+            }
+        }
+
         private void ResetBackoff()
         {
             lock (_backoffLock)
diff --git a/src/Packages/Audience/Tests/Runtime/ConsentSyncTests.cs b/src/Packages/Audience/Tests/Runtime/ConsentSyncTests.cs
@@ -98,6 +98,70 @@ public void SetConsent_PutFailure_InvokesOnErrorWithConsentSyncFailed()
             StringAssert.Contains("500", captured.Message);
         }
 
+        [Test]
+        public void SetConsent_429ThenSuccess_DoesNotFireConsentSyncFailed()
+        {
+            var handler = new CapturingHandler
+            {
+                StatusSequence = new[] { (HttpStatusCode)429, HttpStatusCode.NoContent },
+                RetryAfterSeconds = 0,
+            };
+            AudienceError captured = null;
+            var received = new ManualResetEventSlim(false);
+
+            var config = MakeConfig(handler, ConsentLevel.Anonymous);
+            config.OnError = err =>
+            {
+                if (err.Code == AudienceErrorCode.ConsentSyncFailed)
+                {
+                    captured = err;
+                    received.Set();
+                }
+            };
+            ImmutableAudience.Init(config);
+
+            ImmutableAudience.SetConsent(ConsentLevel.Full);
+
+            // Wait long enough for both attempts (Retry-After: 0).
+            Assert.IsFalse(received.Wait(TimeSpan.FromSeconds(3)),
+                "transient 429 followed by 2xx must not surface ConsentSyncFailed");
+            Assert.IsNull(captured);
+            Assert.GreaterOrEqual(handler.PutCount, 2,
+                "429 must trigger at least one retry");
+        }
+
+        [Test]
+        public void SetConsent_429Repeated_FiresConsentSyncFailedAfterRetries()
+        {
+            // RetryAfterSeconds=0 collapses the 1s/2s/4s production cadence
+            // so the test runs in milliseconds.
+            var handler = new CapturingHandler
+            {
+                Status = (HttpStatusCode)429,
+                RetryAfterSeconds = 0,
+            };
+            AudienceError captured = null;
+            var received = new ManualResetEventSlim(false);
+
+            var config = MakeConfig(handler, ConsentLevel.Anonymous);
+            config.OnError = err =>
+            {
+                if (err.Code == AudienceErrorCode.ConsentSyncFailed)
+                {
+                    captured = err;
+                    received.Set();
+                }
+            };
+            ImmutableAudience.Init(config);
+
+            ImmutableAudience.SetConsent(ConsentLevel.Full);
+
+            Assert.IsTrue(received.Wait(TimeSpan.FromSeconds(5)),
+                "exhausted 429 retries must surface ConsentSyncFailed");
+            StringAssert.Contains("429", captured.Message);
+            Assert.AreEqual(4, handler.PutCount, "must have made the full 4 attempts");
+        }
+
         private AudienceConfig MakeConfig(CapturingHandler handler, ConsentLevel consent) =>
             new AudienceConfig
             {
@@ -128,11 +192,20 @@ private class CapturingHandler : HttpMessageHandler
             internal CapturedRequest LastPut;
             internal HttpStatusCode Status { get; set; } = HttpStatusCode.NoContent;
 
+            // One status per call; falls back to Status once exhausted.
+            internal HttpStatusCode[] StatusSequence { get; set; }
+
+            // Adds Retry-After: <seconds> to 429 responses (0 = retry now).
+            internal int? RetryAfterSeconds { get; set; }
+
+            internal int PutCount { get; private set; }
+
             protected override async Task<HttpResponseMessage> SendAsync(
                 HttpRequestMessage request, CancellationToken ct)
             {
                 if (request.Method == HttpMethod.Put)
                 {
+                    PutCount++;
                     LastPut = new CapturedRequest
                     {
                         Url = request.RequestUri!.ToString(),
@@ -142,7 +215,17 @@ protected override async Task<HttpResponseMessage> SendAsync(
                     };
                     PutReceived.Set();
                 }
-                return new HttpResponseMessage(Status);
+
+                var status = StatusSequence != null && PutCount - 1 < StatusSequence.Length
+                    ? StatusSequence[PutCount - 1]
+                    : Status;
+
+                var response = new HttpResponseMessage(status);
+                if ((int)status == 429 && RetryAfterSeconds.HasValue)
+                {
+                    response.Headers.Add("Retry-After", RetryAfterSeconds.Value.ToString());
+                }
+                return response;
             }
         }
     }
diff --git a/src/Packages/Audience/Tests/Runtime/Transport/HttpTransportTests.cs b/src/Packages/Audience/Tests/Runtime/Transport/HttpTransportTests.cs
@@ -203,6 +203,118 @@ public async Task SendBatchAsync_4xx_DeletesFilesAndResetsBackoff()
             Assert.AreEqual(AudienceErrorCode.ValidationRejected, reportedError!.Code);
         }
 
+        [Test]
+        public async Task SendBatchAsync_429_NoRetryAfter_KeepsFilesAndUsesExpoBackoff_NoError()
+        {
+            _store.Write("{\"type\":\"track\"}");
+
+            var handler = new MockHandler((HttpStatusCode)429, "");
+            AudienceError? reportedError = null;
+            using var transport = new HttpTransport(_store, "pk_imapik-test-key1",
+                onError: e => reportedError = e, handler: handler, getUtcNow: _getUtcNow);
+
+            await transport.SendBatchAsync();
+
+            Assert.AreEqual(1, _store.Count(), "429 must keep files for retry");
+            Assert.IsTrue(transport.IsInBackoffWindow);
+            Assert.AreEqual(5_000, transport.BackoffMs);
+            Assert.IsNull(reportedError, "429 is transient — must not fire onError");
+        }
+
+        [Test]
+        public async Task SendBatchAsync_429_RetryAfterDeltaSeconds_OverridesExpoBackoff()
+        {
+            _store.Write("{\"type\":\"track\"}");
+
+            var handler = new MockHandler(() =>
+            {
+                var resp = new HttpResponseMessage((HttpStatusCode)429);
+                resp.Headers.Add("Retry-After", "12");
+                return resp;
+            });
+            using var transport = new HttpTransport(_store, "pk_imapik-test-key1",
+                handler: handler, getUtcNow: _getUtcNow);
+
+            await transport.SendBatchAsync();
+
+            Assert.IsTrue(transport.IsInBackoffWindow);
+            Assert.AreEqual(_utcNow.AddSeconds(12), transport.NextAttemptAt);
+        }
+
+        [Test]
+        public async Task SendBatchAsync_429_RetryAfterHttpDate_OverridesExpoBackoff()
+        {
+            // ParseRetryAfter computes the delta against DateTimeOffset.UtcNow,
+            // which we can't pin from outside; assert only that a future date
+            // engages the window. The seconds-form test above pins exact math.
+            _store.Write("{\"type\":\"track\"}");
+
+            var handler = new MockHandler(() =>
+            {
+                var resp = new HttpResponseMessage((HttpStatusCode)429);
+                resp.Headers.Add("Retry-After", DateTimeOffset.UtcNow.AddSeconds(20).ToString("R"));
+                return resp;
+            });
+            using var transport = new HttpTransport(_store, "pk_imapik-test-key1",
+                handler: handler, getUtcNow: _getUtcNow);
+
+            await transport.SendBatchAsync();
+
+            Assert.AreEqual(1, _store.Count());
+            Assert.IsTrue(transport.IsInBackoffWindow);
+        }
+
+        [Test]
+        public async Task SendBatchAsync_429_PastRetryAfterDate_FallsBackToExpoBackoff()
+        {
+            // Past Retry-After (clock skew or server bug) must not let
+            // IsInBackoffWindow flip false and trigger instant retry.
+            _store.Write("{\"type\":\"track\"}");
+
+            var handler = new MockHandler(() =>
+            {
+                var resp = new HttpResponseMessage((HttpStatusCode)429);
+                resp.Headers.Add("Retry-After", DateTimeOffset.UtcNow.AddSeconds(-30).ToString("R"));
+                return resp;
+            });
+            using var transport = new HttpTransport(_store, "pk_imapik-test-key1",
+                handler: handler, getUtcNow: _getUtcNow);
+
+            await transport.SendBatchAsync();
+
+            Assert.AreEqual(5_000, transport.BackoffMs);
+            Assert.IsTrue(transport.IsInBackoffWindow);
+        }
+
+        [Test]
+        public async Task SendBatchAsync_429ThenSuccess_DeliversBatchAndClearsBackoff()
+        {
+            _store.Write("{\"type\":\"track\"}");
+
+            var callCount = 0;
+            var handler = new MockHandler(() =>
+            {
+                callCount++;
+                return callCount == 1
+                    ? new HttpResponseMessage((HttpStatusCode)429)
+                    : new HttpResponseMessage(HttpStatusCode.OK)
+                    { Content = new StringContent("{\"accepted\":1,\"rejected\":0}") };
+            });
+            AudienceError? reportedError = null;
+            using var transport = new HttpTransport(_store, "pk_imapik-test-key1",
+                onError: e => reportedError = e, handler: handler, getUtcNow: _getUtcNow);
+
+            await transport.SendBatchAsync();
+            Assert.AreEqual(1, _store.Count(), "429 keeps the batch");
+            Assert.AreEqual(5_000, transport.BackoffMs);
+
+            Advance(5_001);
+            await transport.SendBatchAsync();
+            Assert.AreEqual(0, _store.Count(), "200 on retry deletes the batch");
+            Assert.AreEqual(0, transport.BackoffMs, "backoff resets after success");
+            Assert.IsNull(reportedError, "neither 429 nor success must fire onError");
+        }
+
         [Test]
         public async Task SendBatchAsync_200_WithRejected_DeletesFilesAndSurfacesValidationRejected()
         {
