CSHARP-6089: Client Backpressure with baseBackoffMS

src/MongoDB.Driver/Core/Connections/HelloHelper.cs

@@ -57,7 +57,7 @@ internal static BsonDocument CreateCommand(ServerApi serverApi, bool helloOk = f
                 { "topologyVersion", () => topologyVersion.ToBsonDocument(), topologyVersion != null },
                 { "maxAwaitTimeMS", () => (long)maxAwaitTime.Value.TotalMilliseconds, maxAwaitTime.HasValue },
                 { "loadBalanced", true, loadBalanced },
-                { "backpressure", true }
+                { "backpressure", "2" }
             };
         }
 

src/MongoDB.Driver/Core/Misc/Feature.cs

@@ -112,6 +112,11 @@ public class Feature
     /// </summary>
     public static Feature ChangeStreamSplitEventStage { get; } = new("ChangeStreamSplitEventStage", WireVersion.Server70);
 
+    /// <summary>
+    /// Gets the client backpressure baseBackoffMS feature.
+    /// </summary>
+    public static Feature ClientBackpressureBaseBackoffMs { get; } = new("ClientBackpressureBaseBackoffMs", WireVersion.Server90);
+
     /// <summary>
     /// Gets the client bulk write feature.
     /// </summary>

src/MongoDB.Driver/Core/Operations/AGENTS.md

@@ -95,7 +95,7 @@ Pinned channel/server attach on the first command. Recovery token (returned by m
 
 1. Call `operationContext.ThrowIfTimedOutOrCanceled()` at the top of each iteration.
 2. Execute one attempt against the binding.
-3. On retryable exception, deprioritize the failed server. For ordinary retryable errors there is no back-off — the loop retries immediately against a different server. For `SystemOverloadedError`-labelled exceptions an adaptive back-off (`Thread.Sleep` on the sync path, `Task.Delay` on the async path, computed by `RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random)`) is applied; on the sync path the loop bails early if the back-off would exceed the remaining CSOT deadline (explicit check in `RetryableReadOperationExecutor.cs` lines 66-68 / `RetryableWriteOperationExecutor.cs` lines 66-68: `if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) throw;`). The async path doesn't need an equivalent pre-check — `Task.Delay(backoff, operationContext.CancellationToken)` is interrupted by the CSOT cancellation token. The sync and async paths must stay separate here — do **not** collapse them by calling `Task.Delay(...).GetAwaiter().GetResult()` (sync-over-async on a hot retry path).
+3. On retryable exception, deprioritize the failed server. For ordinary retryable errors there is no back-off — the loop retries immediately against a different server. For `SystemOverloadedError`-labelled exceptions an adaptive back-off (`Thread.Sleep` on the sync path, `Task.Delay` on the async path, computed by `RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random)`) is applied; **both** paths bail early if the back-off would exceed the remaining CSOT deadline (explicit `if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) throw originalException;` check before each sleep/delay in `RetryableReadOperationExecutor.cs` / `RetryableWriteOperationExecutor.cs`). The async path needs its own check because `OperationContext.CancellationToken` is the raw caller token and does **not** fire when the CSOT deadline elapses — `Task.Delay(backoff, operationContext.CancellationToken)` would otherwise sleep the full back-off (up to `MaxBackoff`, or a server-supplied `baseBackoffMS`) regardless of the remaining timeout. The sync and async paths must stay separate here — do **not** collapse them by calling `Task.Delay(...).GetAwaiter().GetResult()` (sync-over-async on a hot retry path).
 4. On non-retryable exception or attempt budget exhausted, throw.
 
 Retry classification lives in `RetryabilityHelper`: network errors, the `RetryableWriteError` error label, and overload errors are retryable; auth/validation errors are not. Backpressure / `SystemOverloadedError` retries follow the separate adaptive-back-off path described above and are additionally gated by the `enableOverloadRetargeting` feature flag in both executors (`RetryableReadOperationExecutor.cs` / `RetryableWriteOperationExecutor.cs`) — with the flag off, overload errors do not get the adaptive back-off and behave like ordinary retryable errors. Both executors also take a `maxAdaptiveRetries` parameter (threaded through to `RetryableReadContext` / `RetryableWriteContext`) that bounds the **adaptive-back-off retry count** — it caps how many times the loop will sleep on the back-off path after observing a `SystemOverloadedError`, not the lifetime retry count from attempt 1. Once that adaptive cap is reached, the loop stops retrying even if the failure would otherwise be retryable. (See the `ShouldRetry` helper, defined in **both** `RetryableReadOperationExecutor.cs` and `RetryableWriteOperationExecutor.cs`.)

src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs

@@ -123,26 +123,57 @@ public static int GetRetryDelayMs(IRandom random, int attempt, double backoffBas
             Ensure.IsGreaterThanZero(attempt, nameof(attempt));
             Ensure.IsGreaterThanZero(backoffBase, nameof(backoffBase));
             Ensure.IsGreaterThanZero(backoffInitial, nameof(backoffInitial));
-            Ensure.IsGreaterThan(backoffMax, backoffInitial, nameof(backoffMax));
+            Ensure.IsGreaterThanOrEqualTo(backoffMax, backoffInitial, nameof(backoffMax));
 
             var j = random.NextDouble();
-            return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1)));
+            return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt)));
         }
 
         /// <summary>
         /// Gets the operation retry backoff delay used for operation retries under client backpressure.
         /// </summary>
-        public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random)
+        /// <param name="attempt">The retry attempt number.</param>
+        /// <param name="random">The random number generator.</param>
+        /// <param name="baseBackoffMs">
+        /// A server-supplied backoff base (from the overload error's <c>baseBackoffMS</c> field) that overrides
+        /// the driver's default initial backoff, or <see langword="null"/> to use the default.
+        /// </param>
+        public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random, int? baseBackoffMs = null)
         {
+            // Limit a server-supplied override to MaxBackoff so the backoff still resolves to min(MaxBackoff, ...) rather than tripping GetRetryDelayMs's guard.
+            var initialBackoff = Math.Min(
+                baseBackoffMs ?? OperationRetryBackpressureConstants.InitialBackoff,
+                OperationRetryBackpressureConstants.MaxBackoff);
             return TimeSpan.FromMilliseconds(
                 GetRetryDelayMs(
                     random,
                     attempt,
                     OperationRetryBackpressureConstants.BasePowerBackoff,
-                    OperationRetryBackpressureConstants.InitialBackoff,
+                    initialBackoff,
                     OperationRetryBackpressureConstants.MaxBackoff));
         }
 
+        /// <summary>
+        /// Reads a server-supplied backoff override (the <c>baseBackoffMS</c> field) from a retryable overload error,
+        /// or <see langword="null"/> when absent. Only <see cref="MongoCommandException"/> results are inspected; a
+        /// missing, non-numeric, or non-positive value is treated as absent.
+        /// </summary>
+        public static int? GetBaseBackoffMs(Exception exception)
+        {
+            if (exception is MongoCommandException { Result: { } result } &&
+                result.TryGetValue("baseBackoffMS", out var value) &&
+                value.IsNumeric)
+            {
+                var ms = value.ToInt64();
+                if (ms > 0)
+                {
+                    return (int)Math.Min(ms, int.MaxValue);
+                }
+            }
+
+            return null;
+        }
+
         public static bool IsCommandRetryable(BsonDocument command)
         {
             return

src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs

@@ -114,6 +114,14 @@ public static async Task<TResult> ExecuteAsync<TResult>(OperationContext operati
                         throw originalException;
                     }
 
+                    // We bail early if the backoff would exceed the CSOT deadline. The cancellation token alone does not
+                    // fire when the deadline elapses, so without this check the delay would ignore the remaining timeout.
+                    var remaining = operationContext.RemainingTimeout;
+                    if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff)
+                    {
+                        throw originalException;
+                    }
+
                     await Task.Delay(backoff, operationContext.CancellationToken).ConfigureAwait(false);
                     deprioritizedServers = UpdateServerList(server, deprioritizedServers, ex, context.EnableOverloadRetargeting);
                 }
@@ -165,7 +173,8 @@ private static bool ShouldRetry(
                     currentTransaction.ResetState();
                 }
 
-                backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random);
+                var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);
+                backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, baseBackoffMs);
 
                 return attempt <= context.MaxAdaptiveRetries;
             }

src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs

@@ -159,6 +159,14 @@ public static async Task<TResult> ExecuteAsync<TResult>(OperationContext operati
                         endTransactionOperation.OnRetry(operationContext, context, ex);
                     }
 
+                    // We bail early if the backoff would exceed the CSOT deadline. The cancellation token alone does not
+                    // fire when the deadline elapses, so without this check the delay would ignore the remaining timeout.
+                    var remaining = operationContext.RemainingTimeout;
+                    if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff)
+                    {
+                        throw originalException;
+                    }
+
                     await Task.Delay(backoff, operationContext.CancellationToken).ConfigureAwait(false);
                     deprioritizedServers = UpdateServerList(server, deprioritizedServers, ex, context.EnableOverloadRetargeting);
                 }
@@ -232,7 +240,8 @@ private static bool ShouldRetry(
                     currentTransaction.ResetState();
                 }
 
-                backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random);
+                var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);
+                backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, baseBackoffMs);
 
                 return attempt <= context.MaxAdaptiveRetries;
             }

tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs

@@ -141,6 +141,26 @@ public static MongoCommandException CreateMongoCommandExceptionWithLabels(int co
             return commandException;
         }
 
+        public static MongoCommandException CreateMongoCommandExceptionWithLabels(BsonDocument result, params string[] labels)
+        {
+            var clusterId = new ClusterId(1);
+            var endPoint = new DnsEndPoint("localhost", 27017);
+            var serverId = new ServerId(clusterId, endPoint);
+            var connectionId = new ConnectionId(serverId);
+            var message = "Fake MongoCommandException";
+            var command = BsonDocument.Parse("{ command : 1 }");
+            var commandException = new MongoCommandException(connectionId, message, command, result);
+            foreach (var label in labels)
+            {
+                if (label != null)
+                {
+                    commandException.AddErrorLabel(label);
+                }
+            }
+
+            return commandException;
+        }
+
         public static MongoCommandException CreateMongoWriteConcernException(BsonDocument writeConcernResultDocument, string label = null)
         {
             var clusterId = new ClusterId(1);

tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs

@@ -653,7 +653,7 @@ public async Task WithTransaction_retry_backoff_is_enforced([Values(true, false)
             var noBackoffTimeMs = await ExecuteWithTransactionAsync(0);
             var backoffTimeMs = await ExecuteWithTransactionAsync(1);
 
-            backoffTimeMs.Should().BeApproximately(noBackoffTimeMs + 1800, 150);
+            backoffTimeMs.Should().BeApproximately(noBackoffTimeMs + 2280, 150);
 
             async Task<double> ExecuteWithTransactionAsync(double randomValue)
             {

tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs

@@ -29,11 +29,11 @@ namespace MongoDB.Driver.Core.Connections
     public class HelloHelperTests
     {
         [Theory]
-        [InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: true }")]
-        [InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: true }")]
-        [InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: true }")]
-        [InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: true }")]
-        [InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: true }")]
+        [InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: '2' }")]
+        [InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: '2' }")]
+        [InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")]
+        [InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")]
+        [InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: '2' }")]
         public void CreateCommand_should_return_correct_hello_command(bool useServerApiVersion, bool helloOk, bool loadBalanced, string expectedResult)
         {
             var serverApi = useServerApiVersion ? new ServerApi(ServerApiVersion.V1) : null;
@@ -51,7 +51,7 @@ public void AddClientDocumentToCommand_with_custom_document_should_return_expect
             var command = HelloHelper.CreateCommand(null);
             var result = HelloHelper.AddClientDocumentToCommand(command, clientDocument);
 
-            result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, client : {clientDocumentString} }}");
+            result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', client : {clientDocumentString} }}");
         }
 
         [Fact]
@@ -104,7 +104,7 @@ public void AddCompressorsToCommand_with_compressors_should_return_expected_resu
             var result = HelloHelper.AddCompressorsToCommand(command, compressors);
 
             var expectedCompressions = string.Join(",", compressorsParameters.Select(c => $"'{CompressorTypeMapper.ToServerName(c)}'"));
-            result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, compression: [{expectedCompressions}] }}"));
+            result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', compression: [{expectedCompressions}] }}"));
         }
     }
 }

tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs

@@ -20,6 +20,7 @@
 using MongoDB.TestHelpers.XunitExtensions;
 using MongoDB.Driver.Core.Misc;
 using MongoDB.Driver.Core.TestHelpers;
+using Moq;
 using Xunit;
 
 namespace MongoDB.Driver.Core.Operations
@@ -102,13 +103,13 @@ public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError
         }
 
         [Theory]
-        [InlineData(1, 2, 100, 10000, 0, 100)]
-        [InlineData(2, 2, 100, 10000, 0, 200)]
-        [InlineData(3, 2, 100, 10000, 0, 400)]
+        [InlineData(1, 2, 100, 10000, 0, 200)]
+        [InlineData(2, 2, 100, 10000, 0, 400)]
+        [InlineData(3, 2, 100, 10000, 0, 800)]
         [InlineData(9999, 2, 100, 10000, 0, 10000)]
-        [InlineData(1, 1.5, 100, 10000, 0, 100)]
-        [InlineData(2, 1.5, 100, 10000, 0, 150)]
-        [InlineData(3, 1.5, 100, 10000, 0, 225)]
+        [InlineData(1, 1.5, 100, 10000, 0, 150)]
+        [InlineData(2, 1.5, 100, 10000, 0, 225)]
+        [InlineData(3, 1.5, 100, 10000, 0, 337)]
         [InlineData(9999, 1.5, 100, 10000, 0, 10000)]
         public void GetRetryDelayMs_should_return_expected_result(int attempt, double backoffBase, int backoffInitial, int backoffMax, int expectedRangeMin, int expectedRangeMax)
         {
@@ -287,5 +288,59 @@ public void IsRetryableWriteException_should_return_expected_result([Values(fals
 
             result.Should().Be(hasRetryableWriteLabel);
         }
+
+        [Fact]
+        public void GetBaseBackoffMs_should_return_null_for_non_command_exception()
+        {
+            var exception = CoreExceptionHelper.CreateException(typeof(MongoConnectionException));
+
+            var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);
+
+            baseBackoffMs.Should().Be(null);
+        }
+
+        [Theory]
+        [InlineData("{ ok : 0, code : 2 }", null)]
+        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 0 }", null)]
+        [InlineData("{ ok : 0, code : 2, baseBackoffMS : -5 }", null)]
+        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 'not-a-number' }", null)]
+        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 50 }", 50)]
+        [InlineData("{ ok : 0, code : 2, baseBackoffMS : NumberLong(9999999999) }", int.MaxValue)]
+        public void GetBaseBackoffMs_should_return_expected_result(string resultJson, int? expectedValue)
+        {
+            var result = BsonDocument.Parse(resultJson);
+            var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result);
+
+            var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);
+
+            baseBackoffMs.Should().Be(expectedValue);
+        }
+
+        [Theory]
+        [InlineData(1, null, 0, 200)]
+        [InlineData(2, null, 0, 400)]
+        [InlineData(1, 50, 0, 100)]
+        [InlineData(2, 50, 0, 200)]
+        [InlineData(1, 10000, 0, 10000)]
+        [InlineData(2, 20000, 0, 10000)]
+        public void GetOperationRetryBackoffDelay_should_apply_baseBackoffMs_override(int attempt, int? baseBackoffMs, int expectedRangeMin, int expectedRangeMax)
+        {
+            var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, DefaultRandom.Instance, baseBackoffMs);
+
+            result.TotalMilliseconds.Should().BeInRange(expectedRangeMin, expectedRangeMax);
+        }
+
+        [Theory]
+        [InlineData(1, 10000, 10000)]
+        [InlineData(2, 20000, 10000)]
+        public void GetOperationRetryBackoffDelay_should_limit_override_to_MaxBackoff(int attempt, int baseBackoffMs, int expectedMs)
+        {
+            var randomMock = new Mock<IRandom>();
+            randomMock.Setup(r => r.NextDouble()).Returns(1.0);
+
+            var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, randomMock.Object, baseBackoffMs);
+
+            result.TotalMilliseconds.Should().Be(expectedMs);
+        }
     }
 }