diff --git a/src/MongoDB.Driver/Core/Connections/HelloHelper.cs b/src/MongoDB.Driver/Core/Connections/HelloHelper.cs index b025d2207c9..f73560f41c1 100644 --- a/src/MongoDB.Driver/Core/Connections/HelloHelper.cs +++ b/src/MongoDB.Driver/Core/Connections/HelloHelper.cs @@ -57,7 +57,7 @@ internal static BsonDocument CreateCommand(ServerApi serverApi, bool helloOk = f { "topologyVersion", () => topologyVersion.ToBsonDocument(), topologyVersion != null }, { "maxAwaitTimeMS", () => (long)maxAwaitTime.Value.TotalMilliseconds, maxAwaitTime.HasValue }, { "loadBalanced", true, loadBalanced }, - { "backpressure", true } + { "backpressure", "2" } }; } diff --git a/src/MongoDB.Driver/Core/Misc/Feature.cs b/src/MongoDB.Driver/Core/Misc/Feature.cs index 9e00e0b83ce..2d30f9f0939 100644 --- a/src/MongoDB.Driver/Core/Misc/Feature.cs +++ b/src/MongoDB.Driver/Core/Misc/Feature.cs @@ -112,6 +112,11 @@ public class Feature /// public static Feature ChangeStreamSplitEventStage { get; } = new("ChangeStreamSplitEventStage", WireVersion.Server70); + /// + /// Gets the client backpressure baseBackoffMS feature. + /// + public static Feature ClientBackpressureBaseBackoffMs { get; } = new("ClientBackpressureBaseBackoffMs", WireVersion.Server90); + /// /// Gets the client bulk write feature. /// diff --git a/src/MongoDB.Driver/Core/Operations/AGENTS.md b/src/MongoDB.Driver/Core/Operations/AGENTS.md index 25e6a8d036e..0e2ee5789bc 100644 --- a/src/MongoDB.Driver/Core/Operations/AGENTS.md +++ b/src/MongoDB.Driver/Core/Operations/AGENTS.md @@ -95,7 +95,7 @@ Pinned channel/server attach on the first command. Recovery token (returned by m 1. Call `operationContext.ThrowIfTimedOutOrCanceled()` at the top of each iteration. 2. Execute one attempt against the binding. -3. On retryable exception, deprioritize the failed server. For ordinary retryable errors there is no back-off — the loop retries immediately against a different server. For `SystemOverloadedError`-labelled exceptions an adaptive back-off (`Thread.Sleep` on the sync path, `Task.Delay` on the async path, computed by `RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random)`) is applied; on the sync path the loop bails early if the back-off would exceed the remaining CSOT deadline (explicit check in `RetryableReadOperationExecutor.cs` lines 66-68 / `RetryableWriteOperationExecutor.cs` lines 66-68: `if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) throw;`). The async path doesn't need an equivalent pre-check — `Task.Delay(backoff, operationContext.CancellationToken)` is interrupted by the CSOT cancellation token. The sync and async paths must stay separate here — do **not** collapse them by calling `Task.Delay(...).GetAwaiter().GetResult()` (sync-over-async on a hot retry path). +3. On retryable exception, deprioritize the failed server. For ordinary retryable errors there is no back-off — the loop retries immediately against a different server. For `SystemOverloadedError`-labelled exceptions an adaptive back-off (`Thread.Sleep` on the sync path, `Task.Delay` on the async path, computed by `RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random)`) is applied; **both** paths bail early if the back-off would exceed the remaining CSOT deadline (explicit `if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) throw originalException;` check before each sleep/delay in `RetryableReadOperationExecutor.cs` / `RetryableWriteOperationExecutor.cs`). The async path needs its own check because `OperationContext.CancellationToken` is the raw caller token and does **not** fire when the CSOT deadline elapses — `Task.Delay(backoff, operationContext.CancellationToken)` would otherwise sleep the full back-off (up to `MaxBackoff`, or a server-supplied `baseBackoffMS`) regardless of the remaining timeout. The sync and async paths must stay separate here — do **not** collapse them by calling `Task.Delay(...).GetAwaiter().GetResult()` (sync-over-async on a hot retry path). 4. On non-retryable exception or attempt budget exhausted, throw. Retry classification lives in `RetryabilityHelper`: network errors, the `RetryableWriteError` error label, and overload errors are retryable; auth/validation errors are not. Backpressure / `SystemOverloadedError` retries follow the separate adaptive-back-off path described above and are additionally gated by the `enableOverloadRetargeting` feature flag in both executors (`RetryableReadOperationExecutor.cs` / `RetryableWriteOperationExecutor.cs`) — with the flag off, overload errors do not get the adaptive back-off and behave like ordinary retryable errors. Both executors also take a `maxAdaptiveRetries` parameter (threaded through to `RetryableReadContext` / `RetryableWriteContext`) that bounds the **adaptive-back-off retry count** — it caps how many times the loop will sleep on the back-off path after observing a `SystemOverloadedError`, not the lifetime retry count from attempt 1. Once that adaptive cap is reached, the loop stops retrying even if the failure would otherwise be retryable. (See the `ShouldRetry` helper, defined in **both** `RetryableReadOperationExecutor.cs` and `RetryableWriteOperationExecutor.cs`.) diff --git a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs index b242e4c02f1..02025370a27 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs @@ -123,26 +123,57 @@ public static int GetRetryDelayMs(IRandom random, int attempt, double backoffBas Ensure.IsGreaterThanZero(attempt, nameof(attempt)); Ensure.IsGreaterThanZero(backoffBase, nameof(backoffBase)); Ensure.IsGreaterThanZero(backoffInitial, nameof(backoffInitial)); - Ensure.IsGreaterThan(backoffMax, backoffInitial, nameof(backoffMax)); + Ensure.IsGreaterThanOrEqualTo(backoffMax, backoffInitial, nameof(backoffMax)); var j = random.NextDouble(); - return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1))); + return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt))); } /// /// Gets the operation retry backoff delay used for operation retries under client backpressure. /// - public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random) + /// The retry attempt number. + /// The random number generator. + /// + /// A server-supplied backoff base (from the overload error's baseBackoffMS field) that overrides + /// the driver's default initial backoff, or to use the default. + /// + public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random, int? baseBackoffMs = null) { + // Limit a server-supplied override to MaxBackoff so the backoff still resolves to min(MaxBackoff, ...) rather than tripping GetRetryDelayMs's guard. + var initialBackoff = Math.Min( + baseBackoffMs ?? OperationRetryBackpressureConstants.InitialBackoff, + OperationRetryBackpressureConstants.MaxBackoff); return TimeSpan.FromMilliseconds( GetRetryDelayMs( random, attempt, OperationRetryBackpressureConstants.BasePowerBackoff, - OperationRetryBackpressureConstants.InitialBackoff, + initialBackoff, OperationRetryBackpressureConstants.MaxBackoff)); } + /// + /// Reads a server-supplied backoff override (the baseBackoffMS field) from a retryable overload error, + /// or when absent. Only results are inspected; a + /// missing, non-numeric, or non-positive value is treated as absent. + /// + public static int? GetBaseBackoffMs(Exception exception) + { + if (exception is MongoCommandException { Result: { } result } && + result.TryGetValue("baseBackoffMS", out var value) && + value.IsNumeric) + { + var ms = value.ToInt64(); + if (ms > 0) + { + return (int)Math.Min(ms, int.MaxValue); + } + } + + return null; + } + public static bool IsCommandRetryable(BsonDocument command) { return diff --git a/src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs b/src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs index 2f5557f4d41..a009d6fc823 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs @@ -114,6 +114,14 @@ public static async Task ExecuteAsync(OperationContext operati throw originalException; } + // We bail early if the backoff would exceed the CSOT deadline. The cancellation token alone does not + // fire when the deadline elapses, so without this check the delay would ignore the remaining timeout. + var remaining = operationContext.RemainingTimeout; + if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) + { + throw originalException; + } + await Task.Delay(backoff, operationContext.CancellationToken).ConfigureAwait(false); deprioritizedServers = UpdateServerList(server, deprioritizedServers, ex, context.EnableOverloadRetargeting); } @@ -165,7 +173,8 @@ private static bool ShouldRetry( currentTransaction.ResetState(); } - backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random); + var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception); + backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, baseBackoffMs); return attempt <= context.MaxAdaptiveRetries; } diff --git a/src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs b/src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs index 77734be69c2..dcdcb36077c 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs @@ -159,6 +159,14 @@ public static async Task ExecuteAsync(OperationContext operati endTransactionOperation.OnRetry(operationContext, context, ex); } + // We bail early if the backoff would exceed the CSOT deadline. The cancellation token alone does not + // fire when the deadline elapses, so without this check the delay would ignore the remaining timeout. + var remaining = operationContext.RemainingTimeout; + if (remaining != Timeout.InfiniteTimeSpan && remaining < backoff) + { + throw originalException; + } + await Task.Delay(backoff, operationContext.CancellationToken).ConfigureAwait(false); deprioritizedServers = UpdateServerList(server, deprioritizedServers, ex, context.EnableOverloadRetargeting); } @@ -232,7 +240,8 @@ private static bool ShouldRetry( currentTransaction.ResetState(); } - backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random); + var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception); + backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, baseBackoffMs); return attempt <= context.MaxAdaptiveRetries; } diff --git a/tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs b/tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs index 766c833712a..be639686d33 100644 --- a/tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs +++ b/tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs @@ -141,6 +141,26 @@ public static MongoCommandException CreateMongoCommandExceptionWithLabels(int co return commandException; } + public static MongoCommandException CreateMongoCommandExceptionWithLabels(BsonDocument result, params string[] labels) + { + var clusterId = new ClusterId(1); + var endPoint = new DnsEndPoint("localhost", 27017); + var serverId = new ServerId(clusterId, endPoint); + var connectionId = new ConnectionId(serverId); + var message = "Fake MongoCommandException"; + var command = BsonDocument.Parse("{ command : 1 }"); + var commandException = new MongoCommandException(connectionId, message, command, result); + foreach (var label in labels) + { + if (label != null) + { + commandException.AddErrorLabel(label); + } + } + + return commandException; + } + public static MongoCommandException CreateMongoWriteConcernException(BsonDocument writeConcernResultDocument, string label = null) { var clusterId = new ClusterId(1); diff --git a/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs b/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs index 2cff45f8bba..033373a8d43 100644 --- a/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs +++ b/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs @@ -653,7 +653,7 @@ public async Task WithTransaction_retry_backoff_is_enforced([Values(true, false) var noBackoffTimeMs = await ExecuteWithTransactionAsync(0); var backoffTimeMs = await ExecuteWithTransactionAsync(1); - backoffTimeMs.Should().BeApproximately(noBackoffTimeMs + 1800, 150); + backoffTimeMs.Should().BeApproximately(noBackoffTimeMs + 2280, 150); async Task ExecuteWithTransactionAsync(double randomValue) { diff --git a/tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs b/tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs index c70b3940347..a9ac6075ca3 100644 --- a/tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs @@ -29,11 +29,11 @@ namespace MongoDB.Driver.Core.Connections public class HelloHelperTests { [Theory] - [InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: true }")] - [InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: true }")] - [InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: true }")] - [InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: true }")] - [InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: true }")] + [InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: '2' }")] + [InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: '2' }")] + [InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")] + [InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")] + [InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: '2' }")] public void CreateCommand_should_return_correct_hello_command(bool useServerApiVersion, bool helloOk, bool loadBalanced, string expectedResult) { var serverApi = useServerApiVersion ? new ServerApi(ServerApiVersion.V1) : null; @@ -51,7 +51,7 @@ public void AddClientDocumentToCommand_with_custom_document_should_return_expect var command = HelloHelper.CreateCommand(null); var result = HelloHelper.AddClientDocumentToCommand(command, clientDocument); - result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, client : {clientDocumentString} }}"); + result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', client : {clientDocumentString} }}"); } [Fact] @@ -104,7 +104,7 @@ public void AddCompressorsToCommand_with_compressors_should_return_expected_resu var result = HelloHelper.AddCompressorsToCommand(command, compressors); var expectedCompressions = string.Join(",", compressorsParameters.Select(c => $"'{CompressorTypeMapper.ToServerName(c)}'")); - result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, compression: [{expectedCompressions}] }}")); + result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', compression: [{expectedCompressions}] }}")); } } } diff --git a/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs b/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs index 9673f2914e5..be2576df78e 100644 --- a/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs @@ -20,6 +20,7 @@ using MongoDB.TestHelpers.XunitExtensions; using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.TestHelpers; +using Moq; using Xunit; namespace MongoDB.Driver.Core.Operations @@ -102,13 +103,13 @@ public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError } [Theory] - [InlineData(1, 2, 100, 10000, 0, 100)] - [InlineData(2, 2, 100, 10000, 0, 200)] - [InlineData(3, 2, 100, 10000, 0, 400)] + [InlineData(1, 2, 100, 10000, 0, 200)] + [InlineData(2, 2, 100, 10000, 0, 400)] + [InlineData(3, 2, 100, 10000, 0, 800)] [InlineData(9999, 2, 100, 10000, 0, 10000)] - [InlineData(1, 1.5, 100, 10000, 0, 100)] - [InlineData(2, 1.5, 100, 10000, 0, 150)] - [InlineData(3, 1.5, 100, 10000, 0, 225)] + [InlineData(1, 1.5, 100, 10000, 0, 150)] + [InlineData(2, 1.5, 100, 10000, 0, 225)] + [InlineData(3, 1.5, 100, 10000, 0, 337)] [InlineData(9999, 1.5, 100, 10000, 0, 10000)] public void GetRetryDelayMs_should_return_expected_result(int attempt, double backoffBase, int backoffInitial, int backoffMax, int expectedRangeMin, int expectedRangeMax) { @@ -287,5 +288,59 @@ public void IsRetryableWriteException_should_return_expected_result([Values(fals result.Should().Be(hasRetryableWriteLabel); } + + [Fact] + public void GetBaseBackoffMs_should_return_null_for_non_command_exception() + { + var exception = CoreExceptionHelper.CreateException(typeof(MongoConnectionException)); + + var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception); + + baseBackoffMs.Should().Be(null); + } + + [Theory] + [InlineData("{ ok : 0, code : 2 }", null)] + [InlineData("{ ok : 0, code : 2, baseBackoffMS : 0 }", null)] + [InlineData("{ ok : 0, code : 2, baseBackoffMS : -5 }", null)] + [InlineData("{ ok : 0, code : 2, baseBackoffMS : 'not-a-number' }", null)] + [InlineData("{ ok : 0, code : 2, baseBackoffMS : 50 }", 50)] + [InlineData("{ ok : 0, code : 2, baseBackoffMS : NumberLong(9999999999) }", int.MaxValue)] + public void GetBaseBackoffMs_should_return_expected_result(string resultJson, int? expectedValue) + { + var result = BsonDocument.Parse(resultJson); + var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result); + + var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception); + + baseBackoffMs.Should().Be(expectedValue); + } + + [Theory] + [InlineData(1, null, 0, 200)] + [InlineData(2, null, 0, 400)] + [InlineData(1, 50, 0, 100)] + [InlineData(2, 50, 0, 200)] + [InlineData(1, 10000, 0, 10000)] + [InlineData(2, 20000, 0, 10000)] + public void GetOperationRetryBackoffDelay_should_apply_baseBackoffMs_override(int attempt, int? baseBackoffMs, int expectedRangeMin, int expectedRangeMax) + { + var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, DefaultRandom.Instance, baseBackoffMs); + + result.TotalMilliseconds.Should().BeInRange(expectedRangeMin, expectedRangeMax); + } + + [Theory] + [InlineData(1, 10000, 10000)] + [InlineData(2, 20000, 10000)] + public void GetOperationRetryBackoffDelay_should_limit_override_to_MaxBackoff(int attempt, int baseBackoffMs, int expectedMs) + { + var randomMock = new Mock(); + randomMock.Setup(r => r.NextDouble()).Returns(1.0); + + var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, randomMock.Object, baseBackoffMs); + + result.TotalMilliseconds.Should().Be(expectedMs); + } } } diff --git a/tests/MongoDB.Driver.Tests/Core/Operations/RetryableReadOperationExecutorTests.cs b/tests/MongoDB.Driver.Tests/Core/Operations/RetryableReadOperationExecutorTests.cs index 4a2fe3032e4..293684ad546 100644 --- a/tests/MongoDB.Driver.Tests/Core/Operations/RetryableReadOperationExecutorTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Operations/RetryableReadOperationExecutorTests.cs @@ -14,12 +14,19 @@ */ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Net; using System.Reflection; +using System.Threading.Tasks; using FluentAssertions; +using MongoDB.Bson; using MongoDB.Driver.Core.Bindings; +using MongoDB.Driver.Core.Clusters; using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.Operations; +using MongoDB.Driver.Core.Servers; using MongoDB.Driver.Core.TestHelpers; using Moq; using Xunit; @@ -135,7 +142,80 @@ public void ShouldRetry_with_system_overloaded_exception_should_not_retry_when_r result.Should().BeFalse(); } + [Theory] + [InlineData(1, 50, 0, 100)] + [InlineData(2, 50, 0, 200)] + [InlineData(1, 10000, 0, 10000)] + [InlineData(2, 20000, 0, 10000)] + public void ShouldRetry_with_baseBackoffMs_should_use_it_as_backoff_base( + int attempt, + int baseBackoffMs, + int expectedRangeMinMs, + int expectedRangeMaxMs) + { + var context = CreateContext(retryRequested: true); + var result = BsonDocument.Parse($"{{ ok : 0, code : 2, baseBackoffMS : {baseBackoffMs} }}"); + var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError"); + using var operationContext = new OperationContext(NoCoreSession.NewHandle()); + var randomMock = new Mock(); + randomMock.Setup(r => r.NextDouble()).Returns(1.0); + + var didRetry = RetryableReadOperationExecutorReflector.ShouldRetry( + operationContext, context, isOperationRetryable: true, exception, attempt, randomMock.Object, overloadErrorSeen: false, out var backoff); + + didRetry.Should().BeTrue(); + backoff.TotalMilliseconds.Should().BeInRange(expectedRangeMinMs, expectedRangeMaxMs); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Backoff_should_honor_CSOT_deadline(bool async) + { + using var session = NoCoreSession.NewHandle(); + using var operationContext = new OperationContext(session, timeout: TimeSpan.FromMilliseconds(200)); + var randomMock = new Mock(); + randomMock.Setup(r => r.NextDouble()).Returns(1.0); + + // baseBackoffMS = 5000 yields a first-retry backoff clamped to MaxBackoff (~10s), far larger than the 200ms deadline. + var result = BsonDocument.Parse("{ ok : 0, code : 2, baseBackoffMS : 5000 }"); + var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError"); + var operationMock = new Mock>(); + operationMock.Setup(o => o.ExecuteAttempt(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Throws(exception); + operationMock.Setup(o => o.ExecuteAttemptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ThrowsAsync(exception); + var context = CreateExecutableContext(randomMock.Object); + + var stopwatch = Stopwatch.StartNew(); + var thrown = async + ? await Record.ExceptionAsync(() => RetryableReadOperationExecutor.ExecuteAsync(operationContext, operationMock.Object, context)) + : Record.Exception(() => RetryableReadOperationExecutor.Execute(operationContext, operationMock.Object, context)); + stopwatch.Stop(); + + // The retry loop must bail with the overload error once the backoff would exceed the remaining CSOT deadline, + // rather than sleeping the full backoff (regression test for the async path, which lacked this guard). + thrown.Should().BeOfType(); + stopwatch.ElapsedMilliseconds.Should().BeLessThan(1000); + } + // private methods + private static RetryableReadContext CreateExecutableContext(IRandom random) + { + var serverId = new ServerId(new ClusterId(), new DnsEndPoint("localhost", 27017)); + var serverDescription = new ServerDescription(serverId, serverId.EndPoint); + var channel = new Mock().Object; + + var channelSource = new Mock(); + channelSource.SetupGet(cs => cs.ServerDescription).Returns(serverDescription); + channelSource.Setup(cs => cs.GetChannel(It.IsAny())).Returns(channel); + channelSource.Setup(cs => cs.GetChannelAsync(It.IsAny())).ReturnsAsync(channel); + + var binding = new Mock(); + binding.Setup(b => b.GetReadChannelSource(It.IsAny(), It.IsAny>())).Returns(channelSource.Object); + binding.Setup(b => b.GetReadChannelSourceAsync(It.IsAny(), It.IsAny>())).ReturnsAsync(channelSource.Object); + + return new RetryableReadContext(binding.Object, retryRequested: true, RetryabilityHelper.OperationRetryBackpressureConstants.DefaultMaxRetries, enableOverloadRetargeting: false, random); + } + private static RetryableReadContext CreateContext(bool retryRequested) { var bindingMock = new Mock(); diff --git a/tests/MongoDB.Driver.Tests/Core/Operations/RetryableWriteOperationExecutorTests.cs b/tests/MongoDB.Driver.Tests/Core/Operations/RetryableWriteOperationExecutorTests.cs index fe88768a9c1..c741c0fdcf1 100644 --- a/tests/MongoDB.Driver.Tests/Core/Operations/RetryableWriteOperationExecutorTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Operations/RetryableWriteOperationExecutorTests.cs @@ -15,14 +15,21 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Net; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; using FluentAssertions; using MongoDB.Bson; using MongoDB.Bson.TestHelpers; using MongoDB.Driver.Core.Bindings; using MongoDB.Driver.Core.Clusters; +using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.Operations; using MongoDB.Driver.Core.Servers; +using MongoDB.Driver.Core.TestHelpers; using Moq; using Xunit; @@ -85,7 +92,92 @@ public void IsOperationAcknowledged_should_return_expected_result(bool? isAcknow result.Should().Be(expectedResult); } + [Theory] + [InlineData(1, 50, 0, 100)] + [InlineData(2, 50, 0, 200)] + [InlineData(1, 10000, 0, 10000)] + [InlineData(2, 20000, 0, 10000)] + public void ShouldRetry_with_baseBackoffMs_should_use_it_as_backoff_base( + int attempt, + int baseBackoffMs, + int expectedRangeMinMs, + int expectedRangeMaxMs) + { + var context = CreateContext(retryRequested: true, areRetryableWritesSupported: true, hasSessionId: true, isInTransaction: false); + var result = BsonDocument.Parse($"{{ ok : 0, code : 2, baseBackoffMS : {baseBackoffMs} }}"); + var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError"); + using var operationContext = new OperationContext(NoCoreSession.NewHandle()); + var randomMock = new Mock(); + randomMock.Setup(r => r.NextDouble()).Returns(1.0); + + var didRetry = RetryableWriteOperationExecutorReflector.ShouldRetry( + operationContext, + errorDuringChannelAcquisition: false, + context.ChannelSource.ServerDescription, + WriteConcern.Acknowledged, + context, + exception, + attempt, + randomMock.Object, + isEndTransactionOperation: false, + isOperationRetryable: true, + overloadErrorSeen: false, + out var backoff); + + didRetry.Should().BeTrue(); + backoff.TotalMilliseconds.Should().BeInRange(expectedRangeMinMs, expectedRangeMaxMs); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Backoff_should_honor_CSOT_deadline(bool async) + { + using var session = NoCoreSession.NewHandle(); + using var operationContext = new OperationContext(session, timeout: TimeSpan.FromMilliseconds(200)); + var randomMock = new Mock(); + randomMock.Setup(r => r.NextDouble()).Returns(1.0); + + // baseBackoffMS = 5000 yields a first-retry backoff clamped to MaxBackoff (~10s), far larger than the 200ms deadline. + var result = BsonDocument.Parse("{ ok : 0, code : 2, baseBackoffMS : 5000 }"); + var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError"); + var operationMock = new Mock>(); + operationMock.SetupGet(o => o.WriteConcern).Returns(WriteConcern.Acknowledged); + operationMock.Setup(o => o.ExecuteAttempt(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Throws(exception); + operationMock.Setup(o => o.ExecuteAttemptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ThrowsAsync(exception); + var context = CreateExecutableContext(randomMock.Object); + + var stopwatch = Stopwatch.StartNew(); + var thrown = async + ? await Record.ExceptionAsync(() => RetryableWriteOperationExecutor.ExecuteAsync(operationContext, operationMock.Object, context)) + : Record.Exception(() => RetryableWriteOperationExecutor.Execute(operationContext, operationMock.Object, context)); + stopwatch.Stop(); + + // The retry loop must bail with the overload error once the backoff would exceed the remaining CSOT deadline, + // rather than sleeping the full backoff (regression test for the async path, which lacked this guard). + thrown.Should().BeOfType(); + stopwatch.ElapsedMilliseconds.Should().BeLessThan(1000); + } + // private methods + private static RetryableWriteContext CreateExecutableContext(IRandom random) + { + var serverId = new ServerId(new ClusterId(), new DnsEndPoint("localhost", 27017)); + var serverDescription = new ServerDescription(serverId, serverId.EndPoint); + var channel = new Mock().Object; + + var channelSource = new Mock(); + channelSource.SetupGet(cs => cs.ServerDescription).Returns(serverDescription); + channelSource.Setup(cs => cs.GetChannel(It.IsAny())).Returns(channel); + channelSource.Setup(cs => cs.GetChannelAsync(It.IsAny())).ReturnsAsync(channel); + + var binding = new Mock(); + binding.Setup(b => b.GetWriteChannelSource(It.IsAny(), It.IsAny>())).Returns(channelSource.Object); + binding.Setup(b => b.GetWriteChannelSourceAsync(It.IsAny(), It.IsAny>())).ReturnsAsync(channelSource.Object); + + return new RetryableWriteContext(binding.Object, retryRequested: true, RetryabilityHelper.OperationRetryBackpressureConstants.DefaultMaxRetries, false, random); + } + private IWriteBinding CreateBinding(bool areRetryableWritesSupported, bool hasSessionId, bool isInTransaction) { var mockBinding = new Mock(); @@ -144,5 +236,37 @@ public static bool DoesContextAllowRetries(OperationContext operationContext, Re public static bool IsOperationAcknowledged(WriteConcern writeConcern) => (bool)Reflector.InvokeStatic(typeof(RetryableWriteOperationExecutor), nameof(IsOperationAcknowledged), writeConcern); + + public static bool ShouldRetry( + OperationContext operationContext, + bool errorDuringChannelAcquisition, + ServerDescription server, + WriteConcern writeConcern, + RetryableWriteContext context, + Exception exception, + int attempt, + IRandom random, + bool isEndTransactionOperation, + bool isOperationRetryable, + bool overloadErrorSeen, + out TimeSpan backoff) + { + var methodInfo = typeof(RetryableWriteOperationExecutor) + .GetMethods(BindingFlags.NonPublic | BindingFlags.Static) + .Single(m => m.Name == nameof(ShouldRetry) && m.GetParameters().Length == 12); + + var args = new object[] { operationContext, errorDuringChannelAcquisition, server, writeConcern, context, exception, attempt, random, isEndTransactionOperation, isOperationRetryable, overloadErrorSeen, default(TimeSpan) }; + + try + { + var result = (bool)methodInfo.Invoke(null, args); + backoff = (TimeSpan)args[11]; + return result; + } + catch (TargetInvocationException ex) + { + throw ex.InnerException; + } + } } } diff --git a/tests/MongoDB.Driver.Tests/Core/Servers/RoundTripTimeMonitorTests.cs b/tests/MongoDB.Driver.Tests/Core/Servers/RoundTripTimeMonitorTests.cs index de424f0784a..efa7edbc4dc 100644 --- a/tests/MongoDB.Driver.Tests/Core/Servers/RoundTripTimeMonitorTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Servers/RoundTripTimeMonitorTests.cs @@ -192,10 +192,10 @@ public void Start_should_use_serverApi() var sentMessages = connection.GetSentMessages(); sentMessages.Count.Should().BeInRange(1, 2); - MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }"); + MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }"); if (sentMessages.Count > 1) { - MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }"); + MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }"); } } @@ -232,10 +232,10 @@ public void RoundTripTimeMonitor_without_serverApi_but_with_loadBalancedConnecti var sentMessages = connection.GetSentMessages(); sentMessages.Count.Should().BeInRange(1, 2); - MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }"); + MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }"); if (sentMessages.Count > 1) { - MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }"); + MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }"); } } diff --git a/tests/MongoDB.Driver.Tests/Core/Servers/ServerMonitorTests.cs b/tests/MongoDB.Driver.Tests/Core/Servers/ServerMonitorTests.cs index 54e5d460959..b557268c2e5 100644 --- a/tests/MongoDB.Driver.Tests/Core/Servers/ServerMonitorTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Servers/ServerMonitorTests.cs @@ -451,7 +451,7 @@ public void ServerMonitor_should_use_serverApi() } var sentMessages = connection.GetSentMessages(); - MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk: true, topologyVersion : { processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }, maxAwaitTimeMS : NumberLong(86400000), backpressure : true, $db : 'admin', apiVersion : '1' }"); + MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk: true, topologyVersion : { processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }, maxAwaitTimeMS : NumberLong(86400000), backpressure : '2', $db : 'admin', apiVersion : '1' }"); } [Fact] @@ -470,7 +470,7 @@ public void ServerMonitor_without_serverApi_should_use_legacy_hello_to_set_up_st } var sentMessages = connection.GetSentMessages(); - MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, topologyVersion : {{ processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }}, maxAwaitTimeMS : NumberLong(86400000), backpressure : true, $db : 'admin' }}"); + MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, topologyVersion : {{ processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }}, maxAwaitTimeMS : NumberLong(86400000), backpressure : '2', $db : 'admin' }}"); } [Fact] @@ -489,7 +489,7 @@ public void ServerMonitor_without_serverApi_but_with_loadBalancedConnection_shou } var sentMessages = connection.GetSentMessages(); - MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk: true, topologyVersion : { processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }, maxAwaitTimeMS : NumberLong(86400000), loadBalanced: true, backpressure : true, $db : 'admin' }"); + MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk: true, topologyVersion : { processId : ObjectId('000000000000000000000000'), counter : NumberLong(0) }, maxAwaitTimeMS : NumberLong(86400000), loadBalanced: true, backpressure : '2', $db : 'admin' }"); } [Theory] diff --git a/tests/MongoDB.Driver.Tests/Core/WireProtocol/CommandWriteProtocolTests.cs b/tests/MongoDB.Driver.Tests/Core/WireProtocol/CommandWriteProtocolTests.cs index 136745f383f..84494a1ddb6 100644 --- a/tests/MongoDB.Driver.Tests/Core/WireProtocol/CommandWriteProtocolTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/WireProtocol/CommandWriteProtocolTests.cs @@ -32,6 +32,7 @@ using MongoDB.Driver.Core.Connections; using MongoDB.Driver.Core.Helpers; using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.Operations; using MongoDB.Driver.Core.Servers; using MongoDB.Driver.Core.TestHelpers; using MongoDB.Driver.Core.WireProtocol.Messages; @@ -484,6 +485,47 @@ ICoreSession CreateTransactionSession() } } + [Theory] + [ParameterAttributeData] + public async Task Execute_should_preserve_server_supplied_baseBackoffMS_in_the_command_exception( + [Values(false, true)] bool async) + { + var connection = new MockConnection(); + connection.Description = __connectionDescription; + var errorResponse = new BsonDocument + { + { "ok", 0 }, + { "code", 462 }, + { "errmsg", "overloaded" }, + { "errorLabels", new BsonArray { "SystemOverloadedError", "RetryableError" } }, + { "baseBackoffMS", 50 } + }; + connection.EnqueueCommandResponseMessage( + MessageHelper.BuildCommandResponse(CreateRawBsonDocument(errorResponse))); + + var subject = new CommandWireProtocol( + NoCoreSession.Instance, + ReadPreference.Primary, + new DatabaseNamespace("test"), + new BsonDocument("insert", "testCollection"), + commandPayloads: null, + postWriteAction: null, + CommandResponseHandling.Return, + BsonDocumentSerializer.Instance, + new MessageEncoderSettings(), + null, // serverApi + TimeSpan.FromMilliseconds(42)); + + using var operationContext = new OperationContext(NoCoreSession.NewHandle()); + var exception = async + ? await Record.ExceptionAsync(() => subject.ExecuteAsync(operationContext, connection)) + : Record.Exception(() => subject.Execute(operationContext, connection)); + + var commandException = exception.Should().BeOfType().Subject; + commandException.Result["baseBackoffMS"].ToInt32().Should().Be(50); + RetryabilityHelper.GetBaseBackoffMs(commandException).Should().Be(50); + } + // private methods private RawBsonDocument CreateRawBsonDocument(BsonDocument doc) { diff --git a/tests/MongoDB.Driver.Tests/Specifications/client-backpressure/prose-tests/ClientBackpressureProseTests.cs b/tests/MongoDB.Driver.Tests/Specifications/client-backpressure/prose-tests/ClientBackpressureProseTests.cs index c04362f8135..ea170ee78d1 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/client-backpressure/prose-tests/ClientBackpressureProseTests.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/client-backpressure/prose-tests/ClientBackpressureProseTests.cs @@ -83,6 +83,55 @@ await AssertBackoffBehavior( async (operationContext, context) => await RetryableWriteOperationExecutor.ExecuteAsync(operationContext, operationMock.Object, context)); } + [Theory] + [InlineData(false)] + [InlineData(true)] + // https://github.com/mongodb/specifications/pull/1953 (Test 5: overload errors with baseBackoffMS override base backoff) + public async Task ReadExecute_should_use_baseBackoffMs_as_backoff_base(bool async) + { + await AssertBaseBackoffMsOverridesBackoff( + async, + random => CreateRetryableReadContext(random), + exception => + { + var operationMock = new Mock>(); + operationMock + .Setup(o => o.ExecuteAttempt(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(exception); + operationMock + .Setup(o => o.ExecuteAttemptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(exception); + return operationMock.Object; + }, + (operationContext, operation, context) => RetryableReadOperationExecutor.Execute(operationContext, operation, context), + (operationContext, operation, context) => RetryableReadOperationExecutor.ExecuteAsync(operationContext, operation, context)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + // https://github.com/mongodb/specifications/pull/1953 (Test 5: overload errors with baseBackoffMS override base backoff) + public async Task WriteExecute_should_use_baseBackoffMs_as_backoff_base(bool async) + { + await AssertBaseBackoffMsOverridesBackoff( + async, + random => CreateRetryableWriteContext(random), + exception => + { + var operationMock = new Mock>(); + operationMock.SetupGet(o => o.WriteConcern).Returns(WriteConcern.Acknowledged); + operationMock + .Setup(o => o.ExecuteAttempt(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(exception); + operationMock + .Setup(o => o.ExecuteAttemptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(exception); + return operationMock.Object; + }, + (operationContext, operation, context) => RetryableWriteOperationExecutor.Execute(operationContext, operation, context), + (operationContext, operation, context) => RetryableWriteOperationExecutor.ExecuteAsync(operationContext, operation, context)); + } + private async Task AssertBackoffBehavior( bool async, Func createContext, @@ -120,11 +169,67 @@ private async Task AssertBackoffBehavior( withBackoffException.Should().NotBeNull().And.BeOfType(); - // With MAX_RETRIES=2 and jitter=1, total backoff = 100ms + 200ms = 300ms. - // Per the spec: assertTrue(abs(with_backoff_time - (no_backoff_time + 0.3s)) < 0.3s) + // With MAX_RETRIES=2 and jitter=1, total backoff = 200ms + 400ms = 600ms. + // Per the spec: assertTrue(abs(with_backoff_time - (no_backoff_time + 0.6s)) < 0.6s) var difference = withBackoffTime - noBackoffTime; - Math.Abs(difference - 300).Should().BeLessThan(300, - $"backoff difference should be approximately 300ms, got {difference}ms (noBackoff: {noBackoffTime}ms, withBackoff: {withBackoffTime}ms)"); + Math.Abs(difference - 600).Should().BeLessThan(600, + $"backoff difference should be approximately 600ms, got {difference}ms (noBackoff: {noBackoffTime}ms, withBackoff: {withBackoffTime}ms)"); + } + + private async Task AssertBaseBackoffMsOverridesBackoff( + bool async, + Func createContext, + Func createOperation, + Action executeSync, + Func executeAsync) + { + using var session = NoCoreSession.NewHandle(); + using var operationContext = new OperationContext(session, timeout: TimeSpan.FromSeconds(30)); + + var fullJitterRandom = new Mock(); + fullJitterRandom.Setup(r => r.NextDouble()).Returns(1.0); + + // Exponential backoff run: overload error without baseBackoffMS -> 200ms + 400ms = 600ms with jitter = 1. + var exponentialException = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(462, "SystemOverloadedError", "RetryableError"); + var exponentialMs = await MeasureBackoff(async, executeSync, executeAsync, operationContext, createOperation(exponentialException), createContext(fullJitterRandom.Object)); + + // baseBackoffMS override run: overload error with baseBackoffMS = 50 -> 100ms + 200ms = 300ms with jitter = 1. + var overrideResult = BsonDocument.Parse("{ ok : 0, code : 462, baseBackoffMS : 50 }"); + var overrideException = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(overrideResult, "SystemOverloadedError", "RetryableError"); + var withBaseBackoffMs = await MeasureBackoff(async, executeSync, executeAsync, operationContext, createOperation(overrideException), createContext(fullJitterRandom.Object)); + + // The driver must have parsed the server-supplied baseBackoffMS off the error. + RetryabilityHelper.GetBaseBackoffMs(overrideException).Should().Be(50); + + // Per the spec (jitter pinned to 1): the default backoffs sum to 0.2 + 0.4 = 0.6s and the + // baseBackoffMS = 50 backoffs sum to 0.1 + 0.2 = 0.3s. A run can never be faster than the sum of its backoffs. + // The operations here are mocked, so there is no server-round-trip cushion above the raw sleeps; allow a small + // tolerance on the lower bounds for timer granularity while still asserting baseBackoffMS shortens the backoff. + const long timerToleranceMs = 50; + exponentialMs.Should().BeGreaterOrEqualTo(600 - timerToleranceMs, + $"the default exponential backoff should sum to ~600ms (got {exponentialMs}ms)"); + withBaseBackoffMs.Should().BeGreaterOrEqualTo(300 - timerToleranceMs, + $"the baseBackoffMS=50 backoff should sum to ~300ms (got {withBaseBackoffMs}ms)"); + withBaseBackoffMs.Should().BeLessThan(600, + $"baseBackoffMS should shorten the backoff base (withBaseBackoffMS: {withBaseBackoffMs}ms, exponential: {exponentialMs}ms)"); + } + + private static async Task MeasureBackoff( + bool async, + Action executeSync, + Func executeAsync, + OperationContext operationContext, + TOperation operation, + TContext context) + { + var stopwatch = Stopwatch.StartNew(); + var exception = async + ? await Record.ExceptionAsync(() => executeAsync(operationContext, operation, context)) + : Record.Exception(() => executeSync(operationContext, operation, context)); + stopwatch.Stop(); + + exception.Should().NotBeNull().And.BeOfType(); + return stopwatch.ElapsedMilliseconds; } private static RetryableReadContext CreateRetryableReadContext(IRandom random, int maxAdaptiveRetries = 2) @@ -274,4 +379,69 @@ public void Overload_errors_are_retried_a_maximum_of_maxAdaptiveRetries_times_wh // maxAdaptiveRetries = 1, so total started commands = maxAdaptiveRetries + 1 = 2 eventCapturer.Events.OfType().Count().Should().Be(2); } + + [Fact] + // https://github.com/mongodb/specifications/pull/1953 (Test 5: overload errors with baseBackoffMS override base backoff) + public void Overload_errors_with_baseBackoffMS_shorten_the_backoff_base() + { + RequireServer.Check() + .ClusterTypes(ClusterType.ReplicaSet) + .Supports(Feature.ClientBackpressureBaseBackoffMs); + + var failPointCommand = BsonDocument.Parse( + @"{ + configureFailPoint: ""failCommand"", + mode: ""alwaysOn"", + data: + { + failCommands: [""insert""], + errorCode: 462, + errorLabels: [""SystemOverloadedError"", ""RetryableError""] + } + }"); + + using var client = DriverTestConfiguration.CreateMongoClient(s => s.RetryWrites = true); + var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName); + var collection = database.GetCollection(DriverTestConfiguration.CollectionNamespace.CollectionName); + var adminDatabase = client.GetDatabase(DatabaseNamespace.Admin.DatabaseName); + + long MeasureFailedInsertMs() + { + using var failPoint = FailPoint.Configure(failPointCommand); + var stopwatch = Stopwatch.StartNew(); + var exception = Record.Exception(() => collection.InsertOne(new BsonDocument("a", 1))); + stopwatch.Stop(); + + var mongoException = exception.Should().BeAssignableTo().Subject; + mongoException.HasErrorLabel("SystemOverloadedError").Should().BeTrue(); + return stopwatch.ElapsedMilliseconds; + } + + // Baseline: no override. With MaxAdaptiveRetries = 2, the worst case (jitter = 1 on both retries) + // is 200ms + 400ms = 600ms of backoff. + var baselineMs = MeasureFailedInsertMs(); + + // A value at/above MaxBackoff (10000ms) clamps to 10000ms on every attempt, so both retries draw + // their backoff from [0, 10000) instead of [0, 100)/[0, 200) -- an order of magnitude bigger + // regardless of jitter. + const int overrideBaseBackoffMs = 20000; + adminDatabase.RunCommand( + $@"{{ setParameter : 1, externalClientBaseBackoffMS : {overrideBaseBackoffMs} }}"); + try + { + var overrideMs = MeasureFailedInsertMs(); + + // Baseline maxes out at 600ms; the override's two draws from [0, 10000) average 10000ms + // combined. A 1s gap requires both override jitters to independently land below ~0.13, which + // happens well under 1% of the time -- comfortably low flake risk while still being a real, + // unmocked timing check. + (overrideMs - baselineMs).Should().BeGreaterThan(1000, + $"a large baseBackoffMS override should produce a much longer backoff (baseline: {baselineMs}ms, override: {overrideMs}ms)"); + } + finally + { + adminDatabase.RunCommand( + @"{ setParameter : 1, externalClientBaseBackoffMS : 0 }"); + } + } } diff --git a/tests/MongoDB.Driver.Tests/Specifications/mongodb-handshake/prose-tests/MongoDbHandshakeProseTests.cs b/tests/MongoDB.Driver.Tests/Specifications/mongodb-handshake/prose-tests/MongoDbHandshakeProseTests.cs index 163efa7b226..e5cf1ff1baf 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/mongodb-handshake/prose-tests/MongoDbHandshakeProseTests.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/mongodb-handshake/prose-tests/MongoDbHandshakeProseTests.cs @@ -108,8 +108,8 @@ public async Task DriverAcceptsArbitraryAuthMechanism([Values(false, true)] bool } [Fact] - // https://github.com/mongodb/specifications/blob/7039e69945d463a14b1b727d16db063e21f48f53/source/mongodb-handshake/tests/README.md#test-9-handshake-documents-include-backpressure-true - public async Task HandshakeDocumentsIncludeBackpressureTrue() + // https://github.com/mongodb/specifications/pull/1953 (spec PR updating Test 9 from `backpressure: true` to `backpressure: "2"`) + public async Task HandshakeDocumentsIncludeBackpressureVersion() { RequireServer.Check().Authentication(authentication: false); // speculative authentication makes events asserting hard @@ -126,7 +126,7 @@ public async Task HandshakeDocumentsIncludeBackpressureTrue() foreach (var doc in commandStartedEvents.Select(ev => ev.Command)) { doc.Contains("backpressure").Should().BeTrue(); - doc["backpressure"].AsBoolean.Should().BeTrue(); + doc["backpressure"].AsString.Should().Be("2"); } }