Skip to content

Commit f60f1ed

Browse files
papafepapafe
committed
CSHARP-6089: Client Backpressure with retryAfterMS
1 parent 082798b commit f60f1ed

14 files changed

Lines changed: 364 additions & 23 deletions

File tree

‎src/MongoDB.Driver/Core/Connections/HelloHelper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ internal static BsonDocument CreateCommand(ServerApi serverApi, bool helloOk = f
5757
{ "topologyVersion", () => topologyVersion.ToBsonDocument(), topologyVersion != null },
5858
{ "maxAwaitTimeMS", () => (long)maxAwaitTime.Value.TotalMilliseconds, maxAwaitTime.HasValue },
5959
{ "loadBalanced", true, loadBalanced },
60-
{ "backpressure", true }
60+
{ "backpressure", "2" }
6161
};
6262
}
6363

‎src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ public static int GetRetryDelayMs(IRandom random, int attempt, double backoffBas
123123
Ensure.IsGreaterThanZero(attempt, nameof(attempt));
124124
Ensure.IsGreaterThanZero(backoffBase, nameof(backoffBase));
125125
Ensure.IsGreaterThanZero(backoffInitial, nameof(backoffInitial));
126-
Ensure.IsGreaterThan(backoffMax, backoffInitial, nameof(backoffMax));
126+
Ensure.IsGreaterThanOrEqualTo(backoffMax, backoffInitial, nameof(backoffMax));
127127

128128
var j = random.NextDouble();
129129
return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1)));
@@ -132,17 +132,48 @@ public static int GetRetryDelayMs(IRandom random, int attempt, double backoffBas
132132
/// <summary>
133133
/// Gets the operation retry backoff delay used for operation retries under client backpressure.
134134
/// </summary>
135-
public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random)
135+
/// <param name="attempt">The retry attempt number.</param>
136+
/// <param name="random">The random number generator.</param>
137+
/// <param name="retryAfterMs">
138+
/// A server-supplied backoff base (from the overload error's <c>retryAfterMS</c> field) that overrides
139+
/// the driver's default initial backoff, or <see langword="null"/> to use the default.
140+
/// </param>
141+
public static TimeSpan GetOperationRetryBackoffDelay(int attempt, IRandom random, int? retryAfterMs = null)
136142
{
143+
// Limit a server-supplied override to MaxBackoff so the backoff still resolves to min(MaxBackoff, ...) rather than tripping GetRetryDelayMs's guard.
144+
var initialBackoff = Math.Min(
145+
retryAfterMs ?? OperationRetryBackpressureConstants.InitialBackoff,
146+
OperationRetryBackpressureConstants.MaxBackoff);
137147
return TimeSpan.FromMilliseconds(
138148
GetRetryDelayMs(
139149
random,
140150
attempt,
141151
OperationRetryBackpressureConstants.BasePowerBackoff,
142-
OperationRetryBackpressureConstants.InitialBackoff,
152+
initialBackoff,
143153
OperationRetryBackpressureConstants.MaxBackoff));
144154
}
145155

156+
/// <summary>
157+
/// Reads a server-supplied backoff override (the <c>retryAfterMS</c> field) from a retryable overload error,
158+
/// or <see langword="null"/> when absent. Only <see cref="MongoCommandException"/> results are inspected; a
159+
/// missing, non-numeric, or non-positive value is treated as absent.
160+
/// </summary>
161+
public static int? GetRetryAfterMs(Exception exception)
162+
{
163+
if (exception is MongoCommandException { Result: { } result } &&
164+
result.TryGetValue("retryAfterMS", out var value) &&
165+
value.IsNumeric)
166+
{
167+
var ms = value.ToInt64();
168+
if (ms > 0)
169+
{
170+
return (int)Math.Min(ms, int.MaxValue);
171+
}
172+
}
173+
174+
return null;
175+
}
176+
146177
public static bool IsCommandRetryable(BsonDocument command)
147178
{
148179
return

‎src/MongoDB.Driver/Core/Operations/RetryableReadOperationExecutor.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,8 @@ private static bool ShouldRetry(
165165
currentTransaction.ResetState();
166166
}
167167

168-
backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random);
168+
var retryAfterMs = RetryabilityHelper.GetRetryAfterMs(exception);
169+
backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, retryAfterMs);
169170

170171
return attempt <= context.MaxAdaptiveRetries;
171172
}

‎src/MongoDB.Driver/Core/Operations/RetryableWriteOperationExecutor.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,8 @@ private static bool ShouldRetry(
232232
currentTransaction.ResetState();
233233
}
234234

235-
backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random);
235+
var retryAfterMs = RetryabilityHelper.GetRetryAfterMs(exception);
236+
backoff = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, random, retryAfterMs);
236237

237238
return attempt <= context.MaxAdaptiveRetries;
238239
}

‎tests/MongoDB.Driver.TestHelpers/Core/CoreExceptionHelper.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,26 @@ public static MongoCommandException CreateMongoCommandExceptionWithLabels(int co
141141
return commandException;
142142
}
143143

144+
public static MongoCommandException CreateMongoCommandExceptionWithLabels(BsonDocument result, params string[] labels)
145+
{
146+
var clusterId = new ClusterId(1);
147+
var endPoint = new DnsEndPoint("localhost", 27017);
148+
var serverId = new ServerId(clusterId, endPoint);
149+
var connectionId = new ConnectionId(serverId);
150+
var message = "Fake MongoCommandException";
151+
var command = BsonDocument.Parse("{ command : 1 }");
152+
var commandException = new MongoCommandException(connectionId, message, command, result);
153+
foreach (var label in labels)
154+
{
155+
if (label != null)
156+
{
157+
commandException.AddErrorLabel(label);
158+
}
159+
}
160+
161+
return commandException;
162+
}
163+
144164
public static MongoCommandException CreateMongoWriteConcernException(BsonDocument writeConcernResultDocument, string label = null)
145165
{
146166
var clusterId = new ClusterId(1);

‎tests/MongoDB.Driver.Tests/Core/Connections/HelloHelperTests.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ namespace MongoDB.Driver.Core.Connections
2929
public class HelloHelperTests
3030
{
3131
[Theory]
32-
[InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: true }")]
33-
[InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: true }")]
34-
[InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: true }")]
35-
[InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: true }")]
36-
[InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: true }")]
32+
[InlineData(true, false, false, "{ hello : 1, helloOk : true, backpressure: '2' }")]
33+
[InlineData(false, false, false,"{ " + OppressiveLanguageConstants.LegacyHelloCommandName + " : 1, helloOk : true, backpressure: '2' }")]
34+
[InlineData(false, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")]
35+
[InlineData(true, true, false,"{ hello : 1, helloOk : true, backpressure: '2' }")]
36+
[InlineData(false, false, true,"{ hello : 1, helloOk : true, loadBalanced : true, backpressure: '2' }")]
3737
public void CreateCommand_should_return_correct_hello_command(bool useServerApiVersion, bool helloOk, bool loadBalanced, string expectedResult)
3838
{
3939
var serverApi = useServerApiVersion ? new ServerApi(ServerApiVersion.V1) : null;
@@ -51,7 +51,7 @@ public void AddClientDocumentToCommand_with_custom_document_should_return_expect
5151
var command = HelloHelper.CreateCommand(null);
5252
var result = HelloHelper.AddClientDocumentToCommand(command, clientDocument);
5353

54-
result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, client : {clientDocumentString} }}");
54+
result.Should().Be($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', client : {clientDocumentString} }}");
5555
}
5656

5757
[Fact]
@@ -104,7 +104,7 @@ public void AddCompressorsToCommand_with_compressors_should_return_expected_resu
104104
var result = HelloHelper.AddCompressorsToCommand(command, compressors);
105105

106106
var expectedCompressions = string.Join(",", compressorsParameters.Select(c => $"'{CompressorTypeMapper.ToServerName(c)}'"));
107-
result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: true, compression: [{expectedCompressions}] }}"));
107+
result.Should().Be(BsonDocument.Parse($"{{ {OppressiveLanguageConstants.LegacyHelloCommandName} : 1, helloOk : true, backpressure: '2', compression: [{expectedCompressions}] }}"));
108108
}
109109
}
110110
}

‎tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
using MongoDB.TestHelpers.XunitExtensions;
2121
using MongoDB.Driver.Core.Misc;
2222
using MongoDB.Driver.Core.TestHelpers;
23+
using Moq;
2324
using Xunit;
2425

2526
namespace MongoDB.Driver.Core.Operations
@@ -287,5 +288,59 @@ public void IsRetryableWriteException_should_return_expected_result([Values(fals
287288

288289
result.Should().Be(hasRetryableWriteLabel);
289290
}
291+
292+
[Fact]
293+
public void GetRetryAfterMs_should_return_null_for_non_command_exception()
294+
{
295+
var exception = CoreExceptionHelper.CreateException(typeof(MongoConnectionException));
296+
297+
var retryAfterMs = RetryabilityHelper.GetRetryAfterMs(exception);
298+
299+
retryAfterMs.Should().Be(null);
300+
}
301+
302+
[Theory]
303+
[InlineData("{ ok : 0, code : 2 }", null)]
304+
[InlineData("{ ok : 0, code : 2, retryAfterMS : 0 }", null)]
305+
[InlineData("{ ok : 0, code : 2, retryAfterMS : -5 }", null)]
306+
[InlineData("{ ok : 0, code : 2, retryAfterMS : 'not-a-number' }", null)]
307+
[InlineData("{ ok : 0, code : 2, retryAfterMS : 50 }", 50)]
308+
[InlineData("{ ok : 0, code : 2, retryAfterMS : NumberLong(9999999999) }", int.MaxValue)]
309+
public void GetRetryAfterMs_should_return_expected_result(string resultJson, int? expectedValue)
310+
{
311+
var result = BsonDocument.Parse(resultJson);
312+
var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result);
313+
314+
var retryAfterMs = RetryabilityHelper.GetRetryAfterMs(exception);
315+
316+
retryAfterMs.Should().Be(expectedValue);
317+
}
318+
319+
[Theory]
320+
[InlineData(1, null, 0, 100)]
321+
[InlineData(2, null, 0, 200)]
322+
[InlineData(1, 50, 0, 50)]
323+
[InlineData(2, 50, 0, 100)]
324+
[InlineData(1, 10000, 0, 10000)]
325+
[InlineData(2, 20000, 0, 10000)]
326+
public void GetOperationRetryBackoffDelay_should_apply_retryAfterMs_override(int attempt, int? retryAfterMs, int expectedRangeMin, int expectedRangeMax)
327+
{
328+
var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, DefaultRandom.Instance, retryAfterMs);
329+
330+
result.TotalMilliseconds.Should().BeInRange(expectedRangeMin, expectedRangeMax);
331+
}
332+
333+
[Theory]
334+
[InlineData(1, 10000, 10000)]
335+
[InlineData(2, 20000, 10000)]
336+
public void GetOperationRetryBackoffDelay_should_limit_override_to_MaxBackoff(int attempt, int retryAfterMs, int expectedMs)
337+
{
338+
var randomMock = new Mock<IRandom>();
339+
randomMock.Setup(r => r.NextDouble()).Returns(1.0);
340+
341+
var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, randomMock.Object, retryAfterMs);
342+
343+
result.TotalMilliseconds.Should().Be(expectedMs);
344+
}
290345
}
291346
}

‎tests/MongoDB.Driver.Tests/Core/Operations/RetryableReadOperationExecutorTests.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
using System.Linq;
1818
using System.Reflection;
1919
using FluentAssertions;
20+
using MongoDB.Bson;
2021
using MongoDB.Driver.Core.Bindings;
2122
using MongoDB.Driver.Core.Misc;
2223
using MongoDB.Driver.Core.Operations;
@@ -135,6 +136,31 @@ public void ShouldRetry_with_system_overloaded_exception_should_not_retry_when_r
135136
result.Should().BeFalse();
136137
}
137138

139+
[Theory]
140+
[InlineData(1, 50, 0, 50)]
141+
[InlineData(2, 50, 0, 100)]
142+
[InlineData(1, 10000, 0, 10000)]
143+
[InlineData(2, 20000, 0, 10000)]
144+
public void ShouldRetry_with_retryAfterMs_should_use_it_as_backoff_base(
145+
int attempt,
146+
int retryAfterMs,
147+
int expectedRangeMinMs,
148+
int expectedRangeMaxMs)
149+
{
150+
var context = CreateContext(retryRequested: true, isInTransaction: false);
151+
var result = BsonDocument.Parse($"{{ ok : 0, code : 2, retryAfterMS : {retryAfterMs} }}");
152+
var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError");
153+
var operationContext = new OperationContext(null, CancellationToken.None);
154+
var randomMock = new Mock<IRandom>();
155+
randomMock.Setup(r => r.NextDouble()).Returns(1.0);
156+
157+
var didRetry = RetryableReadOperationExecutorReflector.ShouldRetry(
158+
operationContext, context, isOperationRetryable: true, exception, attempt, randomMock.Object, overloadErrorSeen: false, out var backoff);
159+
160+
didRetry.Should().BeTrue();
161+
backoff.TotalMilliseconds.Should().BeInRange(expectedRangeMinMs, expectedRangeMaxMs);
162+
}
163+
138164
// private methods
139165
private static RetryableReadContext CreateContext(bool retryRequested)
140166
{

‎tests/MongoDB.Driver.Tests/Core/Operations/RetryableWriteOperationExecutorTests.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,19 @@
1515

1616
using System;
1717
using System.Collections.Generic;
18+
using System.Linq;
1819
using System.Net;
20+
using System.Reflection;
21+
using System.Threading;
1922
using FluentAssertions;
2023
using MongoDB.Bson;
2124
using MongoDB.Bson.TestHelpers;
2225
using MongoDB.Driver.Core.Bindings;
2326
using MongoDB.Driver.Core.Clusters;
27+
using MongoDB.Driver.Core.Misc;
2428
using MongoDB.Driver.Core.Operations;
2529
using MongoDB.Driver.Core.Servers;
30+
using MongoDB.Driver.Core.TestHelpers;
2631
using Moq;
2732
using Xunit;
2833

@@ -85,6 +90,42 @@ public void IsOperationAcknowledged_should_return_expected_result(bool? isAcknow
8590
result.Should().Be(expectedResult);
8691
}
8792

93+
[Theory]
94+
[InlineData(1, 50, 0, 50)]
95+
[InlineData(2, 50, 0, 100)]
96+
[InlineData(1, 10000, 0, 10000)]
97+
[InlineData(2, 20000, 0, 10000)]
98+
public void ShouldRetry_with_retryAfterMs_should_use_it_as_backoff_base(
99+
int attempt,
100+
int retryAfterMs,
101+
int expectedRangeMinMs,
102+
int expectedRangeMaxMs)
103+
{
104+
var context = CreateContext(retryRequested: true, areRetryableWritesSupported: true, hasSessionId: true, isInTransaction: false);
105+
var result = BsonDocument.Parse($"{{ ok : 0, code : 2, retryAfterMS : {retryAfterMs} }}");
106+
var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result, "SystemOverloadedError", "RetryableError");
107+
var operationContext = new OperationContext(null, CancellationToken.None);
108+
var randomMock = new Mock<IRandom>();
109+
randomMock.Setup(r => r.NextDouble()).Returns(1.0);
110+
111+
var didRetry = RetryableWriteOperationExecutorReflector.ShouldRetry(
112+
operationContext,
113+
errorDuringChannelAcquisition: false,
114+
context.ChannelSource.ServerDescription,
115+
WriteConcern.Acknowledged,
116+
context,
117+
exception,
118+
attempt,
119+
randomMock.Object,
120+
isEndTransactionOperation: false,
121+
isOperationRetryable: true,
122+
overloadErrorSeen: false,
123+
out var backoff);
124+
125+
didRetry.Should().BeTrue();
126+
backoff.TotalMilliseconds.Should().BeInRange(expectedRangeMinMs, expectedRangeMaxMs);
127+
}
128+
88129
// private methods
89130
private IWriteBinding CreateBinding(bool areRetryableWritesSupported, bool hasSessionId, bool isInTransaction)
90131
{
@@ -144,5 +185,37 @@ public static bool DoesContextAllowRetries(OperationContext operationContext, Re
144185

145186
public static bool IsOperationAcknowledged(WriteConcern writeConcern)
146187
=> (bool)Reflector.InvokeStatic(typeof(RetryableWriteOperationExecutor), nameof(IsOperationAcknowledged), writeConcern);
188+
189+
public static bool ShouldRetry(
190+
OperationContext operationContext,
191+
bool errorDuringChannelAcquisition,
192+
ServerDescription server,
193+
WriteConcern writeConcern,
194+
RetryableWriteContext context,
195+
Exception exception,
196+
int attempt,
197+
IRandom random,
198+
bool isEndTransactionOperation,
199+
bool isOperationRetryable,
200+
bool overloadErrorSeen,
201+
out TimeSpan backoff)
202+
{
203+
var methodInfo = typeof(RetryableWriteOperationExecutor)
204+
.GetMethods(BindingFlags.NonPublic | BindingFlags.Static)
205+
.Single(m => m.Name == nameof(ShouldRetry) && m.GetParameters().Length == 12);
206+
207+
var args = new object[] { operationContext, errorDuringChannelAcquisition, server, writeConcern, context, exception, attempt, random, isEndTransactionOperation, isOperationRetryable, overloadErrorSeen, default(TimeSpan) };
208+
209+
try
210+
{
211+
var result = (bool)methodInfo.Invoke(null, args);
212+
backoff = (TimeSpan)args[11];
213+
return result;
214+
}
215+
catch (TargetInvocationException ex)
216+
{
217+
throw ex.InnerException;
218+
}
219+
}
147220
}
148221
}

‎tests/MongoDB.Driver.Tests/Core/Servers/RoundTripTimeMonitorTests.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,10 @@ public void Start_should_use_serverApi()
192192
var sentMessages = connection.GetSentMessages();
193193
sentMessages.Count.Should().BeInRange(1, 2);
194194

195-
MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }");
195+
MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }");
196196
if (sentMessages.Count > 1)
197197
{
198-
MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }");
198+
MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' }, apiVersion : '1' }");
199199
}
200200
}
201201

@@ -232,10 +232,10 @@ public void RoundTripTimeMonitor_without_serverApi_but_with_loadBalancedConnecti
232232
var sentMessages = connection.GetSentMessages();
233233
sentMessages.Count.Should().BeInRange(1, 2);
234234

235-
MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }");
235+
MessageHelper.ToCommandPayload(sentMessages[0]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }");
236236
if (sentMessages.Count > 1)
237237
{
238-
MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : true, $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }");
238+
MessageHelper.ToCommandPayload(sentMessages[1]).Should().Be("{ hello : 1, helloOk : true, loadBalanced : true, backpressure : '2', $db : 'admin', $readPreference : { mode : 'primaryPreferred' } }");
239239
}
240240
}
241241

0 commit comments

Comments
 (0)