RetryabilityHelperTests.cs

/* Copyright 2010-present MongoDB Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.IO;
using FluentAssertions;
using MongoDB.Bson;
using MongoDB.TestHelpers.XunitExtensions;
using MongoDB.Driver.Core.Misc;
using MongoDB.Driver.Core.TestHelpers;
using Moq;
using Xunit;

namespace MongoDB.Driver.Core.Operations
{
    public class RetryabilityHelperTests
    {
        [Theory]
        [InlineData(1, false)]
        [InlineData(ServerErrorCode.InterruptedAtShutdown, true)]
        [InlineData(ServerErrorCode.InterruptedDueToReplStateChange, true)]
        [InlineData(ServerErrorCode.LegacyNotPrimary, true)]
        [InlineData(ServerErrorCode.NotWritablePrimary, true)]
        [InlineData(ServerErrorCode.NotPrimaryNoSecondaryOk, true)]
        [InlineData(ServerErrorCode.NotPrimaryOrSecondary, true)]
        [InlineData(ServerErrorCode.PrimarySteppedDown, true)]
        [InlineData(ServerErrorCode.ShutdownInProgress, true)]
        [InlineData(ServerErrorCode.HostNotFound, true)]
        [InlineData(ServerErrorCode.HostUnreachable, true)]
        [InlineData(ServerErrorCode.NetworkTimeout, true)]
        [InlineData(ServerErrorCode.SocketException, true)]
        [InlineData(ServerErrorCode.WriteConcernTimeout, false)]
        [InlineData(ServerErrorCode.ExceededTimeLimit, true)]
        public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError_for_MongoWriteConcernException_when_required(int errorCode, bool shouldAddErrorLabel)
        {
            var exception = CoreExceptionHelper.CreateMongoWriteConcernException(BsonDocument.Parse($"{{ writeConcernError : {{ code : {errorCode} }} }}"));
            var maxWireVersion = Feature.ServerReturnsRetryableWriteErrorLabel.LastNotSupportedWireVersion;
            var connectionDescription = OperationTestHelper.CreateConnectionDescription(maxWireVersion);

            RetryabilityHelper.AddRetryableWriteErrorLabelIfRequired(exception, connectionDescription);

            var hasRetryableWriteErrorLabel = exception.HasErrorLabel("RetryableWriteError");
            hasRetryableWriteErrorLabel.Should().Be(shouldAddErrorLabel);
        }

        [Theory]
        [ParameterAttributeData]
        public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError_for_network_errors([Values(false, true)] bool serverReturnsRetryableWriteErrorLabel)
        {
            var exception = (MongoException)CoreExceptionHelper.CreateException(typeof(MongoConnectionException));
            var feature = Feature.ServerReturnsRetryableWriteErrorLabel;
            var wireVersion = serverReturnsRetryableWriteErrorLabel ? feature.FirstSupportedWireVersion : feature.LastNotSupportedWireVersion;
            var connectionDescription = OperationTestHelper.CreateConnectionDescription(wireVersion);

            RetryabilityHelper.AddRetryableWriteErrorLabelIfRequired(exception, connectionDescription);

            var hasRetryableWriteErrorLabel = exception.HasErrorLabel("RetryableWriteError");
            hasRetryableWriteErrorLabel.Should().BeTrue();
        }

        [Theory]
        [InlineData(1, false)]
        [InlineData(typeof(MongoCursorNotFoundException), false)]
        [InlineData(typeof(MongoConnectionException), true)]
        [InlineData(typeof(MongoNodeIsRecoveringException), true)]
        [InlineData(typeof(MongoNotPrimaryException), true)]
        [InlineData(ServerErrorCode.HostNotFound, true)]
        [InlineData(ServerErrorCode.HostUnreachable, true)]
        [InlineData(ServerErrorCode.NetworkTimeout, true)]
        [InlineData(ServerErrorCode.SocketException, true)]
        [InlineData(ServerErrorCode.WriteConcernTimeout, false)]
        [InlineData(ServerErrorCode.ExceededTimeLimit, true)]
        public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError_when_required(object exceptionDescription, bool shouldAddErrorLabel)
        {
            MongoException exception;
            if (exceptionDescription is Type exceptionType)
            {
                exception = (MongoException)CoreExceptionHelper.CreateException(exceptionType);
            }
            else
            {
                exception = CoreExceptionHelper.CreateMongoCommandException((int)exceptionDescription);
            }
            var maxWireVersion = Feature.ServerReturnsRetryableWriteErrorLabel.LastNotSupportedWireVersion;
            var connectionDescription = OperationTestHelper.CreateConnectionDescription(maxWireVersion);

            RetryabilityHelper.AddRetryableWriteErrorLabelIfRequired(exception, connectionDescription);

            var hasRetryableWriteErrorLabel = exception.HasErrorLabel("RetryableWriteError");
            hasRetryableWriteErrorLabel.Should().Be(shouldAddErrorLabel);
        }

        [Theory]
        [InlineData(1, 2, 100, 10000, 0, 200)]
        [InlineData(2, 2, 100, 10000, 0, 400)]
        [InlineData(3, 2, 100, 10000, 0, 800)]
        [InlineData(9999, 2, 100, 10000, 0, 10000)]
        [InlineData(1, 1.5, 100, 10000, 0, 150)]
        [InlineData(2, 1.5, 100, 10000, 0, 225)]
        [InlineData(3, 1.5, 100, 10000, 0, 337)]
        [InlineData(9999, 1.5, 100, 10000, 0, 10000)]
        public void GetRetryDelayMs_should_return_expected_result(int attempt, double backoffBase, int backoffInitial, int backoffMax, int expectedRangeMin, int expectedRangeMax)
        {
            var result = RetryabilityHelper.GetRetryDelayMs(DefaultRandom.Instance, attempt, backoffBase, backoffInitial, backoffMax);

            result.Should().BeInRange(expectedRangeMin, expectedRangeMax);
        }

        [Theory]
        [InlineData(-1, 2, 100, 1000, "attempt")]
        [InlineData(0, 2, 100, 1000, "attempt")]
        [InlineData(1, -1, 100, 1000, "backoffBase")]
        [InlineData(1, 0, 100, 1000, "backoffBase")]
        [InlineData(1, 2, -1, 1000, "backoffInitial")]
        [InlineData(1, 2, 0, 1000, "backoffInitial")]
        [InlineData(1, 2, 100, -1, "backoffMax")]
        [InlineData(1, 2, 100, 0, "backoffMax")]
        [InlineData(1, 2, 100, 50, "backoffMax")]
        public void GetRetryDelayMs_throws_on_wrong_parameters(int attempt, double backoffBase, int backoffInitial, int backoffMax, string expectedParameterName)
        {
            var exception = Record.Exception(() => RetryabilityHelper.GetRetryDelayMs(DefaultRandom.Instance, attempt, backoffBase, backoffInitial, backoffMax));

            exception.Should().BeOfType<ArgumentOutOfRangeException>().Subject
                .ParamName.Should().Be(expectedParameterName);
        }

        [Theory]
        [InlineData("{ txnNumber : 1 }", true)]
        [InlineData("{ commitTransaction : 1 }", true)]
        [InlineData("{ abortTransaction : 1 }", true)]
        [InlineData("{ ping : 1 }", false)]
        public void IsCommandRetryable_should_return_expected_result(string command, bool isRetryable)
        {
            var commandDocument = BsonDocument.Parse(command);

            var result = RetryabilityHelper.IsCommandRetryable(commandDocument);

            result.Should().Be(isRetryable);
        }

        [Theory]
        [InlineData(typeof(MongoConnectionException), true)]
        [InlineData(typeof(MongoConnectionClosedException), false)]
        [InlineData(typeof(MongoNodeIsRecoveringException), true)]
        [InlineData(typeof(MongoNotPrimaryException), true)]
        [InlineData(typeof(MongoCursorNotFoundException), true)]
        [InlineData(typeof(MongoConnectionPoolPausedException), true)]
        [InlineData(ServerErrorCode.HostNotFound, true)]
        [InlineData(ServerErrorCode.HostUnreachable, true)]
        [InlineData(ServerErrorCode.NetworkTimeout, true)]
        [InlineData(ServerErrorCode.ShutdownInProgress, true)]
        [InlineData(ServerErrorCode.PrimarySteppedDown, true)]
        [InlineData(ServerErrorCode.ExceededTimeLimit, true)]
        [InlineData(ServerErrorCode.SocketException, true)]
        [InlineData(ServerErrorCode.LegacyNotPrimary, true)]
        [InlineData(ServerErrorCode.NotWritablePrimary, true)]
        [InlineData(ServerErrorCode.InterruptedAtShutdown, true)]
        [InlineData(ServerErrorCode.InterruptedDueToReplStateChange, true)]
        [InlineData(ServerErrorCode.NotPrimaryNoSecondaryOk, true)]
        [InlineData(ServerErrorCode.NotPrimaryOrSecondary, true)]
        [InlineData(ServerErrorCode.StaleShardVersion, true)]
        [InlineData(ServerErrorCode.StaleEpoch, true)]
        [InlineData(ServerErrorCode.StaleConfig, true)]
        [InlineData(ServerErrorCode.RetryChangeStream, true)]
        [InlineData(ServerErrorCode.FailedToSatisfyReadPreference, true)]
        [InlineData(ServerErrorCode.ElectionInProgress, false)]
        [InlineData(ServerErrorCode.WriteConcernTimeout, false)]
        [InlineData(ServerErrorCode.CappedPositionLost, false)]
        [InlineData(ServerErrorCode.CursorKilled, false)]
        [InlineData(ServerErrorCode.Interrupted, false)]
        public void IsResumableChangeStreamException_should_return_expected_result_for_servers_with_old_behavior(object exceptionDescription, bool isResumable)
        {
            MongoException exception;
            if (exceptionDescription is Type exceptionType)
            {
                exception = (MongoException)CoreExceptionHelper.CreateException(exceptionType);
            }
            else
            {
                exception = CoreExceptionHelper.CreateMongoCommandException((int)exceptionDescription);
            }

            var result = RetryabilityHelper.IsResumableChangeStreamException(exception, Feature.ServerReturnsResumableChangeStreamErrorLabel.LastNotSupportedWireVersion);

            result.Should().Be(isResumable);
        }

        [Theory]
        [ParameterAttributeData]
        public void IsResumableChangeStreamException_should_return_expected_result_for_servers_with_new_behavior([Values(false, true)] bool hasResumableChangeStreamErrorLabel)
        {
            var exception = CoreExceptionHelper.CreateMongoCommandException(-1);
            if (hasResumableChangeStreamErrorLabel)
            {
                exception.AddErrorLabel("ResumableChangeStreamError");
            }

            var result = RetryabilityHelper.IsResumableChangeStreamException(exception, Feature.ServerReturnsResumableChangeStreamErrorLabel.FirstSupportedWireVersion);

            result.Should().Be(hasResumableChangeStreamErrorLabel);
        }

        [Theory]
        [InlineData(typeof(MongoConnectionException), true)] // network exception
        [InlineData(typeof(MongoConnectionClosedException), false)]
        [InlineData(typeof(MongoCursorNotFoundException), true)]
        [InlineData(typeof(MongoConnectionPoolPausedException), true)]
        public void IsResumableChangeStreamException_should_return_expected_result_for_servers_with_new_behavior_and_errors(Type exceptionType, bool isResumable)
        {
            var exception = (MongoException)CoreExceptionHelper.CreateException(exceptionType);

            var result = RetryabilityHelper.IsResumableChangeStreamException(exception, Feature.ServerReturnsResumableChangeStreamErrorLabel.FirstSupportedWireVersion);

            result.Should().Be(isResumable);
        }

        [Theory]
        [InlineData(ServerErrorCode.ReauthenticationRequired, "saslStart", false)]
        [InlineData(ServerErrorCode.ReauthenticationRequired, "saslContinue", false)]
        [InlineData(ServerErrorCode.ReauthenticationRequired, "saslDummy", true)]
        [InlineData(ServerErrorCode.ReauthenticationRequired, "dummy", true)]
        [InlineData(1, "saslStart", false)]
        [InlineData(1, "saslContinue", false)]
        [InlineData(1, "saslNotExisted", false)]
        [InlineData(1, "dummy", false)]
        public void IsRetryableCommandAuthenticationException_should_return_expected_result_using_exception_type(int errorCode, string commandName, bool expectedResult)
        {
            var exception = CoreExceptionHelper.CreateMongoCommandException(errorCode);

            var result = RetryabilityHelper.IsReauthenticationRequested(exception, new BsonDocument(commandName, 1));

            result.Should().Be(expectedResult);
        }

        [Theory]
        [InlineData(typeof(IOException), false)]
        [InlineData(typeof(MongoCursorNotFoundException), false)]
        [InlineData(typeof(MongoConnectionException), true)]
        [InlineData(typeof(MongoNodeIsRecoveringException), true)]
        [InlineData(typeof(MongoNotPrimaryException), true)]
        public void IsRetryableReadException_should_return_expected_result_using_exception_type(Type exceptionType, bool expectedResult)
        {
            var exception = CoreExceptionHelper.CreateException(exceptionType);

            var result = RetryabilityHelper.IsRetryableReadException(exception);

            result.Should().Be(expectedResult);
        }

        [Theory]
        [InlineData(1, false)]
        [InlineData(ServerErrorCode.HostNotFound, true)]
        [InlineData(ServerErrorCode.HostUnreachable, true)]
        [InlineData(ServerErrorCode.NetworkTimeout, true)]
        [InlineData(ServerErrorCode.SocketException, true)]
        public void IsRetryableReadException_should_return_expected_result_using_code(int code, bool expectedResult)
        {
            var exception = CoreExceptionHelper.CreateMongoCommandException(code);

            var result = RetryabilityHelper.IsRetryableReadException(exception);

            result.Should().Be(expectedResult);
        }

        [Theory]
        [ParameterAttributeData]
        public void IsRetryableWriteException_should_return_expected_result([Values(false, true)] bool hasRetryableWriteLabel)
        {
            var exception = CoreExceptionHelper.CreateMongoCommandException(-1);
            if (hasRetryableWriteLabel)
            {
                exception.AddErrorLabel("RetryableWriteError");
            }

            var result = RetryabilityHelper.IsRetryableWriteException(exception);

            result.Should().Be(hasRetryableWriteLabel);
        }

        [Fact]
        public void GetBaseBackoffMs_should_return_null_for_non_command_exception()
        {
            var exception = CoreExceptionHelper.CreateException(typeof(MongoConnectionException));

            var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);

            baseBackoffMs.Should().Be(null);
        }

        [Theory]
        [InlineData("{ ok : 0, code : 2 }", null)]
        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 0 }", null)]
        [InlineData("{ ok : 0, code : 2, baseBackoffMS : -5 }", null)]
        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 'not-a-number' }", null)]
        [InlineData("{ ok : 0, code : 2, baseBackoffMS : 50 }", 50)]
        [InlineData("{ ok : 0, code : 2, baseBackoffMS : NumberLong(9999999999) }", int.MaxValue)]
        public void GetBaseBackoffMs_should_return_expected_result(string resultJson, int? expectedValue)
        {
            var result = BsonDocument.Parse(resultJson);
            var exception = CoreExceptionHelper.CreateMongoCommandExceptionWithLabels(result);

            var baseBackoffMs = RetryabilityHelper.GetBaseBackoffMs(exception);

            baseBackoffMs.Should().Be(expectedValue);
        }

        [Theory]
        [InlineData(1, null, 0, 200)]
        [InlineData(2, null, 0, 400)]
        [InlineData(1, 50, 0, 100)]
        [InlineData(2, 50, 0, 200)]
        [InlineData(1, 10000, 0, 10000)]
        [InlineData(2, 20000, 0, 10000)]
        public void GetOperationRetryBackoffDelay_should_apply_baseBackoffMs_override(int attempt, int? baseBackoffMs, int expectedRangeMin, int expectedRangeMax)
        {
            var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, DefaultRandom.Instance, baseBackoffMs);

            result.TotalMilliseconds.Should().BeInRange(expectedRangeMin, expectedRangeMax);
        }

        [Theory]
        [InlineData(1, 10000, 10000)]
        [InlineData(2, 20000, 10000)]
        public void GetOperationRetryBackoffDelay_should_limit_override_to_MaxBackoff(int attempt, int baseBackoffMs, int expectedMs)
        {
            var randomMock = new Mock<IRandom>();
            randomMock.Setup(r => r.NextDouble()).Returns(1.0);

            var result = RetryabilityHelper.GetOperationRetryBackoffDelay(attempt, randomMock.Object, baseBackoffMs);

            result.TotalMilliseconds.Should().Be(expectedMs);
        }
    }
}