[fix] Harden upload retries on slow networks

Let adaptive upload timeouts control storage uploads and classify client-side transport resets so slow, unstable connections downscale instead of surfacing as server failures.

Made-with: Cursor

Author: paul-fresquet

src/ByteSync.Client/Services/Communications/Transfers/Strategies/AzureBlobStorageUploadStrategy.cs

@@ -21,7 +21,7 @@ public async Task<UploadFileResponse> UploadAsync(FileUploaderSlice slice, FileS
         try
         {
             var options = new BlobClientOptions();
-            options.Retry.NetworkTimeout = TimeSpan.FromMinutes(1);
+            options.Retry.NetworkTimeout = TimeSpan.FromMinutes(10);
 
             slice.MemoryStream.Position = 0;
             var blob = new BlobClient(new Uri(storageLocation.Url), options);

src/ByteSync.Client/Services/Communications/Transfers/Strategies/CloudflareR2UploadStrategy.cs

@@ -29,7 +29,7 @@ public async Task<UploadFileResponse> UploadAsync(FileUploaderSlice slice, FileS
             slice.MemoryStream.Position = 0;
 
             using var httpClient = _httpClientFactory.CreateClient();
-            httpClient.Timeout = TimeSpan.FromMinutes(1);
+            httpClient.Timeout = Timeout.InfiniteTimeSpan;
             httpClient.DefaultRequestHeaders.ExpectContinue = false;
 
             // Build ReadOnlyMemory without copying when possible; fallback to ToArray otherwise

src/ByteSync.Client/Services/Communications/Transfers/Strategies/UploadFailureClassifier.cs

@@ -1,4 +1,7 @@
 using System;
+using System.IO;
+using System.Net.Http;
+using System.Net.Sockets;
 using System.Threading;
 using ByteSync.Common.Business.Communications.Transfers;
 
@@ -12,12 +15,44 @@ public static UploadFileResponse Classify(Exception exception, CancellationToken
         {
             return UploadFileResponse.ClientCancellation(exception);
         }
-        
+
         if (exception is OperationCanceledException)
         {
             return UploadFileResponse.ClientTimeout(exception);
         }
 
+        if (IsClientNetworkError(exception))
+        {
+            return UploadFileResponse.ClientNetworkError(exception);
+        }
+
         return UploadFileResponse.Failure(500, exception);
     }
+
+    private static bool IsClientNetworkError(Exception exception)
+    {
+        if (exception is not HttpRequestException and not IOException)
+        {
+            return false;
+        }
+
+        var current = exception;
+        while (current != null)
+        {
+            if (current is SocketException socketException)
+            {
+                return socketException.SocketErrorCode is SocketError.ConnectionReset
+                    or SocketError.ConnectionAborted
+                    or SocketError.TimedOut
+                    or SocketError.NetworkDown
+                    or SocketError.NetworkUnreachable
+                    or SocketError.HostDown
+                    or SocketError.HostUnreachable;
+            }
+
+            current = current.InnerException;
+        }
+
+        return false;
+    }
 }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/AdaptiveUploadController.cs

@@ -94,9 +94,9 @@ public void RecordUploadResult(UploadResult uploadResult)
                 return;
             }
 
-            if (uploadResult.FailureKind == UploadFailureKind.ClientTimeout)
+            if (uploadResult.FailureKind is UploadFailureKind.ClientTimeout or UploadFailureKind.ClientNetworkError)
             {
-                HandleClientTimeout(uploadResult.FileId);
+                HandleClientNetworkIssue(uploadResult.FileId, uploadResult.FailureKind);
                 return;
             }
 
@@ -162,26 +162,27 @@ private void EnqueueSample(TimeSpan elapsed, bool isSuccess, long actualBytes)
         }
     }
 
-    private void HandleClientTimeout(string? fileId)
+    private void HandleClientNetworkIssue(string? fileId, UploadFailureKind failureKind)
     {
         _consecutiveClientTimeouts += 1;
         if (_consecutiveClientTimeouts < CLIENT_TIMEOUTS_BEFORE_DOWNSCALE)
         {
             _logger.LogDebug(
-                "Adaptive: file {FileId} client timeout {TimeoutCount}/{Threshold}. Waiting before downscale",
+                "Adaptive: file {FileId} client network issue {FailureKind} {TimeoutCount}/{Threshold}. Waiting before downscale",
                 fileId ?? "-",
+                failureKind,
                 _consecutiveClientTimeouts,
                 CLIENT_TIMEOUTS_BEFORE_DOWNSCALE);
 
             return;
         }
 
         _logger.LogInformation(
-            "Adaptive: file {FileId} client timeout threshold reached ({TimeoutCount}). Downscaling upload settings",
+            "Adaptive: file {FileId} client network issue threshold reached ({TimeoutCount}). Downscaling upload settings",
             fileId ?? "-",
             _consecutiveClientTimeouts);
         _consecutiveClientTimeouts = 0;
-        Downscale(fileId, "client timeouts");
+        Downscale(fileId, "client network issues");
     }
 
     private bool HandleBandwidthReset(bool isSuccess, int? statusCode)

src/ByteSync.Client/Services/Communications/Transfers/Uploading/UploadAttemptTimeoutPolicy.cs

@@ -5,7 +5,7 @@ namespace ByteSync.Services.Communications.Transfers.Uploading;
 public static class UploadAttemptTimeoutPolicy
 {
     private const int AttemptTimeoutFloorSeconds = 60;
-    private const int AttemptTimeoutCeilingSeconds = 120;
+    private const int AttemptTimeoutCeilingSeconds = 180;
     private const int SecondsPerMegabyteHeuristic = 3;
     private const int RetryGrowthSeconds = 15;
     private const int StaleChunkPenaltySeconds = 5;

src/ByteSync.Common/Business/Communications/Transfers/UploadFailureKind.cs

@@ -6,4 +6,5 @@ public enum UploadFailureKind
     ServerError = 1,
     ClientCancellation = 2,
     ClientTimeout = 3,
+    ClientNetworkError = 4,
 }

src/ByteSync.Common/Business/Communications/Transfers/UploadFileResponse.cs

@@ -66,4 +66,16 @@ public static UploadFileResponse ClientTimeout(Exception exception)
             FailureKind = UploadFailureKind.ClientTimeout,
         };
     }
+
+    public static UploadFileResponse ClientNetworkError(Exception exception)
+    {
+        return new UploadFileResponse
+        {
+            IsSuccess = false,
+            StatusCode = 0,
+            ErrorMessage = exception.Message,
+            Exception = exception,
+            FailureKind = UploadFailureKind.ClientNetworkError,
+        };
+    }
 }

tests/ByteSync.Client.UnitTests/Services/Communications/Transfers/Strategies/CloudflareR2UploadStrategyTests.cs

@@ -26,15 +26,28 @@ private static FileUploaderSlice CreateSlice(int partNumber = 1, int sizeBytes =
     private static FileStorageLocation CreateLocation() => new(UploadUrl, StorageProvider.CloudflareR2);
 
     private static (CloudflareR2UploadStrategy strategy, Mock<HttpMessageHandler> handler) CreateStrategy()
+    {
+        var (strategy, handler, _) = CreateStrategyWithCreatedClients();
+        return (strategy, handler);
+    }
+
+    private static (CloudflareR2UploadStrategy strategy, Mock<HttpMessageHandler> handler, List<HttpClient> createdClients)
+        CreateStrategyWithCreatedClients()
     {
         var handler = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+        var createdClients = new List<HttpClient>();
         var factory = new Mock<IHttpClientFactory>();
         factory
             .Setup(f => f.CreateClient(It.IsAny<string>()))
-            .Returns(() => new HttpClient(handler.Object, disposeHandler: false));
+            .Returns(() =>
+            {
+                var httpClient = new HttpClient(handler.Object, disposeHandler: false);
+                createdClients.Add(httpClient);
+                return httpClient;
+            });
 
         var strategy = new CloudflareR2UploadStrategy(NullLogger<CloudflareR2UploadStrategy>.Instance, factory.Object);
-        return (strategy, handler);
+        return (strategy, handler, createdClients);
     }
 
     private static void SetupHandler(Mock<HttpMessageHandler> handler, HttpResponseMessage response)
@@ -70,6 +83,18 @@ public async Task UploadAsync_On2xx_ShouldReturnSuccess()
         response.FailureKind.Should().Be(UploadFailureKind.None);
     }
 
+    [Test]
+    public async Task UploadAsync_ShouldLetCallerTokenControlAttemptTimeout()
+    {
+        var (strategy, handler, createdClients) = CreateStrategyWithCreatedClients();
+        SetupHandler(handler, new HttpResponseMessage(HttpStatusCode.OK));
+
+        await strategy.UploadAsync(CreateSlice(), CreateLocation(), CancellationToken.None);
+
+        createdClients.Should().ContainSingle();
+        createdClients[0].Timeout.Should().Be(Timeout.InfiniteTimeSpan);
+    }
+
     [Test]
     public async Task UploadAsync_On500_ShouldReturnServerFailure_WithRealStatusCode()
     {

tests/ByteSync.Client.UnitTests/Services/Communications/Transfers/Strategies/UploadFailureClassifierTests.cs

@@ -2,6 +2,8 @@
 using ByteSync.Services.Communications.Transfers.Strategies;
 using FluentAssertions;
 using NUnit.Framework;
+using System.Net.Http;
+using System.Net.Sockets;
 
 namespace ByteSync.Client.UnitTests.Services.Communications.Transfers.Strategies;
 
@@ -80,16 +82,32 @@ public void Classify_GenericException_ShouldReturnServerError500()
     }
 
     [Test]
-    public void Classify_HttpRequestException_ShouldReturnServerError500()
+    public void Classify_HttpRequestExceptionWithoutSocketFailure_ShouldReturnServerError500()
     {
         using var cts = new CancellationTokenSource();
         var ex = new HttpRequestException("network issue");
-        
+
         var response = UploadFailureClassifier.Classify(ex, cts.Token);
-        
+
         response.IsSuccess.Should().BeFalse();
         response.StatusCode.Should().Be(500);
         response.FailureKind.Should().Be(UploadFailureKind.ServerError);
         response.Exception.Should().BeSameAs(ex);
     }
+
+    [Test]
+    public void Classify_HttpRequestExceptionWithConnectionReset_ShouldReturnClientNetworkError()
+    {
+        using var cts = new CancellationTokenSource();
+        var socketException = new SocketException((int)SocketError.ConnectionReset);
+        var ioException = new IOException("transport closed", socketException);
+        var ex = new HttpRequestException("copy failed", ioException);
+
+        var response = UploadFailureClassifier.Classify(ex, cts.Token);
+
+        response.IsSuccess.Should().BeFalse();
+        response.StatusCode.Should().Be(0);
+        response.FailureKind.Should().Be(UploadFailureKind.ClientNetworkError);
+        response.Exception.Should().BeSameAs(ex);
+    }
 }

tests/ByteSync.Client.UnitTests/Services/Communications/Transfers/Uploading/AdaptiveUploadControllerTests.cs

@@ -199,7 +199,22 @@ public void ClientTimeouts_DownscaleBelowInitialChunkSize_WhenAtMinParallelism()
         _controller.CurrentChunkSizeBytes.Should().BeLessThan(500 * 1024);
         _controller.CurrentChunkSizeBytes.Should().Be(375 * 1024);
     }
-    
+
+    [Test]
+    public void ClientNetworkErrors_DownscaleBelowInitialChunkSize_WhenAtMinParallelism()
+    {
+        // Arrange
+        _controller.CurrentParallelism.Should().Be(2);
+        _controller.CurrentChunkSizeBytes.Should().Be(500 * 1024);
+
+        // Act
+        FeedClientNetworkErrors(_controller, 2);
+
+        // Assert
+        _controller.CurrentParallelism.Should().Be(2);
+        _controller.CurrentChunkSizeBytes.Should().Be(375 * 1024);
+    }
+
     [Test]
     public void ClientTimeouts_ReduceParallelismFirst_WhenAboveMinParallelism()
     {
@@ -280,7 +295,21 @@ private static void FeedClientTimeouts(AdaptiveUploadController controller, int
                 failureKind: UploadFailureKind.ClientTimeout);
         }
     }
-    
+
+    private static void FeedClientNetworkErrors(AdaptiveUploadController controller, int count)
+    {
+        for (var i = 0; i < count; i++)
+        {
+            RecordUploadResult(
+                controller,
+                TimeSpan.FromSeconds(90),
+                isSuccess: false,
+                partNumber: i + 1,
+                statusCode: 0,
+                failureKind: UploadFailureKind.ClientNetworkError);
+        }
+    }
+
     private static void RecordUploadResult(
         AdaptiveUploadController controller,
         TimeSpan elapsed,