[fix] Improve upload retry resilience (#292)

Author: paul-fresquet

src/ByteSync.Client/Services/Communications/Transfers/Strategies/AzureBlobStorageUploadStrategy.cs

@@ -21,7 +21,7 @@ public async Task<UploadFileResponse> UploadAsync(FileUploaderSlice slice, FileS
         try
         {
             var options = new BlobClientOptions();
-            options.Retry.NetworkTimeout = TimeSpan.FromMinutes(1);
+            options.Retry.NetworkTimeout = TimeSpan.FromMinutes(10);
 
             slice.MemoryStream.Position = 0;
             var blob = new BlobClient(new Uri(storageLocation.Url), options);

src/ByteSync.Client/Services/Communications/Transfers/Strategies/CloudflareR2UploadStrategy.cs

@@ -29,7 +29,7 @@ public async Task<UploadFileResponse> UploadAsync(FileUploaderSlice slice, FileS
             slice.MemoryStream.Position = 0;
 
             using var httpClient = _httpClientFactory.CreateClient();
-            httpClient.Timeout = TimeSpan.FromMinutes(1);
+            httpClient.Timeout = Timeout.InfiniteTimeSpan;
             httpClient.DefaultRequestHeaders.ExpectContinue = false;
 
             // Build ReadOnlyMemory without copying when possible; fallback to ToArray otherwise

src/ByteSync.Client/Services/Communications/Transfers/Strategies/UploadFailureClassifier.cs

@@ -1,4 +1,7 @@
 using System;
+using System.IO;
+using System.Net.Http;
+using System.Net.Sockets;
 using System.Threading;
 using ByteSync.Common.Business.Communications.Transfers;
 
@@ -12,12 +15,44 @@ public static UploadFileResponse Classify(Exception exception, CancellationToken
         {
             return UploadFileResponse.ClientCancellation(exception);
         }
-        
+
         if (exception is OperationCanceledException)
         {
             return UploadFileResponse.ClientTimeout(exception);
         }
 
+        if (IsClientNetworkError(exception))
+        {
+            return UploadFileResponse.ClientNetworkError(exception);
+        }
+
         return UploadFileResponse.Failure(500, exception);
     }
+
+    private static bool IsClientNetworkError(Exception exception)
+    {
+        if (exception is not HttpRequestException and not IOException and not SocketException)
+        {
+            return false;
+        }
+
+        var current = exception;
+        while (current != null)
+        {
+            if (current is SocketException socketException)
+            {
+                return socketException.SocketErrorCode is SocketError.ConnectionReset
+                    or SocketError.ConnectionAborted
+                    or SocketError.TimedOut
+                    or SocketError.NetworkDown
+                    or SocketError.NetworkUnreachable
+                    or SocketError.HostDown
+                    or SocketError.HostUnreachable;
+            }
+
+            current = current.InnerException;
+        }
+
+        return false;
+    }
 }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/AdaptiveUploadController.cs

@@ -14,7 +14,7 @@ public class AdaptiveUploadController : IAdaptiveUploadController
     private const int MAX_CHUNK_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB
     private const int MIN_PARALLELISM = 2;
     private const int MAX_PARALLELISM = 4;
-    private const int CLIENT_TIMEOUTS_BEFORE_DOWNSCALE = 2;
+    private const int CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE = 2;
 
     private const double MULTIPLIER_2_X = 2.0;
     private const double MULTIPLIER_1_75_X = 1.75;
@@ -37,7 +37,7 @@ public class AdaptiveUploadController : IAdaptiveUploadController
     private readonly Queue<long> _recentBytes;
     private int _successesInWindow;
     private int _windowSize;
-    private int _consecutiveClientTimeouts;
+    private int _consecutiveClientNetworkIssues;
     private readonly ILogger<AdaptiveUploadController> _logger;
     private readonly object _syncRoot = new();
 
@@ -90,17 +90,17 @@ public void RecordUploadResult(UploadResult uploadResult)
         {
             if (uploadResult.FailureKind == UploadFailureKind.ClientCancellation)
             {
-                _consecutiveClientTimeouts = 0;
+                _consecutiveClientNetworkIssues = 0;
                 return;
             }
 
-            if (uploadResult.FailureKind == UploadFailureKind.ClientTimeout)
+            if (uploadResult.FailureKind is UploadFailureKind.ClientTimeout or UploadFailureKind.ClientNetworkError)
             {
-                HandleClientTimeout(uploadResult.FileId);
+                HandleClientNetworkIssue(uploadResult.FileId, uploadResult.FailureKind);
                 return;
             }
 
-            _consecutiveClientTimeouts = 0;
+            _consecutiveClientNetworkIssues = 0;
 
             EnqueueSample(uploadResult.Elapsed, uploadResult.IsSuccess, uploadResult.ActualBytes);
 
@@ -162,26 +162,27 @@ private void EnqueueSample(TimeSpan elapsed, bool isSuccess, long actualBytes)
         }
     }
 
-    private void HandleClientTimeout(string? fileId)
+    private void HandleClientNetworkIssue(string? fileId, UploadFailureKind failureKind)
     {
-        _consecutiveClientTimeouts += 1;
-        if (_consecutiveClientTimeouts < CLIENT_TIMEOUTS_BEFORE_DOWNSCALE)
+        _consecutiveClientNetworkIssues += 1;
+        if (_consecutiveClientNetworkIssues < CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE)
         {
             _logger.LogDebug(
-                "Adaptive: file {FileId} client timeout {TimeoutCount}/{Threshold}. Waiting before downscale",
+                "Adaptive: file {FileId} client network issue {FailureKind} {IssueCount}/{Threshold}. Waiting before downscale",
                 fileId ?? "-",
-                _consecutiveClientTimeouts,
-                CLIENT_TIMEOUTS_BEFORE_DOWNSCALE);
+                failureKind,
+                _consecutiveClientNetworkIssues,
+                CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE);
 
             return;
         }
 
         _logger.LogInformation(
-            "Adaptive: file {FileId} client timeout threshold reached ({TimeoutCount}). Downscaling upload settings",
+            "Adaptive: file {FileId} client network issue threshold reached ({IssueCount}). Downscaling upload settings",
             fileId ?? "-",
-            _consecutiveClientTimeouts);
-        _consecutiveClientTimeouts = 0;
-        Downscale(fileId, "client timeouts");
+            _consecutiveClientNetworkIssues);
+        _consecutiveClientNetworkIssues = 0;
+        Downscale(fileId, "client network issues");
     }
 
     private bool HandleBandwidthReset(bool isSuccess, int? statusCode)
@@ -395,7 +396,7 @@ private void ResetState()
             _currentChunkSizeBytes = Math.Clamp(INITIAL_CHUNK_SIZE_BYTES, MIN_CHUNK_SIZE_BYTES, MAX_CHUNK_SIZE_BYTES);
             _currentParallelism = MIN_PARALLELISM;
             _windowSize = _currentParallelism;
-            _consecutiveClientTimeouts = 0;
+            _consecutiveClientNetworkIssues = 0;
         }
 
         ResetWindow();

src/ByteSync.Client/Services/Communications/Transfers/Uploading/FileSlicer.cs

@@ -63,7 +63,10 @@ public async Task SliceAndEncryptAsync(SharedFileDefinition sharedFileDefinition
                         _semaphoreSlim.Release();
                     }
 
-                    await _availableSlices.Writer.WriteAsync(fileUploaderSlice);
+                    if (!await TryWriteSliceAsync(fileUploaderSlice))
+                    {
+                        return;
+                    }
                 }
                 else
                 {
@@ -126,7 +129,10 @@ public async Task SliceAndEncryptAdaptiveAsync(SharedFileDefinition sharedFileDe
                         _semaphoreSlim.Release();
                     }
 
-                    await _availableSlices.Writer.WriteAsync(fileUploaderSlice);
+                    if (!await TryWriteSliceAsync(fileUploaderSlice))
+                    {
+                        return;
+                    }
                 }
                 else
                 {
@@ -152,4 +158,20 @@ public async Task SliceAndEncryptAdaptiveAsync(SharedFileDefinition sharedFileDe
             _availableSlices.Writer.TryComplete(ex);
         }
     }
+
+    private async Task<bool> TryWriteSliceAsync(FileUploaderSlice fileUploaderSlice)
+    {
+        try
+        {
+            await _availableSlices.Writer.WriteAsync(fileUploaderSlice);
+
+            return true;
+        }
+        catch (ChannelClosedException) when (_exceptionOccurred.WaitOne(0))
+        {
+            await fileUploaderSlice.MemoryStream.DisposeAsync();
+
+            return false;
+        }
+    }
 }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/FileUploadCoordinator.cs

@@ -12,7 +12,7 @@ public class FileUploadCoordinator : IFileUploadCoordinator
     private readonly ManualResetEvent _uploadingIsFinished;
     private readonly ManualResetEvent _exceptionOccurred;
     private readonly ILogger<FileUploadCoordinator> _logger;
-    private const int CHANNEL_CAPACITY = 8;
+    private const int CHANNEL_CAPACITY = 2;
 
     public FileUploadCoordinator(ILogger<FileUploadCoordinator> logger)
     {