Catch Send/Receive errors and make them sync errors (#1924)

LfMergeBridgeResult instances now contain ErrorEncountered property,
passed through to the IProgress instance they wrap.

Author: rmunn

backend/FwHeadless/Services/CombiningProgress.cs

@@ -0,0 +1,71 @@
+using SIL.Progress;
+
+namespace FwHeadless.Services;
+
+public class CombiningProgress(params ICollection<IProgress?> progresses) : IProgress
+{
+
+    public bool ShowVerbose
+    {
+        set { foreach (var p in progresses) { if (p is not null) p.ShowVerbose = value; } }
+    }
+
+    public bool CancelRequested
+    {
+        get => progresses.Any(p => p?.CancelRequested ?? false);
+        set { foreach (var p in progresses) { if (p is not null) p.CancelRequested = value; } }
+    }
+
+    public bool ErrorEncountered
+    {
+        get => progresses.Any(p => p?.ErrorEncountered ?? false);
+        set { foreach (var p in progresses) { if (p is not null) p.ErrorEncountered = value; } }
+    }
+
+    public IProgressIndicator ProgressIndicator
+    {
+        get => progresses.FirstOrDefault(p => p?.ProgressIndicator is not null)?.ProgressIndicator ?? null!;
+        set { foreach (var p in progresses) { if (p is not null) p.ProgressIndicator = value; } }
+    }
+
+    public SynchronizationContext SyncContext
+    {
+        get => progresses.FirstOrDefault(p => p?.SyncContext is not null)?.SyncContext ?? null!;
+        set { foreach (var p in progresses) { if (p is not null) p.SyncContext = value; } }
+    }
+
+    public void WriteError(string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteError(message, args); }
+    }
+
+    public void WriteException(Exception error)
+    {
+        foreach (var p in progresses) { p?.WriteException(error); }
+    }
+
+    public void WriteMessage(string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteMessage(message, args); }
+    }
+
+    public void WriteMessageWithColor(string colorName, string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteMessageWithColor(colorName, message, args); }
+    }
+
+    public void WriteStatus(string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteStatus(message, args); }
+    }
+
+    public void WriteVerbose(string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteVerbose(message, args); }
+    }
+
+    public void WriteWarning(string message, params object[] args)
+    {
+        foreach (var p in progresses) { p?.WriteWarning(message, args); }
+    }
+}

backend/FwHeadless/Services/SendReceiveHelpers.cs

@@ -4,6 +4,8 @@
 
 namespace FwHeadless.Services;
 
+public class SendReceiveException(string? message) : Exception(message);
+
 public static class SendReceiveHelpers
 {
     private const string HgUsername = "FieldWorks Lite";
@@ -17,17 +19,26 @@ public record SendReceiveAuth(string Username, string Password)
         public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, config.LexboxPassword) { }
     };
 
-    public record LfMergeBridgeResult(string Output, string ProgressMessages);
+    public record LfMergeBridgeResult(string Output)
+    {
+        private readonly IProgress? _progress = null;
+        public bool ErrorEncountered => _progress?.ErrorEncountered ?? false;
+        public LfMergeBridgeResult(string output, IProgress progress) : this(output)
+        {
+            _progress = progress;
+        }
+    }
 
     private static async Task<LfMergeBridgeResult> CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions, IProgress? progress = null)
     {
         var sbProgress = new StringBuilderProgress();
+        var combinedProgress = new CombiningProgress(sbProgress, progress);
         var lfMergeBridgeOutputForClient = await Task.Run(() =>
         {
-            LfMergeBridge.LfMergeBridge.Execute(method, progress ?? sbProgress, flexBridgeOptions.ToDictionary(), out var output);
+            LfMergeBridge.LfMergeBridge.Execute(method, combinedProgress, flexBridgeOptions.ToDictionary(), out var output);
             return output;
         });
-        return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress == null ? sbProgress.ToString() : "");
+        return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, sbProgress);
     }
 
     private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendReceiveAuth? auth, bool forChorus = true)

backend/FwHeadless/Services/SyncHostedService.cs

@@ -144,7 +144,15 @@ public async Task<SyncJobResult> ExecuteSync(CancellationToken stoppingToken)
         logger.LogDebug("crdtFile: {crdtFile}", crdtFile);
         logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath);
 
-        var fwdataApi = await SetupFwData(fwDataProject, projectCode);
+        FwDataMiniLcmApi? fwdataApi;
+        try
+        {
+            fwdataApi = await SetupFwData(fwDataProject, projectCode);
+        }
+        catch (SendReceiveException e)
+        {
+            return new SyncJobResult(SyncJobStatusEnum.SendReceiveFailed, e.Message);
+        }
         //always do this as existing projects need to run this even if they didn't S&R due to no pending changes
         await mediaFileService.SyncMediaFiles(fwdataApi.Cache);
 
@@ -181,6 +189,10 @@ public async Task<SyncJobResult> ExecuteSync(CancellationToken stoppingToken)
         {
             var srResult2 = await srService.SendReceive(fwDataProject, projectCode);
             logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
+            if (srResult2.ErrorEncountered)
+            {
+                return new SyncJobResult(SyncJobStatusEnum.SendReceiveFailed, $"Send/Receive after CRDT sync failed: {srResult2.Output}");
+            }
         }
         activity?.SetStatus(ActivityStatusCode.Ok, "Sync finished");
         return new SyncJobResult(result);
@@ -199,6 +211,10 @@ private async Task<FwDataMiniLcmApi> SetupFwData(FwDataProject fwDataProject, st
             {
                 var srResult = await srService.SendReceive(fwDataProject, projectCode);
                 logger.LogInformation("Send/Receive result before CRDT sync: {srResult}", srResult.Output);
+                if (srResult.ErrorEncountered)
+                {
+                    throw new SendReceiveException($"Send/Receive before CRDT sync failed: {srResult.Output}");
+                }
             }
         }
         else