Execute message-append interrupt as a separate command to avoid lock contention

Routing single appends through AppendMessages merged the INSERT and the
interrupt UPDATE into one command/connection in the SqlServer and MariaDB
stores. Holding the insert's locks while the interrupt UPDATE ran deadlocked
tight message-exchange loops (ping-pong) where two executing flows interrupt
each other - SQL Server's unbounded lock-wait made it hang indefinitely.

Execute the interrupt as a separate command after the insert (matching the
previous append-then-schedule design). PostgreSQL already runs them as
separate auto-committing batch commands and is unaffected.

Author: stidsborg

Stores/MariaDB/Cleipnir.ResilientFunctions.MariaDB/MariaDbMessageStore.cs

@@ -58,12 +58,17 @@ public async Task AppendMessages(IReadOnlyList<StoredIdAndMessage> messages)
         var appendMessagesCommand = _sqlGenerator.AppendMessages(messages);
         var interruptsCommand = _sqlGenerator.Interrupt(storedIds);
 
-        var command = StoreCommand.Merge(appendMessagesCommand, interruptsCommand);
-
         await using var conn = await DatabaseHelper.CreateOpenConnection(_connectionString);
-        await using var sqlCommand = command.ToSqlCommand(conn);
 
-        await sqlCommand.ExecuteNonQueryAsync();
+        // The append and the interrupt are executed as separate commands - not merged into one - so the
+        // insert's locks are released before the interrupt UPDATE runs. Merging them caused lock contention
+        // that deadlocked tight message-exchange loops (e.g. ping-pong) where two executing flows interrupt
+        // each other.
+        await using var appendCommand = appendMessagesCommand.ToSqlCommand(conn);
+        await appendCommand.ExecuteNonQueryAsync();
+
+        await using var interruptCommand = interruptsCommand.ToSqlCommand(conn);
+        await interruptCommand.ExecuteNonQueryAsync();
     }
 
     private string? _replaceMessageSql;

Stores/SqlServer/Cleipnir.ResilientFunctions.SqlServer/SqlServerMessageStore.cs

@@ -75,9 +75,7 @@ public async Task AppendMessages(IReadOnlyList<StoredIdAndMessage> messages)
             INSERT INTO {_tablePrefix}_Messages
                 (Id, Replica, Content)
             VALUES
-                 {messages.Select((_, i) => $"(@Id{i}, COALESCE((SELECT Owner FROM {_tablePrefix} WHERE Id = @Id{i}), @Replica{i}), @Content{i})").StringJoin($",{Environment.NewLine}")};
-
-            {interuptsSql.Sql}";
+                 {messages.Select((_, i) => $"(@Id{i}, COALESCE((SELECT Owner FROM {_tablePrefix} WHERE Id = @Id{i}), @Replica{i}), @Content{i})").StringJoin($",{Environment.NewLine}")};";
 
         await using var command = new SqlCommand(sql, conn);
         for (var i = 0; i < messages.Count; i++)
@@ -88,10 +86,14 @@ INSERT INTO {_tablePrefix}_Messages
             command.Parameters.AddWithValue($"@Replica{i}", replica.AsGuid);
             command.Parameters.AddWithValue($"@Content{i}", content);
         }
-        foreach (var (value, name) in interuptsSql.Parameters)
-            command.Parameters.AddWithValue(name, value);
-
         await command.ExecuteNonQueryAsync();
+
+        // The interrupt is executed as a separate command - not merged into the insert above - so the
+        // insert's locks are released before the interrupt UPDATE runs. Merging them caused lock contention
+        // that deadlocked tight message-exchange loops (e.g. ping-pong) where two executing flows interrupt
+        // each other (SQL Server's lock-wait is unbounded, so it hung indefinitely).
+        await using var interruptCommand = interuptsSql.ToSqlCommand(conn);
+        await interruptCommand.ExecuteNonQueryAsync();
     }
 
     private string? _replaceMessageSql;