fix(restore): polly retry on Azure blob storage single file handle issue (#66)

* fix(move): lsn null overwrite

* fix(restore): retry restore exception azure single file handle

* test(build): azure pipeline

* fix(retry): custom retry policy time

* fix(retry): custom retry policy time

* Update Server.cs

Author: alexirion10

src/AgDatabase.cs

@@ -26,7 +26,8 @@ public interface IAgDatabase
     List<BackupMetadata> RecentBackups();
     void JoinAg();
 
-    void Restore(IEnumerable<BackupMetadata> backupOrder, Func<string, string> fileRelocation = null);
+    void Restore(IEnumerable<BackupMetadata> backupOrder, Func<int, TimeSpan> retryDurationProvider,
+      Func<string, string> fileRelocation = null);
 
     void CopyLogins(IEnumerable<LoginProperties> logins);
     IEnumerable<LoginProperties> AssociatedLogins();
@@ -100,10 +101,12 @@ public void LogBackup()
     ///   We suggest using <see cref="AgDatabaseMove" /> to assist with restores.
     /// </summary>
     /// <param name="backupOrder">An ordered list of backups to restore.</param>
+    /// <param name="retryDurationProvider">Retry duration function.</param>
     /// <param name="fileRelocation">A method to generate the new file location when moving the database.</param>
-    public void Restore(IEnumerable<BackupMetadata> backupOrder, Func<string, string> fileRelocation = null)
+    public void Restore(IEnumerable<BackupMetadata> backupOrder, Func<int, TimeSpan> retryDurationProvider,
+      Func<string, string> fileRelocation = null)
     {
-      _listener.ForEachAgInstance(s => s.Restore(backupOrder, Name, fileRelocation));
+      _listener.ForEachAgInstance(s => s.Restore(backupOrder, Name, retryDurationProvider, fileRelocation));
     }
 
     /// <summary>

src/AgDatabaseMove.Cli/Program.cs

@@ -15,6 +15,7 @@ internal class MoveArgs
     public bool DeleteSource { get; set; } = false;
   }
 
+
   internal class Program
   {
     private static void Main(string[] args)
@@ -30,7 +31,7 @@ private static void Main(string[] args)
         Overwrite = arguments.Overwrite,
         Finalize = arguments.Finalize,
         CopyLogins = arguments.CopyLogins,
-        DeleteSource = arguments.DeleteSource,
+        RetryDuration = attemptNumber => TimeSpan.FromSeconds(10 * attemptNumber),
         FileRelocator = filename =>
           RestoreFileRelocator(arguments.From.DatabaseName, arguments.To.DatabaseName, filename)
       });

src/AgDatabaseMove.cs

@@ -17,9 +17,9 @@ public class MoveOptions
     public IAgDatabase Source { get; set; }
     public IAgDatabase Destination { get; set; }
     public bool Overwrite { get; set; }
-    public bool DeleteSource { get; set; } = true;
     public bool Finalize { get; set; }
     public bool CopyLogins { get; set; }
+    public Func<int, TimeSpan> RetryDuration { get; set; }
     public Func<string, string> FileRelocator { get; set; }
   }
 
@@ -44,6 +44,7 @@ internal LoginProperties UpdateDefaultDb(LoginProperties loginProperties)
       return loginProperties;
     }
 
+
     /// <summary>
     ///   AgDatabaseMove the database to all instances of the availability group.
     ///   To join the AG, Finalize must be set.
@@ -55,13 +56,13 @@ public decimal Move(decimal? lastLsn = null)
       if(!_options.Overwrite && _options.Destination.Exists() && !_options.Destination.Restoring)
         throw new ArgumentException("Database exists and overwrite option is not set");
 
+      if(_options.Overwrite && lastLsn == null)
+        _options.Destination.Delete();
+
       if(lastLsn == null && _options.Destination.Restoring)
         throw new
           ArgumentException("lastLsn parameter can only be used if the Destination database is in a restoring state");
 
-      if(_options.Overwrite)
-        _options.Destination.Delete();
-
       _options.Source.LogBackup();
 
       var backupChain = new BackupChain(_options.Source);
@@ -73,17 +74,14 @@ public decimal Move(decimal? lastLsn = null)
       if(!backupList.Any())
         throw new BackupChainException("No backups found to restore");
 
-      _options.Destination.Restore(backupList, _options.FileRelocator);
+      _options.Destination.Restore(backupList, _options.RetryDuration, _options.FileRelocator);
 
       if(_options.CopyLogins)
         _options.Destination.CopyLogins(_options.Source.AssociatedLogins().Select(UpdateDefaultDb).ToList());
 
       if(_options.Finalize)
         _options.Destination.JoinAg();
 
-      if(_options.DeleteSource)
-        _options.Source.Delete();
-
       return backupList.Max(bl => bl.LastLsn);
     }
   }

src/BackupChain.cs

@@ -1,8 +1,6 @@
 namespace AgDatabaseMove
 {
-  using System;
   using System.Collections.Generic;
-  using System.IO;
   using System.Linq;
   using Exceptions;
   using SmoFacade;
@@ -23,9 +21,8 @@ public class BackupChain : IBackupChain
     // This also handles any striped backups
     private BackupChain(IList<BackupMetadata> recentBackups)
     {
-      if (recentBackups == null || recentBackups.Count == 0) {
+      if(recentBackups == null || recentBackups.Count == 0)
         throw new BackupChainException("There are no recent backups to form a chain");
-      }
 
       var backups = recentBackups.Distinct(new BackupMetadataEqualityComparer())
         .Where(IsValidFilePath) // A third party application caused invalid path strings to be inserted into backupmediafamily
@@ -64,35 +61,33 @@ private static IEnumerable<BackupMetadata> MostRecentFullBackup(IEnumerable<Back
       var fullBackupsOrdered = backups
         .Where(b => b.BackupType == BackupFileTools.BackupType.Full)
         .OrderByDescending(d => d.CheckpointLsn).ToList();
-      
-      if(!fullBackupsOrdered.Any()) {
-        throw new BackupChainException("Could not find any full backups");
-      }
+
+      if(!fullBackupsOrdered.Any()) throw new BackupChainException("Could not find any full backups");
 
       var targetCheckpointLsn = fullBackupsOrdered.First().CheckpointLsn;
       // get all the stripes of this backup
-      return fullBackupsOrdered.Where(fullBackup => fullBackup.CheckpointLsn == targetCheckpointLsn); 
+      return fullBackupsOrdered.Where(fullBackup => fullBackup.CheckpointLsn == targetCheckpointLsn);
     }
 
-    private static IEnumerable<BackupMetadata> MostRecentDiffBackup(IEnumerable<BackupMetadata> backups, BackupMetadata lastFullBackup)
+    private static IEnumerable<BackupMetadata> MostRecentDiffBackup(IEnumerable<BackupMetadata> backups,
+      BackupMetadata lastFullBackup)
     {
       var diffBackupsOrdered = backups
         .Where(b => b.BackupType == BackupFileTools.BackupType.Diff &&
                     b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn)
         .OrderByDescending(b => b.LastLsn).ToList();
 
-      if (!diffBackupsOrdered.Any()) {
-        return new List<BackupMetadata>();
-      }
+      if(!diffBackupsOrdered.Any()) return new List<BackupMetadata>();
       var targetLastLsn = diffBackupsOrdered.First().LastLsn;
       // get all the stripes of this backup
-      return diffBackupsOrdered.Where(diffBackup => diffBackup.LastLsn == targetLastLsn); 
+      return diffBackupsOrdered.Where(diffBackup => diffBackup.LastLsn == targetLastLsn);
     }
 
-    private static IEnumerable<BackupMetadata> NextLogBackup(IEnumerable<BackupMetadata> backups, BackupMetadata prevBackup)
+    private static IEnumerable<BackupMetadata> NextLogBackup(IEnumerable<BackupMetadata> backups,
+      BackupMetadata prevBackup)
     {
       // also gets all the stripes of the next backup
-      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log && 
+      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log &&
                                 prevBackup.LastLsn >= b.FirstLsn && prevBackup.LastLsn + 1 < b.LastLsn);
     }
 

src/SmoFacade/BackupFileTools.cs

@@ -3,7 +3,6 @@
   using System;
   using System.IO;
   using System.Linq;
-  using System.Text.RegularExpressions;
 
 
   public static class BackupFileTools
@@ -53,11 +52,9 @@ public static BackupType BackupTypeAbbrevToType(string type)
     public static bool IsValidFilePath(string path)
     {
       // A quick check before leaning on exceptions
-      if(Path.GetInvalidPathChars().Any(path.Contains)) {
-        return false;
-      }
+      if(Path.GetInvalidPathChars().Any(path.Contains)) return false;
 
-      try { 
+      try {
         // This will throw an argument exception if the path is invalid
         Path.GetFullPath(path);
         // A relative path won't help us much if the destination is another server. It needs to be rooted.

src/SmoFacade/Database.cs

@@ -71,10 +71,10 @@ public List<BackupMetadata> RecentBackups()
                   "FROM msdb.dbo.backupset s " +
                   "INNER JOIN msdb.dbo.backupmediafamily m ON s.media_set_id = m.media_set_id " +
                   "WHERE s.last_lsn >= (" +
-                    "SELECT MAX(last_lsn) FROM msdb.dbo.backupset " +
-                    "WHERE [type] = 'D' " +
-                    "AND database_name = @dbName " +
-                    "AND is_copy_only = 0" +
+                  "SELECT MAX(last_lsn) FROM msdb.dbo.backupset " +
+                  "WHERE [type] = 'D' " +
+                  "AND database_name = @dbName " +
+                  "AND is_copy_only = 0" +
                   ") " +
                   "AND s.database_name = @dbName " +
                   "AND is_copy_only = 0" +

src/SmoFacade/Listener.cs

@@ -5,6 +5,7 @@ namespace AgDatabaseMove.SmoFacade
   using System.Data.SqlClient;
   using System.Linq;
   using System.Net;
+  using System.Net.Sockets;
   using System.Threading.Tasks;
 
 
@@ -133,21 +134,20 @@ private static string AgListenerName(string dataSource)
     private static Server AgInstanceNameToServer(ref SqlConnectionStringBuilder connBuilder, string agInstanceName,
       string credentialName)
     {
-      try 
-      {
+      try {
         connBuilder.DataSource = ResolveDnsHostNameForInstance(agInstanceName, connBuilder.DataSource);
         return new Server(connBuilder.ToString(), credentialName);
       }
-      catch (Exception e)
-      {
+      catch(Exception e) {
         throw new ArgumentException($"agInstanceName param {agInstanceName} cannot be resolved by DNS", e);
       }
     }
 
     /// <summary>
     ///   Resolves 'agReplicaInstanceName' to a FQDN
     ///   However on Unix OS, when 'val' in 'Dns.GetHostEntry(val)' is not a FQDN, it fails intermittently
-    ///   Therefore, if dns lookup on just the instance name fails, we retry after appending the domain fragments from the listener to the instance name
+    ///   Therefore, if dns lookup on just the instance name fails, we retry after appending the domain fragments from the
+    ///   listener to the instance name
     /// </summary>
     /// <param name="agReplicaInstanceName"> The name for an instance within the AG (for which we are trying to get the FQDN)</param>
     /// <param name="agListenerDomain"> The complete domain for the AG listener</param>
@@ -159,12 +159,10 @@ private static string ResolveDnsHostNameForInstance(string agReplicaInstanceName
       var (instanceName, instancePortOrNamedInstance) = SplitDomainAndPort(agReplicaInstanceName);
       var preferredPortOrNamedInstance = GetPreferredPort(instancePortOrNamedInstance, listenerPortOrNamedInstance);
 
-      try
-      {
+      try {
         return $"{Dns.GetHostEntry(instanceName).HostName}{preferredPortOrNamedInstance}";
       }
-      catch (System.Net.Sockets.SocketException)
-      {
+      catch(SocketException) {
         // Re-try by appending the domain fragments from listener to the instance name 
         // However, we don't need the listener's "host name" (first fragment), so we need to strip that off
         // eg: if listener is "abc.def.ghi" we want to append only ".def.ghi" to the instance name
@@ -181,12 +179,10 @@ private static string ResolveDnsHostNameForInstance(string agReplicaInstanceName
     // If both are same type, then prioritize instance over listener (in almost all cases they should be identical)
     internal static string GetPreferredPort(string instancePortOrNamedInstance, string listenerPortOrNamedInstance)
     {
-      if (string.IsNullOrEmpty(instancePortOrNamedInstance) || string.IsNullOrEmpty(listenerPortOrNamedInstance))
-      {
-        return (instancePortOrNamedInstance ?? listenerPortOrNamedInstance);
-      }
-      return instancePortOrNamedInstance.StartsWith("\\") && listenerPortOrNamedInstance.StartsWith(",") 
-        ? listenerPortOrNamedInstance 
+      if(string.IsNullOrEmpty(instancePortOrNamedInstance) || string.IsNullOrEmpty(listenerPortOrNamedInstance))
+        return instancePortOrNamedInstance ?? listenerPortOrNamedInstance;
+      return instancePortOrNamedInstance.StartsWith("\\") && listenerPortOrNamedInstance.StartsWith(",")
+        ? listenerPortOrNamedInstance
         : instancePortOrNamedInstance;
     }
 
@@ -196,17 +192,13 @@ internal static (string domain, string port) SplitDomainAndPort(string domainAnd
       var domain = domainAndPort;
       var splitValue = domainAndPort.Contains(',') ? "," : domainAndPort.Contains('\\') ? "\\" : null;
 
-      if(splitValue == null)
-      {
-        return (domain, null);
-      }
+      if(splitValue == null) return (domain, null);
 
       var fragments = domainAndPort.Split(splitValue.ToCharArray(), 2);
       domain = fragments[0];
       var port = $"{splitValue}{fragments[1]}";
 
       return (domain, port);
     }
-
   }
 }

src/SmoFacade/Server.cs

@@ -9,6 +9,7 @@ namespace AgDatabaseMove.SmoFacade
   using Exceptions;
   using Microsoft.SqlServer.Management.Common;
   using Microsoft.SqlServer.Management.Smo;
+  using Polly;
 
 
   /// <summary>
@@ -125,10 +126,20 @@ public Database Database(string dbName)
     /// </summary>
     /// <param name="backupOrder">An ordered list of backups to apply.</param>
     /// <param name="databaseName">Database to restore to.</param>
+    /// <param name="retryDurationProvider">Retry duration function. Retry 10 times, input retry number, output timespan to wait</param>
     /// <param name="fileRelocation">Option for renaming files during the restore.</param>
     public void Restore(IEnumerable<BackupMetadata> backupOrder, string databaseName,
+      Func<int, TimeSpan> retryDurationProvider,
       Func<string, string> fileRelocation = null)
     {
+      var policy = Policy
+        .Handle<ExecutionFailureException>(e => e.InnerException != null
+                                                && e.InnerException is SqlException
+                                                && e.InnerException.Message
+                                                  .Contains("The process cannot access the file because it is being used by another process"))
+        .WaitAndRetry(10, retryDurationProvider);
+
+
       var restore = new Restore { Database = databaseName, NoRecovery = true };
 
       foreach(var backup in backupOrder) {
@@ -142,7 +153,8 @@ public void Restore(IEnumerable<BackupMetadata> backupOrder, string databaseName
         var defaultFileLocations = DefaultFileLocations();
         if(defaultFileLocations != null) {
           restore.RelocateFiles.Clear();
-          foreach(var file in restore.ReadFileList(_server).AsEnumerable()) {
+          var fileList = policy.Execute(() => restore.ReadFileList(_server).AsEnumerable());
+          foreach(var file in fileList) {
             var physicalName = (string)file["PhysicalName"];
             var fileName = Path.GetFileName(physicalName) ??
                            throw new InvalidBackupException($"Physical name in backup is incomplete: {physicalName}");
@@ -161,7 +173,7 @@ public void Restore(IEnumerable<BackupMetadata> backupOrder, string databaseName
 
         _server.ConnectionContext.StatementTimeout = 86400; // 60 * 60 * 24 = 24 hours
 
-        restore.SqlRestore(_server);
+        policy.Execute(() => restore.SqlRestore(_server));
         restore.Devices.Remove(backupDeviceItem);
       }
     }
@@ -227,4 +239,4 @@ public void EnsureLogins(IEnumerable<LoginProperties> newLogins)
       }
     }
   }
-}
+}