Skip to content

Commit 43a0983

Browse files
AdityaNPLAdityaNPL
authored
Merge pull request #101 from factset/feat/striped-backups
feat(striped-backup): supporting striped backups
2 parents 40717c9 + 68862f5 commit 43a0983

7 files changed

Lines changed: 180 additions & 53 deletions

File tree

‎src/AgDatabase.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public interface IAgDatabase
3131
List<BackupMetadata> RecentBackups();
3232
void JoinAg();
3333

34-
void Restore(IEnumerable<BackupMetadata> backupOrder, Func<int, TimeSpan> retryDurationProvider,
34+
void Restore(IEnumerable<StripedBackupSet> stripedBackupSetChain, Func<int, TimeSpan> retryDurationProvider,
3535
Func<string, string> fileRelocation = null);
3636

3737
void RenameLogicalFileName(Func<string, string> fileRenamer);
@@ -118,10 +118,10 @@ public void LogBackup()
118118
/// <param name="backupOrder">An ordered list of backups to restore.</param>
119119
/// <param name="retryDurationProvider">Retry duration function.</param>
120120
/// <param name="fileRelocation">A method to generate the new file location when moving the database.</param>
121-
public void Restore(IEnumerable<BackupMetadata> backupOrder, Func<int, TimeSpan> retryDurationProvider,
121+
public void Restore(IEnumerable<StripedBackupSet> stripedBackupSetChain, Func<int, TimeSpan> retryDurationProvider,
122122
Func<string, string> fileRelocation = null)
123123
{
124-
_listener.ForEachAgInstance(s => s.Restore(backupOrder, Name, retryDurationProvider, fileRelocation));
124+
_listener.ForEachAgInstance(s => s.Restore(stripedBackupSetChain, Name, retryDurationProvider, fileRelocation));
125125
}
126126

127127
public void RenameLogicalFileName(Func<string, string> fileRenamer)

‎src/AgDatabaseMove.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,10 @@ public decimal Move(decimal? lastLsn = null)
7373

7474
if(!backupList.Any())
7575
throw new BackupChainException("No backups found to restore");
76+
77+
var stripedBackupSetChain = StripedBackupSet.GetStripedBackupSetChain(backupList);
7678

77-
_options.Destination.Restore(backupList, _options.RetryDuration, _options.FileRelocator);
79+
_options.Destination.Restore(stripedBackupSetChain, _options.RetryDuration, _options.FileRelocator);
7880

7981
if(_options.CopyLogins)
8082
foreach(var loginProperty in _options.Source.AssociatedLogins().Select(UpdateDefaultDb))

‎src/BackupChain.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ private BackupChain(IList<BackupMetadata> recentBackups)
2424
if(recentBackups == null || recentBackups.Count == 0)
2525
throw new BackupChainException("There are no recent backups to form a chain");
2626

27-
var backups = recentBackups.Distinct(new BackupMetadataEqualityComparer())
27+
var backups = recentBackups.Distinct(BackupMetadataEqualityComparer.Instance)
2828
.Where(IsValidFilePath) // A third party application caused invalid path strings to be inserted into backupmediafamily
2929
.ToList();
3030

@@ -92,9 +92,9 @@ private static IEnumerable<BackupMetadata> NextLogBackup(IEnumerable<BackupMetad
9292
{
9393
// also gets all the stripes of the next backup
9494
return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log &&
95-
prevBackup.LastLsn >= b.FirstLsn &&
95+
prevBackup.LastLsn >= b.FirstLsn &&
9696
prevBackup.LastLsn <= b.LastLsn &&
97-
!new BackupMetadataEqualityComparer().EqualsExceptForPhysicalDeviceName(prevBackup, b));
97+
!StripedBackupEqualityComparer.Instance.Equals(prevBackup, b));
9898
}
9999

100100
private static bool IsValidFilePath(BackupMetadata meta)

‎src/BackupMetadata.cs‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,29 @@ namespace AgDatabaseMove
22
{
33
using System;
44
using System.Collections.Generic;
5+
using System.Linq;
56
using SmoFacade;
67

7-
public class BackupMetadataEqualityComparer : IEqualityComparer<BackupMetadata>
8+
public class StripedBackupEqualityComparer : IEqualityComparer<BackupMetadata>
89
{
9-
/// <summary>
10-
/// This is used for checking similar backups (like striped backups)
11-
/// </summary>
12-
/// <returns>bool</returns>
13-
public bool EqualsExceptForPhysicalDeviceName(BackupMetadata x, BackupMetadata y)
10+
private static readonly Lazy<StripedBackupEqualityComparer> s_instance = new Lazy<StripedBackupEqualityComparer>(() => new StripedBackupEqualityComparer());
11+
private StripedBackupEqualityComparer() { }
12+
public static StripedBackupEqualityComparer Instance => s_instance.Value;
13+
14+
public bool Equals(BackupMetadata x, BackupMetadata y)
1415
{
1516
return x.LastLsn == y.LastLsn &&
1617
x.FirstLsn == y.FirstLsn &&
1718
x.BackupType == y.BackupType &&
1819
x.DatabaseName == y.DatabaseName &&
1920
x.CheckpointLsn == y.CheckpointLsn &&
20-
x.DatabaseBackupLsn == x.DatabaseBackupLsn;
21-
}
22-
23-
/// <summary>
24-
/// This is used for checking exactly the same backup (like finding duplicates)
25-
/// </summary>
26-
/// <returns>bool</returns>
27-
public bool Equals(BackupMetadata x, BackupMetadata y)
28-
{
29-
return EqualsExceptForPhysicalDeviceName(x, y)
30-
&& x.PhysicalDeviceName == y.PhysicalDeviceName;
21+
x.DatabaseBackupLsn == y.DatabaseBackupLsn;
3122
}
3223

3324
public int GetHashCode(BackupMetadata obj)
3425
{
3526
var hashCode = -1277603921;
3627
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.DatabaseName);
37-
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.PhysicalDeviceName);
3828
hashCode = hashCode * -1521134295 +
3929
EqualityComparer<BackupFileTools.BackupType>.Default.GetHashCode(obj.BackupType);
4030
hashCode = hashCode * -1521134295 + obj.FirstLsn.GetHashCode();
@@ -43,6 +33,32 @@ public int GetHashCode(BackupMetadata obj)
4333
hashCode = hashCode * -1521134295 + obj.DatabaseBackupLsn.GetHashCode();
4434
return hashCode;
4535
}
36+
37+
}
38+
39+
/// <summary>
40+
/// Two BackupMetadatas are the same, if they are like striped backups but also have the same `PhysicalDeviceName`
41+
/// </summary>
42+
public class BackupMetadataEqualityComparer : IEqualityComparer<BackupMetadata>
43+
{
44+
private static readonly StripedBackupEqualityComparer _stripedBackupEqualityComparer = StripedBackupEqualityComparer.Instance;
45+
46+
private static readonly Lazy<BackupMetadataEqualityComparer> s_instance = new Lazy<BackupMetadataEqualityComparer>(() => new BackupMetadataEqualityComparer());
47+
private BackupMetadataEqualityComparer() { }
48+
public static BackupMetadataEqualityComparer Instance => s_instance.Value;
49+
50+
public bool Equals(BackupMetadata x, BackupMetadata y)
51+
{
52+
return _stripedBackupEqualityComparer.Equals(x, y)
53+
&& x.PhysicalDeviceName == y.PhysicalDeviceName;
54+
}
55+
56+
public int GetHashCode(BackupMetadata obj)
57+
{
58+
var hashCode = _stripedBackupEqualityComparer.GetHashCode(obj);
59+
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.PhysicalDeviceName);
60+
return hashCode;
61+
}
4662
}
4763

4864
/// <summary>
@@ -72,4 +88,22 @@ public object Clone()
7288
return MemberwiseClone();
7389
}
7490
}
91+
92+
public class StripedBackupSet
93+
{
94+
public IEnumerable<BackupMetadata> StripedBackups { get; private set; }
95+
96+
private StripedBackupSet(IEnumerable<BackupMetadata> stripedBackups)
97+
{
98+
StripedBackups = stripedBackups;
99+
}
100+
101+
public static IEnumerable<StripedBackupSet> GetStripedBackupSetChain(IEnumerable<BackupMetadata> backups)
102+
{
103+
var chain = backups
104+
.GroupBy(b => b, StripedBackupEqualityComparer.Instance)
105+
.Select(group => new StripedBackupSet(group));
106+
return chain;
107+
}
108+
}
75109
}

‎src/SmoFacade/Server.cs‎

Lines changed: 52 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -178,11 +178,11 @@ public Database Database(string dbName)
178178
/// wait
179179
/// </param>
180180
/// <param name="fileRelocation">Option for renaming files during the restore.</param>
181-
public void Restore(IEnumerable<BackupMetadata> backupOrder, string databaseName,
181+
public void Restore(IEnumerable<StripedBackupSet> stripedBackupSetChain, string databaseName,
182182
Func<int, TimeSpan> retryDurationProvider,
183183
Func<string, string> fileRelocation = null)
184184
{
185-
var policy = Policy
185+
var retryPolicy = Policy
186186
.Handle<ExecutionFailureException>(e => e.InnerException != null
187187
&& e.InnerException is SqlException
188188
&& e.InnerException.Message
@@ -191,39 +191,65 @@ public void Restore(IEnumerable<BackupMetadata> backupOrder, string databaseName
191191

192192

193193
var restore = new Restore { Database = databaseName, NoRecovery = true };
194+
195+
var defaultFileLocations = DefaultFileLocations();
194196

195-
foreach(var backup in backupOrder) {
196-
var device = BackupFileTools.IsValidFileUrl(backup.PhysicalDeviceName) ? DeviceType.Url : DeviceType.File;
197-
var backupDeviceItem = new BackupDeviceItem(backup.PhysicalDeviceName, device);
198-
if(_credentialName != null && device == DeviceType.Url)
199-
backupDeviceItem.CredentialName = _credentialName;
197+
foreach(var stripedBackupSet in stripedBackupSetChain) {
200198

201-
restore.Devices.Add(backupDeviceItem);
199+
AddBackupDeviceItemsToRestore(restore, stripedBackupSet);
202200

203-
var defaultFileLocations = DefaultFileLocations();
204-
if(defaultFileLocations != null) {
205-
restore.RelocateFiles.Clear();
206-
var fileList = policy.Execute(() => restore.ReadFileList(_server).AsEnumerable());
207-
foreach(var file in fileList) {
208-
var physicalName = (string)file["PhysicalName"];
209-
var fileName = GetFileName(physicalName);
201+
if (defaultFileLocations != null)
202+
{
203+
AddRelocateFilesToRestore(restore, defaultFileLocations, retryPolicy, fileRelocation);
204+
}
210205

211-
if(fileRelocation != null)
212-
fileName = fileRelocation(fileName);
206+
_server.ConnectionContext.StatementTimeout = 86400; // 60 * 60 * 24 = 24 hours
207+
retryPolicy.Execute(() => restore.SqlRestore(_server));
213208

214-
var path = (string)file["Type"] == "L" ? defaultFileLocations?.Log : defaultFileLocations?.Data;
215-
path ??= Path.GetFullPath(physicalName);
209+
restore.Devices.Clear();
210+
restore.RelocateFiles.Clear();
211+
}
212+
}
216213

217-
var newFilePath = Path.Combine(path, fileName);
214+
/// <summary>
215+
/// This is like adding the `FROM URL='&lt;backup file url&gt;'` to the RESTORE T-SQL command
216+
/// </summary>
217+
private void AddBackupDeviceItemsToRestore(Restore restore, StripedBackupSet stripedBackupSet)
218+
{
219+
foreach (var stripedBackup in stripedBackupSet.StripedBackups)
220+
{
221+
var physicalDeviceName = stripedBackup.PhysicalDeviceName;
222+
var device = BackupFileTools.IsValidFileUrl(physicalDeviceName) ? DeviceType.Url : DeviceType.File;
218223

219-
restore.RelocateFiles.Add(new RelocateFile((string)file["LogicalName"], newFilePath));
220-
}
221-
}
224+
var backupDeviceItem = new BackupDeviceItem(physicalDeviceName, device);
225+
if (_credentialName != null && device == DeviceType.Url)
226+
backupDeviceItem.CredentialName = _credentialName;
222227

223-
_server.ConnectionContext.StatementTimeout = 86400; // 60 * 60 * 24 = 24 hours
228+
restore.Devices.Add(backupDeviceItem);
229+
}
230+
}
231+
232+
/// <summary>
233+
/// This is like adding the `WITH MOVE=&lt;label&gt; to &lt;path&gt;` to the RESTORE T-SQL command
234+
/// </summary>
235+
private void AddRelocateFilesToRestore(Restore restore, DefaultFileLocations defaultFileLocations,
236+
Policy retryPolicy,
237+
Func<string, string> fileRelocation = null)
238+
{
239+
var fileList = retryPolicy.Execute(() => restore.ReadFileList(_server).AsEnumerable());
240+
foreach(var file in fileList) {
241+
var physicalName = (string)file["PhysicalName"];
242+
var fileName = GetFileName(physicalName);
243+
244+
if(fileRelocation != null)
245+
fileName = fileRelocation(fileName);
246+
247+
var path = (string)file["Type"] == "L" ? defaultFileLocations?.Log : defaultFileLocations?.Data;
248+
path ??= Path.GetFullPath(physicalName);
249+
250+
var newFilePath = Path.Combine(path, fileName);
224251

225-
policy.Execute(() => restore.SqlRestore(_server));
226-
restore.Devices.Remove(backupDeviceItem);
252+
restore.RelocateFiles.Add(new RelocateFile((string)file["LogicalName"], newFilePath));
227253
}
228254
}
229255

‎tests/AgDatabaseMove.Unit/BackupOrder.cs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ private static IEnumerable<BackupMetadata> CloneBackupMetaDataList(List<BackupMe
3232
return result;
3333
}
3434

35-
private static List<BackupMetadata> GetBackupList()
35+
public static List<BackupMetadata> GetBackupList()
3636
{
3737
return new List<BackupMetadata> {
3838
new BackupMetadata {
@@ -107,15 +107,24 @@ private static List<BackupMetadata> GetBackupListWithoutDiff()
107107
return list;
108108
}
109109

110-
private static List<BackupMetadata> GetBackupListWithStripes()
110+
public static List<BackupMetadata> GetBackupListWithStripes()
111111
{
112112
var list = GetBackupList();
113113
var listWithStripes = CloneBackupMetaDataList(list).ToList();
114+
var listWithStripes2 = CloneBackupMetaDataList(list).ToList();
115+
114116
listWithStripes.ForEach(b => {
115117
var path = b.PhysicalDeviceName.Split('.');
116118
b.PhysicalDeviceName = $"{path[0]}_striped.{path[1]}";
117119
});
120+
121+
listWithStripes2.ForEach(b => {
122+
var path = b.PhysicalDeviceName.Split('.');
123+
b.PhysicalDeviceName = $"{path[0]}_striped2.{path[1]}";
124+
});
125+
118126
list.AddRange(listWithStripes);
127+
list.AddRange(listWithStripes2);
119128
return list;
120129
}
121130

‎tests/AgDatabaseMove.Unit/StripedBackupSetTests.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Moq;
5+
using Xunit;
6+
7+
namespace AgDatabaseMove.Unit
8+
{
9+
public class StripedBackupSetTests
10+
{
11+
12+
private static readonly StripedBackupEqualityComparer stripedBackupComparer = StripedBackupEqualityComparer.Instance;
13+
private static readonly BackupMetadataEqualityComparer backupComparer = BackupMetadataEqualityComparer.Instance;
14+
15+
[Fact]
16+
public void CombinesStripedBackups()
17+
{
18+
var backupChain = BackupOrder.GetBackupListWithStripes();
19+
20+
var stripedBackupSetChain = StripedBackupSet.GetStripedBackupSetChain(backupChain);
21+
22+
Assert.Equal(backupChain.Distinct(stripedBackupComparer).Count(), stripedBackupSetChain.Count());
23+
}
24+
25+
[Fact]
26+
public void DoesntCombineNonStripedBackups()
27+
{
28+
var backupChain = BackupOrder.GetBackupList();
29+
var stripedBackupSetChain = StripedBackupSet.GetStripedBackupSetChain(backupChain);
30+
31+
Assert.Equal(backupChain.Count, stripedBackupSetChain.Count());
32+
}
33+
34+
[Fact]
35+
public void StripedBackupsAreEqualExceptForPhysicalDeviceName()
36+
{
37+
var backupChain = BackupOrder.GetBackupListWithStripes();
38+
39+
40+
var stripedBackupSetChain = StripedBackupSet.GetStripedBackupSetChain(backupChain);
41+
42+
foreach (var stripedBackupSet in stripedBackupSetChain)
43+
{
44+
var stripes = stripedBackupSet.StripedBackups.ToList();
45+
var backup = stripes.First();
46+
for (int i = 1; i < stripes.Count; i++)
47+
{
48+
var otherBackup = stripes[i];
49+
50+
Assert.Equal(backup, otherBackup, stripedBackupComparer);
51+
Assert.NotEqual(backup, otherBackup, backupComparer);
52+
}
53+
}
54+
}
55+
}
56+
}

0 commit comments

Comments
 (0)