Skip to content

Commit b35e4f7

Browse files
rmunnhahn-kev
rmunn
and
hahn-kev
authored
JsonSyncable class (#50)
* WIP on JsonSyncable class * Add JsonSyncable file backend * improve GetChanges performance by calculating local sync state while reading commits * Use .jsonl extension for JsonSyncable client commit files. --------- Co-authored-by: Kevin Hahn <kevin_hahn@sil.org>
1 parent 94ffcfa commit b35e4f7

9 files changed

Lines changed: 665 additions & 16 deletions

File tree

‎src/SIL.Harmony.Core/QueryHelpers.cs‎

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,29 +31,24 @@ public static async IAsyncEnumerable<TCommit> GetMissingCommits<TCommit, TChange
3131
if (includeChangeEntities) commits = commits.Include(c => c.ChangeEntities);
3232
foreach (var (clientId, localTimestamp) in localState.ClientHeads)
3333
{
34-
//client is new to the other history
35-
if (!remoteState.ClientHeads.TryGetValue(clientId, out var otherTimestamp))
34+
long? remoteTimestamp = remoteState.ClientHeads.TryGetValue(clientId, out var otherTimestamp)
35+
? otherTimestamp
36+
: null;
37+
var clientCommits = commits.Where(c => c.ClientId == clientId);
38+
if (remoteTimestamp is null)
3639
{
37-
//todo slow, it would be better if we could query on client id and get latest changes per client
38-
await foreach (var commit in commits
39-
.DefaultOrder()
40-
.Where(c => c.ClientId == clientId)
41-
.AsAsyncEnumerable())
42-
{
40+
await foreach (var commit in clientCommits.DefaultOrder().AsAsyncEnumerable())
4341
yield return commit;
44-
}
4542
}
46-
//client has newer history than the other history
47-
else if (localTimestamp > otherTimestamp)
43+
else if (localTimestamp > remoteTimestamp)
4844
{
49-
var otherDt = DateTimeOffset.FromUnixTimeMilliseconds(otherTimestamp);
50-
//todo even slower because we need to filter out changes that are already in the other history
51-
await foreach (var commit in commits
45+
var otherDt = DateTimeOffset.FromUnixTimeMilliseconds(remoteTimestamp.Value);
46+
await foreach (var commit in clientCommits
47+
.Where(c => c.HybridDateTime.DateTime > otherDt)
5248
.DefaultOrder()
53-
.Where(c => c.ClientId == clientId && c.HybridDateTime.DateTime > otherDt)
5449
.AsAsyncEnumerable())
5550
{
56-
if (commit.DateTime.ToUnixTimeMilliseconds() > otherTimestamp)
51+
if (commit.DateTime.ToUnixTimeMilliseconds() > remoteTimestamp)
5752
yield return commit;
5853
}
5954
}
@@ -75,6 +70,47 @@ public static async Task<SortedSet<T>> ToSortedSetAsync<T>(this IQueryable<T> qu
7570
return set;
7671
}
7772

73+
public static IEnumerable<TCommit> GetMissingCommits<TCommit, TChange>(
74+
this IEnumerable<TCommit> commits,
75+
SyncState localState,
76+
SyncState remoteState) where TCommit : CommitBase<TChange>
77+
{
78+
foreach (var (clientId, localTimestamp) in localState.ClientHeads)
79+
{
80+
long? remoteTimestamp = remoteState.ClientHeads.TryGetValue(clientId, out var otherTimestamp)
81+
? otherTimestamp
82+
: null;
83+
foreach (var commit in GetMissingCommitsForClient(
84+
commits.Where(c => c.ClientId == clientId), localTimestamp, remoteTimestamp))
85+
{
86+
yield return commit;
87+
}
88+
}
89+
}
90+
91+
private static IEnumerable<TCommit> GetMissingCommitsForClient<TCommit>(
92+
IEnumerable<TCommit> clientCommits,
93+
long localTimestamp,
94+
long? remoteTimestamp) where TCommit : CommitBase
95+
{
96+
if (remoteTimestamp is null)
97+
{
98+
foreach (var commit in clientCommits.DefaultOrder())
99+
yield return commit;
100+
}
101+
else if (localTimestamp > remoteTimestamp)
102+
{
103+
var otherDt = DateTimeOffset.FromUnixTimeMilliseconds(remoteTimestamp.Value);
104+
foreach (var commit in clientCommits
105+
.Where(c => c.HybridDateTime.DateTime > otherDt)
106+
.DefaultOrder())
107+
{
108+
if (commit.DateTime.ToUnixTimeMilliseconds() > remoteTimestamp)
109+
yield return commit;
110+
}
111+
}
112+
}
113+
78114
public static IQueryable<T> DefaultOrder<T>(this IQueryable<T> queryable) where T: CommitBase
79115
{
80116
return queryable

‎src/SIL.Harmony.Tests/JsonSyncableTests.cs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
using System.Text.Json;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using Microsoft.Extensions.Logging.Abstractions;
4+
using SIL.Harmony.Changes;
5+
using SIL.Harmony.Sample;
6+
using SIL.Harmony.Sample.Changes;
7+
8+
namespace SIL.Harmony.Tests;
9+
10+
public class JsonSyncableTests
11+
{
12+
private static JsonSerializerOptions SerializerOptions { get; } = new ServiceCollection()
13+
.AddCrdtDataSample(":memory:")
14+
.BuildServiceProvider()
15+
.GetRequiredService<JsonSerializerOptions>();
16+
17+
[Fact]
18+
public async Task WritesPerClientFile()
19+
{
20+
var dir = Directory.CreateTempSubdirectory("harmony-json-syncable-test-");
21+
try
22+
{
23+
var syncable = new JsonSyncable(dir, SerializerOptions, NullLogger<JsonSyncable>.Instance);
24+
var clientId = Guid.NewGuid();
25+
var commit = new Commit
26+
{
27+
ClientId = clientId,
28+
HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0),
29+
ChangeEntities =
30+
{
31+
new ChangeEntity<IChange>
32+
{
33+
Change = new SetWordTextChange(Guid.NewGuid(), "hello"),
34+
Index = 0,
35+
CommitId = Guid.Empty,
36+
EntityId = Guid.Empty
37+
}
38+
}
39+
};
40+
41+
await syncable.AddRangeFromSync([commit]);
42+
43+
var file = Path.Combine(dir.FullName, $"{JsonSyncable.FilenamePrefix}{clientId}{JsonSyncable.FilenameExtension}");
44+
File.Exists(file).Should().BeTrue();
45+
}
46+
finally
47+
{
48+
dir.Delete(recursive: true);
49+
}
50+
}
51+
52+
[Fact]
53+
public async Task DoesNotDuplicateFileLines()
54+
{
55+
var dir = Directory.CreateTempSubdirectory("harmony-json-syncable-test-");
56+
try
57+
{
58+
var syncable = new JsonSyncable(dir, SerializerOptions, NullLogger<JsonSyncable>.Instance);
59+
var commit = new Commit
60+
{
61+
ClientId = Guid.NewGuid(),
62+
HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0)
63+
};
64+
65+
await syncable.AddRangeFromSync([commit]);
66+
await syncable.AddRangeFromSync([commit]);
67+
68+
var file = new FileInfo(Path.Combine(dir.FullName,
69+
$"{JsonSyncable.FilenamePrefix}{commit.ClientId}{JsonSyncable.FilenameExtension}"));
70+
var lines = await File.ReadAllLinesAsync(file.FullName);
71+
lines.Should().HaveCount(1);
72+
}
73+
finally
74+
{
75+
dir.Delete(recursive: true);
76+
}
77+
}
78+
}

‎src/SIL.Harmony.Tests/Syncable/CrossSyncableTests.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
using SIL.Harmony.Changes;
2+
using SIL.Harmony.Sample.Changes;
3+
using SIL.Harmony.Sample.Models;
4+
5+
namespace SIL.Harmony.Tests.Syncable;
6+
7+
public class CrossSyncableTests
8+
{
9+
[Fact]
10+
public async Task DataModelAndJsonSyncable_CanSyncBidirectionally()
11+
{
12+
var dataBackend = new DataModelSyncBackend();
13+
var jsonBackend = new JsonSyncableSyncBackend();
14+
await using var dataModel = await dataBackend.CreateAsync();
15+
await using var jsonSyncable = await jsonBackend.CreateAsync();
16+
17+
var entity1Id = Guid.NewGuid();
18+
var entity2Id = Guid.NewGuid();
19+
var dataModelCommit = CreateCommit(dataModel.ClientId, new SetWordTextChange(entity1Id, "from-datamodel"));
20+
await dataModel.Syncable.AddRangeFromSync([dataModelCommit]);
21+
await dataModel.MirrorToReadModelAsync([dataModelCommit]);
22+
var jsonSyncableCommit = CreateCommit(jsonSyncable.ClientId, new SetWordTextChange(entity2Id, "from-json"));
23+
await jsonSyncable.Syncable.AddRangeFromSync([jsonSyncableCommit]);
24+
await jsonSyncable.MirrorToReadModelAsync([jsonSyncableCommit]);
25+
26+
var syncResults = await dataModel.Syncable.SyncWith(jsonSyncable.Syncable);
27+
await SyncableTestHelpers.MirrorToReadModelsAsync(dataModel, jsonSyncable, syncResults);
28+
29+
(await dataModel.ReadModel.GetLatest<Word>(entity1Id))!.Text.Should().Be("from-datamodel");
30+
(await dataModel.ReadModel.GetLatest<Word>(entity2Id))!.Text.Should().Be("from-json");
31+
(await jsonSyncable.ReadModel.GetLatest<Word>(entity1Id))!.Text.Should().Be("from-datamodel");
32+
(await jsonSyncable.ReadModel.GetLatest<Word>(entity2Id))!.Text.Should().Be("from-json");
33+
34+
var dmState = await dataModel.Syncable.GetSyncState();
35+
var jsonState = await jsonSyncable.Syncable.GetSyncState();
36+
dmState.ClientHeads.Should().BeEquivalentTo(jsonState.ClientHeads);
37+
}
38+
39+
private static Commit CreateCommit(Guid clientId, IChange change)
40+
{
41+
var commitId = Guid.NewGuid();
42+
return new Commit(commitId)
43+
{
44+
ClientId = clientId,
45+
HybridDateTime = new HybridDateTime(DateTimeOffset.Now, 0),
46+
ChangeEntities = [new ChangeEntity<IChange>
47+
{
48+
Index = 0,
49+
CommitId = commitId,
50+
EntityId = change.EntityId,
51+
Change = change
52+
}
53+
]
54+
};
55+
}
56+
}

‎src/SIL.Harmony.Tests/Syncable/DataModelSyncBackend.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
namespace SIL.Harmony.Tests.Syncable;
2+
3+
public class DataModelSyncBackend : ISyncableTestBackend
4+
{
5+
public string Name => "DataModel";
6+
public override string ToString() => Name;
7+
8+
public async Task<SyncableTestContext> CreateAsync()
9+
{
10+
var testBase = new DataModelTestBase();
11+
await testBase.InitializeAsync();
12+
return new SyncableTestContext(
13+
testBase.DataModel,
14+
testBase.DataModel,
15+
Guid.NewGuid(),
16+
syncableUpdatesReadModel: true,
17+
() => testBase.DisposeAsync());
18+
}
19+
}

‎src/SIL.Harmony.Tests/Syncable/ISyncableTestBackend.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
namespace SIL.Harmony.Tests.Syncable;
2+
3+
public sealed class SyncableTestContext(
4+
ISyncable syncable,
5+
DataModel readModel,
6+
Guid clientId,
7+
bool syncableUpdatesReadModel,
8+
Func<ValueTask> disposeAsync) : IAsyncDisposable
9+
{
10+
public ISyncable Syncable { get; } = syncable;
11+
public DataModel ReadModel { get; } = readModel;
12+
public Guid ClientId { get; } = clientId;
13+
internal bool SyncableUpdatesReadModel { get; } = syncableUpdatesReadModel;
14+
15+
public ValueTask DisposeAsync() => disposeAsync();
16+
}
17+
18+
/// <summary>Factory for a backend under test (DataModel, JsonSyncable, ...).</summary>
19+
public interface ISyncableTestBackend
20+
{
21+
string Name { get; }
22+
Task<SyncableTestContext> CreateAsync();
23+
}

‎src/SIL.Harmony.Tests/Syncable/JsonSyncableSyncBackend.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
using System.Text.Json;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using Microsoft.Extensions.Logging.Abstractions;
4+
using SIL.Harmony.Sample;
5+
6+
namespace SIL.Harmony.Tests.Syncable;
7+
8+
public class JsonSyncableSyncBackend : ISyncableTestBackend
9+
{
10+
public string Name => "JsonSyncable";
11+
public override string ToString() => Name;
12+
13+
private JsonSerializerOptions SerializerOptions { get; } = new ServiceCollection()
14+
.AddCrdtDataSample(":memory:")
15+
.BuildServiceProvider()
16+
.GetRequiredService<JsonSerializerOptions>();
17+
18+
public async Task<SyncableTestContext> CreateAsync()
19+
{
20+
var testBase = new DataModelTestBase();
21+
await testBase.InitializeAsync();
22+
var tempDir = Directory.CreateTempSubdirectory("harmony-json-syncable-");
23+
var jsonSyncable = new JsonSyncable(tempDir, SerializerOptions, NullLogger<JsonSyncable>.Instance);
24+
return new SyncableTestContext(
25+
jsonSyncable,
26+
testBase.DataModel,
27+
Guid.NewGuid(),
28+
syncableUpdatesReadModel: false,
29+
async () =>
30+
{
31+
await testBase.DisposeAsync();
32+
tempDir.Delete(recursive: true);
33+
});
34+
}
35+
}

‎src/SIL.Harmony.Tests/Syncable/SyncableTestHelpers.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
namespace SIL.Harmony.Tests.Syncable;
2+
3+
public static class SyncableTestHelpers
4+
{
5+
public static ISyncableTestBackend[] Backends =>
6+
[
7+
new DataModelSyncBackend(),
8+
new JsonSyncableSyncBackend()
9+
];
10+
11+
public static IEnumerable<object[]> BackendData => Backends.Select(backend => new object[] { backend });
12+
13+
public static IEnumerable<object[]> BackendPairData =>
14+
Backends.SelectMany(left => Backends, (left, right) => new object[] { left, right });
15+
16+
public static async Task MirrorToReadModelAsync(this SyncableTestContext context, IEnumerable<Commit> commits)
17+
{
18+
if (context.SyncableUpdatesReadModel) return;
19+
20+
var array = commits.ToArray();
21+
if (array.Length > 0)
22+
await ((ISyncable)context.ReadModel).AddRangeFromSync(array);
23+
}
24+
25+
public static async Task MirrorToReadModelsAsync(
26+
SyncableTestContext local,
27+
SyncableTestContext remote,
28+
SyncResults results)
29+
{
30+
await local.MirrorToReadModelAsync(results.MissingFromLocal);
31+
await remote.MirrorToReadModelAsync(results.MissingFromRemote);
32+
}
33+
}

0 commit comments

Comments
 (0)