From 38a4e3539ac3c8fb9c532960f740b94bc5fada96 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 17 Jul 2026 11:13:12 +0200 Subject: [PATCH 1/2] feat(persistence): add PagedResult for paged store reads Carries the page plus the pre-paging total, which is what a caller needs to know how many pages exist. The total is free to compute: the filter has already run over the bucket. --- .../Data/PagedResult.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/SquidStd.Persistence.Abstractions/Data/PagedResult.cs diff --git a/src/SquidStd.Persistence.Abstractions/Data/PagedResult.cs b/src/SquidStd.Persistence.Abstractions/Data/PagedResult.cs new file mode 100644 index 00000000..ba24a988 --- /dev/null +++ b/src/SquidStd.Persistence.Abstractions/Data/PagedResult.cs @@ -0,0 +1,12 @@ +namespace SquidStd.Persistence.Abstractions.Data; + +/// One page of entities, and the total the filter matched before paging. +/// The entities on this page, as detached clones. +/// +/// How many entities the filter matched, before and — +/// which is what a caller needs to know how many pages exist. Counting it costs nothing: the filter has +/// already run. +/// +/// The offset this page was taken from. +/// The page size asked for. The page may hold fewer at the end of the results. +public sealed record PagedResult(IReadOnlyList Items, int Total, int Skip, int Take); From fea50c4bbdec7151b513dfc069f0e4f7742d2553 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 17 Jul 2026 11:15:43 +0200 Subject: [PATCH 2/2] feat(persistence): add QueryPaged, cloning only the returned page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetAll and Query deep-clone the whole store on every call, which makes paging over them cost the entire bucket per page. QueryPaged filters and orders the live entities under the same lock and clones only what survives skip/take: O(bucket) cheap comparisons plus O(page) clones. The sort key is required rather than defaulted. Paging an unordered bucket is meaningless — enumeration order is not contractual and shifts as entries come and go, so a page could repeat or drop an entity with nothing in the result admitting it. Ties break on the entity key, since the caller's key is rarely unique and equal keys would otherwise fall back to that same unstable order. The filter runs under the state lock against live, uncloned entities, which is where the saving comes from and also the constraint: it must be a pure, cheap read. Documented on the interface. A test asserts exactly one clone per returned entity. Nothing else would notice this method quietly degrading into GetAll. --- .../Interfaces/Persistence/IEntityStore.cs | 30 +++ .../Services/EntityStore.cs | 34 +++ .../Persistence/EntityStorePagedTests.cs | 202 ++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 tests/SquidStd.Tests/Persistence/EntityStorePagedTests.cs diff --git a/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/IEntityStore.cs b/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/IEntityStore.cs index 337ee30a..a327824e 100644 --- a/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/IEntityStore.cs +++ b/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/IEntityStore.cs @@ -1,3 +1,5 @@ +using SquidStd.Persistence.Abstractions.Data; + namespace SquidStd.Persistence.Abstractions.Interfaces.Persistence; /// @@ -28,6 +30,34 @@ public interface IEntityStore ValueTask GetByIdAsync(TKey id, CancellationToken cancellationToken = default); IQueryable Query(); + + /// + /// Completes synchronously from memory. Filters, orders and pages under the state lock, cloning only + /// the entities on the returned page — unlike and , whose + /// cost is a deep clone of the whole store on every call. + /// + /// + /// Kept entities, or null for all of them. Runs under the state lock against live, uncloned entities, + /// so it must be a pure, cheap read: no I/O, no locks, no calls back into the store, and it must not + /// mutate its argument — a mutation here would change stored state with no journal entry behind it. + /// + /// + /// The sort key. Required, not optional: paging an unordered bucket is meaningless, because + /// enumeration order is not contractual and shifts as entries come and go, so a page could repeat or + /// drop an entity with nothing in the result admitting it. Ties break on the entity's own key, which + /// requires to be comparable — every realistic key is, and a key that is + /// not throws on the first comparison rather than mis-ordering quietly. + /// + /// Entities to skip. Past the end yields an empty page and the true total. + /// Page size. The last page may hold fewer. + /// Reverses . The tiebreak reverses with it. + PagedResult QueryPaged( + Func? filter, + Func orderBy, + int skip, + int take, + bool descending = false + ); ValueTask RemoveAsync(TKey id, CancellationToken cancellationToken = default); ValueTask UpsertAsync(TEntity entity, CancellationToken cancellationToken = default); } diff --git a/src/SquidStd.Persistence/Services/EntityStore.cs b/src/SquidStd.Persistence/Services/EntityStore.cs index 46e49976..ca718a2c 100644 --- a/src/SquidStd.Persistence/Services/EntityStore.cs +++ b/src/SquidStd.Persistence/Services/EntityStore.cs @@ -71,6 +71,40 @@ public IQueryable Query() } } + public PagedResult QueryPaged( + Func? filter, + Func orderBy, + int skip, + int take, + bool descending = false + ) + { + lock (_stateStore.SyncRoot) + { + // Filtering and ordering run against the live entities, not clones: that is the whole saving. + // Only what survives skip/take is cloned, so the cost is O(bucket) comparisons plus O(page) + // clones instead of O(bucket) clones. + IEnumerable matching = Bucket().Values; + + if (filter is not null) + { + matching = matching.Where(filter); + } + + var matched = matching as IList ?? [.. matching]; + + // The entity key breaks ties. The caller's key is rarely unique, and leaving equal keys to + // Dictionary order is exactly the instability orderBy exists to remove. + var ordered = descending + ? matched.OrderByDescending(orderBy).ThenByDescending(_descriptor.GetKey) + : matched.OrderBy(orderBy).ThenBy(_descriptor.GetKey); + + IReadOnlyList page = [.. ordered.Skip(skip).Take(take).Select(_descriptor.Clone)]; + + return new(page, matched.Count, skip, take); + } + } + public async ValueTask UpsertAsync(TEntity entity, CancellationToken cancellationToken = default) { await _stateStore.WriteLock.WaitAsync(cancellationToken); diff --git a/tests/SquidStd.Tests/Persistence/EntityStorePagedTests.cs b/tests/SquidStd.Tests/Persistence/EntityStorePagedTests.cs new file mode 100644 index 00000000..02a3a443 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/EntityStorePagedTests.cs @@ -0,0 +1,202 @@ +using SquidStd.Core.Json; +using SquidStd.Persistence.Abstractions.Interfaces.Persistence; +using SquidStd.Persistence.Data; +using SquidStd.Persistence.Internal; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class EntityStorePagedTests : IAsyncDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "squidstd-paged-" + Guid.NewGuid().ToString("N")); + private readonly CountingDescriptor _descriptor; + private readonly BinaryJournalService _journal; + private readonly PersistenceStateStore _stateStore = new(); + private readonly EntityStore _store; + + public EntityStorePagedTests() + { + var serializer = new JsonDataSerializer(); + _descriptor = new( + new PersistenceEntityDescriptor(serializer, serializer, 1, "Player", 1, p => p.Id) + ); + _journal = new(Path.Combine(_dir, "world.journal.bin")); + _store = new(_stateStore, _journal, _descriptor); + } + + private async Task SeedAsync(params (int Id, string Name)[] players) + { + foreach (var (id, name) in players) + { + await _store.UpsertAsync(new() { Id = id, Name = name }); + } + } + + [Fact] + public async Task QueryPaged_OrdersAndPages() + { + await SeedAsync((1, "Carol"), (2, "Alice"), (3, "Bob")); + + var page = _store.QueryPaged(null, p => p.Name, 0, 2); + + Assert.Equal(["Alice", "Bob"], page.Items.Select(p => p.Name)); + Assert.Equal(3, page.Total); + Assert.Equal(0, page.Skip); + Assert.Equal(2, page.Take); + } + + [Fact] + public async Task QueryPaged_Descending_ReversesTheOrder() + { + await SeedAsync((1, "Carol"), (2, "Alice"), (3, "Bob")); + + var page = _store.QueryPaged(null, p => p.Name, 0, 2, true); + + Assert.Equal(["Carol", "Bob"], page.Items.Select(p => p.Name)); + } + + [Fact] + public async Task QueryPaged_Total_CountsTheFilterNotThePage() + { + await SeedAsync((1, "Alice"), (2, "Amy"), (3, "Bob")); + + var page = _store.QueryPaged(p => p.Name.StartsWith('A'), p => p.Name, 0, 1); + + Assert.Single(page.Items); + Assert.Equal(2, page.Total); // Alice and Amy matched; only one is on the page + } + + [Fact] + public async Task QueryPaged_SkipPastTheEnd_IsEmptyButKeepsTheTotal() + { + await SeedAsync((1, "Alice"), (2, "Bob")); + + var page = _store.QueryPaged(null, p => p.Name, 99, 10); + + Assert.Empty(page.Items); + Assert.Equal(2, page.Total); + } + + [Fact] + public async Task QueryPaged_ReturnsDetachedClones() + { + await SeedAsync((1, "Alice")); + + var page = _store.QueryPaged(null, p => p.Name, 0, 10); + page.Items[0].Tags.Add("mutated"); + + var again = _store.QueryPaged(null, p => p.Name, 0, 10); + + Assert.Empty(again.Items[0].Tags); + } + + [Fact] + public async Task QueryPaged_ClonesOnlyThePage() + { + // The entire point of this method. GetAll and Query deep-clone the whole store on every call; if + // this did the same there would be no reason for it to exist, and no other assertion would notice. + await SeedAsync((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "E")); + _descriptor.CloneCount = 0; + + var page = _store.QueryPaged(null, p => p.Name, 0, 2); + + Assert.Equal(2, page.Items.Count); + Assert.Equal(2, _descriptor.CloneCount); + } + + [Fact] + public async Task QueryPaged_TiedOrderKeys_AreBrokenByEntityKey() + { + // Two entities sorting equal must not be left to Dictionary order, or page 2 could repeat page 1's + // row and drop another, with nothing in the response admitting it. + await SeedAsync((3, "Same"), (1, "Same"), (2, "Same")); + + var first = _store.QueryPaged(null, p => p.Name, 0, 3); + var second = _store.QueryPaged(null, p => p.Name, 0, 3); + + Assert.Equal([1, 2, 3], first.Items.Select(p => p.Id)); + Assert.Equal(first.Items.Select(p => p.Id), second.Items.Select(p => p.Id)); + } + + [Fact] + public async Task QueryPaged_PagesDoNotOverlapOrSkipWhenKeysTie() + { + await SeedAsync((1, "Same"), (2, "Same"), (3, "Same"), (4, "Same")); + + var first = _store.QueryPaged(null, p => p.Name, 0, 2); + var next = _store.QueryPaged(null, p => p.Name, 2, 2); + + Assert.Equal([1, 2], first.Items.Select(p => p.Id)); + Assert.Equal([3, 4], next.Items.Select(p => p.Id)); + } + + public async ValueTask DisposeAsync() + { + await _journal.DisposeAsync(); + + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, true); + } + } + + /// Wraps a real descriptor and counts Clone calls, so a test can prove what was cloned. + private sealed class CountingDescriptor : IPersistenceEntityDescriptor + { + private readonly IPersistenceEntityDescriptor _inner; + + public CountingDescriptor(IPersistenceEntityDescriptor inner) + { + _inner = inner; + } + + public int CloneCount { get; set; } + + public ushort TypeId => _inner.TypeId; + + public string TypeName => _inner.TypeName; + + public int SchemaVersion => _inner.SchemaVersion; + + public Type EntityType => _inner.EntityType; + + public Type KeyType => _inner.KeyType; + + public int GetKey(Player entity) + => _inner.GetKey(entity); + + public Player Clone(Player entity) + { + CloneCount++; + + return _inner.Clone(entity); + } + + public byte[] SerializeEntity(Player entity) + => _inner.SerializeEntity(entity); + + public Player DeserializeEntity(byte[] payload) + => _inner.DeserializeEntity(payload); + + public byte[] SerializeBucket(IReadOnlyCollection entities) + => _inner.SerializeBucket(entities); + + public IReadOnlyList DeserializeBucket(byte[] payload) + => _inner.DeserializeBucket(payload); + + public byte[] SerializeKey(int key) + => _inner.SerializeKey(key); + + public int DeserializeKey(byte[] payload) + => _inner.DeserializeKey(payload); + } + + private sealed class Player + { + public int Id { get; set; } + + public string Name { get; set; } = string.Empty; + + public List Tags { get; set; } = []; + } +}