Skip to content

Commit 581a58b

Browse files
badrishcCopilot
badrishc
and
Copilot
committed
Merge dev into bf-tree-integration-plan
Integrates 4 new dev commits: - #1716: Fix use-after-free in GetAllocationForRetry when page is evicted - #1712: Internalize heap-size tracking into Tsavorite (IRecordTriggers refactor) - #1711: Fix three broken/flaky Tsavorite and Garnet tests - #1710: Fix native linux device Key changes from #1712 affecting bf-tree: - IRecordDisposer replaced by IRecordTriggers with OnEvict/OnFlush/OnDispose/OnDiskRead - GarnetRecordDisposer replaced by GarnetRecordTriggers - OnEvict now takes EvictionSource parameter (MainLog vs ReadCache) - Heap-size tracking internalized into Tsavorite via logSizeTracker - OnDisposeValueObject removed; disposal routed through OnDispose Resolution: took dev's Tsavorite src/test wholesale, then restored bf-tree additions: CheckpointTrigger enum, OnRecovery/OnCheckpoint/OnRecoverySnapshotRead hooks in IRecordTriggers/IStoreFunctions/StoreFunctions, WrongType in UpsertAction, HybridLogCheckpointSMTask and Recovery call sites. Updated GarnetRecordTriggers OnEvict to accept EvictionSource parameter. Re-added BfTreeInterop project ref and WRONGTYPE guard in UpsertMethods. All tests pass: 55 RangeIndex (3 runs), 345 RespTests. Format clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 29553e3 commit 581a58b

29 files changed

Lines changed: 310 additions & 190 deletions

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,10 @@ jobs:
172172
steps:
173173
- name: Check out code
174174
uses: actions/checkout@v4
175+
- name: Set workaround for libaio on Ubuntu 24.04 (see https://askubuntu.com/questions/1512196/libaio1-on-noble/1512197#1512197)
176+
run: |
177+
sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
178+
if: ${{ matrix.os == 'ubuntu-latest' }}
175179
- name: Set environment variable for Linux
176180
run: echo "RunAzureTests=yes" >> $GITHUB_ENV
177181
if: ${{ matrix.os == 'ubuntu-latest' }}

‎.github/workflows/nightly.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ jobs:
4343
- name: Check out code
4444
uses: actions/checkout@v4
4545

46+
- name: Set workaround for libaio on Ubuntu 24.04 (see https://askubuntu.com/questions/1512196/libaio1-on-noble/1512197#1512197)
47+
run: |
48+
sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
49+
if: ${{ matrix.os == 'ubuntu-latest' }}
50+
4651
- name: Set environment variable for Linux
4752
run: echo "RunAzureTests=yes" >> $GITHUB_ENV
4853
if: ${{ matrix.os == 'ubuntu-latest' }}

‎libs/cluster/Session/RespClusterMigrateCommands.cs‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,7 @@ void Process(BasicGarnetApi basicGarnetApi, byte[] input, bool replaceOption, bo
185185
if (replaceOption || !Exists(keySlice))
186186
_ = basicGarnetApi.SET(in diskLogRecord);
187187

188-
storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk);
189188
diskLogRecord.Dispose();
190-
diskLogRecord = default; // prevent double-trigger in finally
191189
}
192190
else
193191
{
@@ -200,11 +198,7 @@ void Process(BasicGarnetApi basicGarnetApi, byte[] input, bool replaceOption, bo
200198
}
201199
finally
202200
{
203-
if (diskLogRecord.IsSet)
204-
{
205-
storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk);
206-
diskLogRecord.Dispose();
207-
}
201+
diskLogRecord.Dispose();
208202
}
209203
}
210204
}

‎libs/cluster/Session/RespClusterReplicationCommands.cs‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -536,9 +536,7 @@ private bool NetworkClusterSync(out bool invalidParameters)
536536

537537
diskLogRecord = DiskLogRecord.Deserialize(recordSpan, storeWrapper.GarnetObjectSerializer, transientObjectIdMap, storeWrapper.storeFunctions);
538538
_ = basicGarnetApi.SET(in diskLogRecord);
539-
storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk);
540539
diskLogRecord.Dispose();
541-
diskLogRecord = default; // prevent double-trigger in catch
542540
}
543541
else
544542
{
@@ -551,11 +549,7 @@ private bool NetworkClusterSync(out bool invalidParameters)
551549
catch
552550
{
553551
// Dispose the diskLogRecord if there was an exception in SET
554-
if (diskLogRecord.IsSet)
555-
{
556-
storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk);
557-
diskLogRecord.Dispose();
558-
}
552+
diskLogRecord.Dispose();
559553
throw;
560554
}
561555

‎libs/host/GarnetServer.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ private TsavoriteKV<StoreFunctions, StoreAllocator> CreateStore(int dbId, IClust
379379
var store = new TsavoriteKV<StoreFunctions, StoreAllocator>(kvSettings
380380
, Tsavorite.core.StoreFunctions.Create(new GarnetKeyComparer(),
381381
() => new GarnetObjectSerializer(customCommandManager),
382-
new GarnetRecordTriggers())
382+
new GarnetRecordTriggers(cacheSizeTracker, rangeIndexManager))
383383
, (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions));
384384

385385
if (kvSettings.LogMemorySize > 0 || kvSettings.ReadCacheMemorySize > 0)

‎libs/server/Custom/CustomObjectBase.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,10 @@ public sealed override void DoSerialize(BinaryWriter writer)
7575

7676
/// <inheritdoc />
7777
public sealed override bool Operate(ref ObjectInput input, ref ObjectOutput output,
78-
byte respProtocolVersion)
78+
byte respProtocolVersion, out long sizeChange)
7979
{
80+
sizeChange = 0;
81+
8082
switch (input.header.cmd)
8183
{
8284
// Scan Command

‎libs/server/Objects/Hash/HashObject.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,10 @@ public override void Dispose() { }
182182
public override GarnetObjectBase Clone() => new HashObject(hash, expirationTimes, expirationQueue, HeapMemorySize);
183183

184184
/// <inheritdoc />
185-
public override bool Operate(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion)
185+
public override bool Operate(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion, out long memorySizeChange)
186186
{
187+
memorySizeChange = 0;
188+
187189
if (input.header.type != GarnetObjectType.Hash)
188190
{
189191
//Indicates when there is an incorrect type
@@ -192,6 +194,7 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output, byt
192194
return true;
193195
}
194196

197+
var previousMemorySize = HeapMemorySize;
195198
switch (input.header.HashOp)
196199
{
197200
case HashOperation.HSET:
@@ -258,6 +261,8 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output, byt
258261
throw new GarnetException($"Unsupported operation {input.header.HashOp} in HashObject.Operate");
259262
}
260263

264+
memorySizeChange = HeapMemorySize - previousMemorySize;
265+
261266
if (hash.Count == 0)
262267
output.OutputFlags |= ObjectOutputFlags.RemoveKey;
263268

‎libs/server/Objects/List/ListObject.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,10 @@ public override void Dispose() { }
128128

129129
/// <inheritdoc />
130130
public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
131-
byte respProtocolVersion)
131+
byte respProtocolVersion, out long memorySizeChange)
132132
{
133+
memorySizeChange = 0;
134+
133135
if (input.header.type != GarnetObjectType.List)
134136
{
135137
// Indicates an incorrect type of key
@@ -138,6 +140,7 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
138140
return true;
139141
}
140142

143+
var previousMemorySize = HeapMemorySize;
141144
switch (input.header.ListOp)
142145
{
143146
case ListOperation.LPUSH:
@@ -183,6 +186,8 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
183186
throw new GarnetException($"Unsupported operation {input.header.ListOp} in ListObject.Operate");
184187
}
185188

189+
memorySizeChange = HeapMemorySize - previousMemorySize;
190+
186191
if (list.Count == 0)
187192
output.OutputFlags |= ObjectOutputFlags.RemoveKey;
188193

‎libs/server/Objects/Set/SetObject.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,10 @@ public override void Dispose() { }
122122

123123
/// <inheritdoc />
124124
public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
125-
byte respProtocolVersion)
125+
byte respProtocolVersion, out long memorySizeChange)
126126
{
127+
memorySizeChange = 0;
128+
127129
if (input.header.type != GarnetObjectType.Set)
128130
{
129131
// Indicates an incorrect type of key
@@ -132,6 +134,7 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
132134
return true;
133135
}
134136

137+
var prevMemorySize = HeapMemorySize;
135138
switch (input.header.SetOp)
136139
{
137140
case SetOperation.SADD:
@@ -165,6 +168,8 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
165168
throw new GarnetException($"Unsupported operation {input.header.SetOp} in SetObject.Operate");
166169
}
167170

171+
memorySizeChange = HeapMemorySize - prevMemorySize;
172+
168173
if (Set.Count == 0)
169174
output.OutputFlags |= ObjectOutputFlags.RemoveKey;
170175

‎libs/server/Objects/SortedSet/SortedSetObject.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,8 +316,10 @@ public override void Dispose() { }
316316

317317
/// <inheritdoc />
318318
public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
319-
byte respProtocolVersion)
319+
byte respProtocolVersion, out long memorySizeChange)
320320
{
321+
memorySizeChange = 0;
322+
321323
var header = input.header;
322324
if (header.type != GarnetObjectType.SortedSet)
323325
{
@@ -327,6 +329,7 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
327329
return true;
328330
}
329331

332+
var prevMemorySize = HeapMemorySize;
330333
var op = header.SortedSetOp;
331334
switch (op)
332335
{
@@ -412,6 +415,8 @@ public override bool Operate(ref ObjectInput input, ref ObjectOutput output,
412415
throw new GarnetException($"Unsupported operation {op} in SortedSetObject.Operate");
413416
}
414417

418+
memorySizeChange = HeapMemorySize - prevMemorySize;
419+
415420
if (sortedSetDict.Count == 0)
416421
output.OutputFlags |= ObjectOutputFlags.RemoveKey;
417422

0 commit comments

Comments
 (0)