Skip to content

Commit 1c677f2

Browse files
jason-aelfjason-aelf
committed
perf: batch load block pruning data via manager bulk query APIs
1 parent 6176a6b commit 1c677f2

5 files changed

Lines changed: 93 additions & 8 deletions

File tree

‎src/AElf.Kernel.BlockPruning/Application/BlockPruningService.cs‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,27 +84,45 @@ public async Task PruneBlockchainDataAsync()
8484
var allTxIds = new List<Hash>();
8585
var allTxResultBlockHashes = new List<Hash>();
8686
var allBlockHashes = new List<Hash>();
87+
var heights = new List<long>();
88+
var foundBlockBodyCount = 0;
8789

8890
for (var height = batchStart; height <= batchEnd; height++)
8991
{
90-
var chainBlockIndex = await _chainManager.GetChainBlockIndexAsync(height);
92+
heights.Add(height);
93+
}
94+
95+
var chainBlockIndices = await _chainManager.GetChainBlockIndicesAsync(heights);
96+
97+
foreach (var chainBlockIndex in chainBlockIndices)
98+
{
9199
if (chainBlockIndex == null)
92100
continue;
93101

94-
var blockHash = chainBlockIndex.BlockHash;
95-
allBlockHashes.Add(blockHash);
102+
allBlockHashes.Add(chainBlockIndex.BlockHash);
103+
}
96104

97-
var block = await _blockManager.GetBlockAsync(blockHash);
98-
if (block == null)
105+
var blockBodies = await _blockManager.GetBlockBodiesAsync(allBlockHashes);
106+
107+
for (var i = 0; i < allBlockHashes.Count; i++)
108+
{
109+
var blockBody = blockBodies[i];
110+
if (blockBody == null)
99111
continue;
100112

101-
foreach (var txId in block.TransactionIds)
113+
foundBlockBodyCount++;
114+
var blockHash = allBlockHashes[i];
115+
foreach (var txId in blockBody.TransactionIds)
102116
{
103117
allTxIds.Add(txId);
104118
allTxResultBlockHashes.Add(blockHash);
105119
}
106120
}
107121

122+
Logger.LogDebug(
123+
"Pruning batch [{BatchStart}..{BatchEnd}]: found {ChainBlockIndexCount} chain block indices, loaded {BlockBodyCount} block bodies",
124+
batchStart, batchEnd, allBlockHashes.Count, foundBlockBodyCount);
125+
108126
await _transactionResultManager.RemoveTransactionResultsAsync(allTxIds, allTxResultBlockHashes);
109127
await _transactionBlockIndexManager.RemoveTransactionIndicesAsync(allTxIds);
110128
await _transactionManager.RemoveTransactionsAsync(allTxIds);

‎src/AElf.Kernel.Core/Blockchain/Domain/IBlockManager.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public interface IBlockManager
1212
Task AddBlockBodyAsync(Hash blockHash, BlockBody blockBody);
1313
Task<Block> GetBlockAsync(Hash blockHash);
1414
Task<BlockHeader> GetBlockHeaderAsync(Hash blockHash);
15+
Task<List<BlockBody>> GetBlockBodiesAsync(IList<Hash> blockHashes);
1516
Task RemoveBlockAsync(Hash blockHash);
1617
Task RemoveBlocksAsync(IList<Hash> blockHashes);
1718
Task<bool> HasBlockAsync(Hash blockHash);
@@ -67,6 +68,11 @@ public async Task<BlockHeader> GetBlockHeaderAsync(Hash blockHash)
6768
return await _blockHeaderStore.GetAsync(blockHash.ToStorageKey());
6869
}
6970

71+
public async Task<List<BlockBody>> GetBlockBodiesAsync(IList<Hash> blockHashes)
72+
{
73+
return await _blockBodyStore.GetAllAsync(blockHashes.Select(h => h.ToStorageKey()).ToList());
74+
}
75+
7076
public async Task RemoveBlockAsync(Hash blockHash)
7177
{
7278
var blockKey = blockHash.ToStorageKey();
@@ -90,4 +96,4 @@ private async Task<BlockBody> GetBlockBodyAsync(Hash bodyHash)
9096
{
9197
return await _blockBodyStore.GetAsync(bodyHash.ToStorageKey());
9298
}
93-
}
99+
}

‎src/AElf.Kernel.Core/Blockchain/Domain/IChainManager.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public interface IChainManager
2626
Task RemoveChainBlockLinksAsync(IList<Hash> blockHashes);
2727
void CleanCachedChainBlockLinks(long height);
2828
Task<ChainBlockIndex> GetChainBlockIndexAsync(long blockHeight);
29+
Task<List<ChainBlockIndex>> GetChainBlockIndicesAsync(IList<long> blockHeights);
2930
Task<BlockAttachOperationStatus> AttachBlockToChainAsync(Chain chain, ChainBlockLink chainBlockLink);
3031
Task<bool> SetIrreversibleBlockAsync(Chain chain, Hash irreversibleBlockHash);
3132
Task<List<ChainBlockLink>> GetNotExecutedBlocks(Hash blockHash);
@@ -169,6 +170,12 @@ public async Task<ChainBlockIndex> GetChainBlockIndexAsync(long blockHeight)
169170
blockHeight.ToStorageKey());
170171
}
171172

173+
public async Task<List<ChainBlockIndex>> GetChainBlockIndicesAsync(IList<long> blockHeights)
174+
{
175+
var prefix = ChainId.ToStorageKey() + KernelConstants.StorageKeySeparator;
176+
return await _chainBlockIndexes.GetAllAsync(blockHeights.Select(h => prefix + h.ToStorageKey()).ToList());
177+
}
178+
172179
public async Task<BlockAttachOperationStatus> AttachBlockToChainAsync(Chain chain, ChainBlockLink chainBlockLink)
173180
{
174181
var status = BlockAttachOperationStatus.None;
@@ -575,4 +582,4 @@ private async Task<List<string>> GetNotLinkedKeysAsync(Chain chain, long irrever
575582

576583
return toCleanNotLinkedKeys;
577584
}
578-
}
585+
}

‎test/AElf.Kernel.Core.Tests/Blockchain/Domain/BlockManagerTests.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,26 @@ public async Task RemoveBlocksAsync_BatchDelete_Test()
6565
foreach (var h in hashes) (await _blockManager.GetBlockAsync(h)).ShouldBeNull();
6666
}
6767

68+
[Fact]
69+
public async Task GetBlockBodiesAsync_Should_Preserve_Input_Order_And_Missing_Entries_Test()
70+
{
71+
var blockA = _kernelTestHelper.GenerateBlock(0, Hash.Empty);
72+
var blockB = _kernelTestHelper.GenerateBlock(1, blockA.GetHash());
73+
var hashA = blockA.GetHash();
74+
var hashB = blockB.GetHash();
75+
var fakeHash = HashHelper.ComputeFrom("missing-block");
76+
77+
await _blockManager.AddBlockBodyAsync(hashA, blockA.Body);
78+
await _blockManager.AddBlockBodyAsync(hashB, blockB.Body);
79+
80+
var blockBodies = await _blockManager.GetBlockBodiesAsync(new List<Hash> { hashB, fakeHash, hashA });
81+
82+
blockBodies.Count.ShouldBe(3);
83+
blockBodies[0].ShouldBe(blockB.Body);
84+
blockBodies[1].ShouldBeNull();
85+
blockBodies[2].ShouldBe(blockA.Body);
86+
}
87+
6888
[Fact]
6989
public async Task RemoveBlocksAsync_PartialDelete_Test()
7090
{

‎test/AElf.Kernel.Core.Tests/Blockchain/Domain/ChainManagerTests.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,40 @@ public async Task Create_Chain_ThrowInvalidOperationException()
8383
await _chainManager.CreateAsync(_blocks[1]).ShouldThrowAsync<InvalidOperationException>();
8484
}
8585

86+
[Fact]
87+
public async Task GetChainBlockIndicesAsync_Should_Preserve_Input_Order_And_Missing_Entries_Test()
88+
{
89+
var chain = await _chainManager.CreateAsync(_genesis);
90+
91+
await _chainManager.AttachBlockToChainAsync(chain, new ChainBlockLink
92+
{
93+
Height = 1.BlockHeight(),
94+
BlockHash = _blocks[1],
95+
PreviousBlockHash = _genesis
96+
});
97+
98+
await _chainManager.AttachBlockToChainAsync(chain, new ChainBlockLink
99+
{
100+
Height = 2.BlockHeight(),
101+
BlockHash = _blocks[2],
102+
PreviousBlockHash = _blocks[1]
103+
});
104+
105+
(await _chainManager.SetIrreversibleBlockAsync(chain, _blocks[2])).ShouldBeTrue();
106+
107+
var chainBlockIndices = await _chainManager.GetChainBlockIndicesAsync(new List<long>
108+
{
109+
2.BlockHeight(),
110+
3.BlockHeight(),
111+
0.BlockHeight()
112+
});
113+
114+
chainBlockIndices.Count.ShouldBe(3);
115+
chainBlockIndices[0].BlockHash.ShouldBe(_blocks[2]);
116+
chainBlockIndices[1].ShouldBeNull();
117+
chainBlockIndices[2].BlockHash.ShouldBe(_genesis);
118+
}
119+
86120
[Fact]
87121
public async Task LIB_Blocks_Test()
88122
{

0 commit comments

Comments
 (0)