Skip to content

Commit 7180d26

Browse files
karalabekaralabe
authored
core, eth, node: break rawdb -> {leveldb, pebble} dependency (ethereum#30689)
1 parent 98056e1 commit 7180d26

10 files changed

Lines changed: 184 additions & 377 deletions

‎core/bench_test.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/ethereum/go-ethereum/core/vm"
3030
"github.com/ethereum/go-ethereum/crypto"
3131
"github.com/ethereum/go-ethereum/ethdb"
32+
"github.com/ethereum/go-ethereum/ethdb/pebble"
3233
"github.com/ethereum/go-ethereum/params"
3334
)
3435

@@ -173,18 +174,16 @@ func genUncles(i int, gen *BlockGen) {
173174
func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
174175
// Create the database in memory or in a temporary directory.
175176
var db ethdb.Database
176-
var err error
177177
if !disk {
178178
db = rawdb.NewMemoryDatabase()
179179
} else {
180-
dir := b.TempDir()
181-
db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
180+
pdb, err := pebble.New(b.TempDir(), 128, 128, "", false)
182181
if err != nil {
183182
b.Fatalf("cannot create temporary database: %v", err)
184183
}
184+
db = rawdb.NewDatabase(pdb)
185185
defer db.Close()
186186
}
187-
188187
// Generate a chain of b.N blocks using the supplied block
189188
// generator function.
190189
gspec := &Genesis{
@@ -281,11 +280,11 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin
281280
func benchWriteChain(b *testing.B, full bool, count uint64) {
282281
genesis := &Genesis{Config: params.AllEthashProtocolChanges}
283282
for i := 0; i < b.N; i++ {
284-
dir := b.TempDir()
285-
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
283+
pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false)
286284
if err != nil {
287-
b.Fatalf("error opening database at %v: %v", dir, err)
285+
b.Fatalf("error opening database: %v", err)
288286
}
287+
db := rawdb.NewDatabase(pdb)
289288
makeChainForBench(db, genesis, full, count)
290289
db.Close()
291290
}
@@ -294,10 +293,12 @@ func benchWriteChain(b *testing.B, full bool, count uint64) {
294293
func benchReadChain(b *testing.B, full bool, count uint64) {
295294
dir := b.TempDir()
296295

297-
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
296+
pdb, err := pebble.New(dir, 1024, 128, "", false)
298297
if err != nil {
299-
b.Fatalf("error opening database at %v: %v", dir, err)
298+
b.Fatalf("error opening database: %v", err)
300299
}
300+
db := rawdb.NewDatabase(pdb)
301+
301302
genesis := &Genesis{Config: params.AllEthashProtocolChanges}
302303
makeChainForBench(db, genesis, full, count)
303304
db.Close()
@@ -308,15 +309,16 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
308309
b.ResetTimer()
309310

310311
for i := 0; i < b.N; i++ {
311-
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
312+
pdb, err = pebble.New(dir, 1024, 128, "", false)
312313
if err != nil {
313-
b.Fatalf("error opening database at %v: %v", dir, err)
314+
b.Fatalf("error opening database: %v", err)
314315
}
316+
db = rawdb.NewDatabase(pdb)
317+
315318
chain, err := NewBlockChain(db, &cacheConfig, genesis, nil, ethash.NewFaker(), vm.Config{}, nil)
316319
if err != nil {
317320
b.Fatalf("error creating chain: %v", err)
318321
}
319-
320322
for n := uint64(0); n < count; n++ {
321323
header := chain.GetHeaderByNumber(n)
322324
if full {

‎core/blockchain_repair_test.go‎

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/ethereum/go-ethereum/core/rawdb"
3232
"github.com/ethereum/go-ethereum/core/types"
3333
"github.com/ethereum/go-ethereum/core/vm"
34+
"github.com/ethereum/go-ethereum/ethdb/pebble"
3435
"github.com/ethereum/go-ethereum/params"
3536
)
3637

@@ -1764,12 +1765,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
17641765
datadir := t.TempDir()
17651766
ancient := filepath.Join(datadir, "ancient")
17661767

1767-
db, err := rawdb.Open(rawdb.OpenOptions{
1768-
Directory: datadir,
1769-
AncientsDirectory: ancient,
1770-
})
1768+
pdb, err := pebble.New(datadir, 0, 0, "", false)
1769+
if err != nil {
1770+
t.Fatalf("Failed to create persistent key-value database: %v", err)
1771+
}
1772+
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
17711773
if err != nil {
1772-
t.Fatalf("Failed to create persistent database: %v", err)
1774+
t.Fatalf("Failed to create persistent freezer database: %v", err)
17731775
}
17741776
defer db.Close() // Might double close, should be fine
17751777

@@ -1848,12 +1850,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
18481850
chain.stopWithoutSaving()
18491851

18501852
// Start a new blockchain back up and see where the repair leads us
1851-
db, err = rawdb.Open(rawdb.OpenOptions{
1852-
Directory: datadir,
1853-
AncientsDirectory: ancient,
1854-
})
1853+
pdb, err = pebble.New(datadir, 0, 0, "", false)
1854+
if err != nil {
1855+
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
1856+
}
1857+
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
18551858
if err != nil {
1856-
t.Fatalf("Failed to reopen persistent database: %v", err)
1859+
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
18571860
}
18581861
defer db.Close()
18591862

@@ -1912,12 +1915,13 @@ func testIssue23496(t *testing.T, scheme string) {
19121915
datadir := t.TempDir()
19131916
ancient := filepath.Join(datadir, "ancient")
19141917

1915-
db, err := rawdb.Open(rawdb.OpenOptions{
1916-
Directory: datadir,
1917-
AncientsDirectory: ancient,
1918-
})
1918+
pdb, err := pebble.New(datadir, 0, 0, "", false)
1919+
if err != nil {
1920+
t.Fatalf("Failed to create persistent key-value database: %v", err)
1921+
}
1922+
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
19191923
if err != nil {
1920-
t.Fatalf("Failed to create persistent database: %v", err)
1924+
t.Fatalf("Failed to create persistent freezer database: %v", err)
19211925
}
19221926
defer db.Close() // Might double close, should be fine
19231927

@@ -1969,12 +1973,13 @@ func testIssue23496(t *testing.T, scheme string) {
19691973
chain.stopWithoutSaving()
19701974

19711975
// Start a new blockchain back up and see where the repair leads us
1972-
db, err = rawdb.Open(rawdb.OpenOptions{
1973-
Directory: datadir,
1974-
AncientsDirectory: ancient,
1975-
})
1976+
pdb, err = pebble.New(datadir, 0, 0, "", false)
1977+
if err != nil {
1978+
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
1979+
}
1980+
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
19761981
if err != nil {
1977-
t.Fatalf("Failed to reopen persistent database: %v", err)
1982+
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
19781983
}
19791984
defer db.Close()
19801985

‎core/blockchain_sethead_test.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/ethereum/go-ethereum/core/state"
3434
"github.com/ethereum/go-ethereum/core/types"
3535
"github.com/ethereum/go-ethereum/core/vm"
36+
"github.com/ethereum/go-ethereum/ethdb/pebble"
3637
"github.com/ethereum/go-ethereum/params"
3738
"github.com/ethereum/go-ethereum/triedb"
3839
"github.com/ethereum/go-ethereum/triedb/hashdb"
@@ -1968,12 +1969,13 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
19681969
datadir := t.TempDir()
19691970
ancient := filepath.Join(datadir, "ancient")
19701971

1971-
db, err := rawdb.Open(rawdb.OpenOptions{
1972-
Directory: datadir,
1973-
AncientsDirectory: ancient,
1974-
})
1972+
pdb, err := pebble.New(datadir, 0, 0, "", false)
1973+
if err != nil {
1974+
t.Fatalf("Failed to create persistent key-value database: %v", err)
1975+
}
1976+
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
19751977
if err != nil {
1976-
t.Fatalf("Failed to create persistent database: %v", err)
1978+
t.Fatalf("Failed to create persistent freezer database: %v", err)
19771979
}
19781980
defer db.Close()
19791981

‎core/blockchain_snapshot_test.go‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"github.com/ethereum/go-ethereum/core/types"
3636
"github.com/ethereum/go-ethereum/core/vm"
3737
"github.com/ethereum/go-ethereum/ethdb"
38+
"github.com/ethereum/go-ethereum/ethdb/pebble"
3839
"github.com/ethereum/go-ethereum/params"
3940
)
4041

@@ -65,12 +66,13 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
6566
datadir := t.TempDir()
6667
ancient := filepath.Join(datadir, "ancient")
6768

68-
db, err := rawdb.Open(rawdb.OpenOptions{
69-
Directory: datadir,
70-
AncientsDirectory: ancient,
71-
})
69+
pdb, err := pebble.New(datadir, 0, 0, "", false)
7270
if err != nil {
73-
t.Fatalf("Failed to create persistent database: %v", err)
71+
t.Fatalf("Failed to create persistent key-value database: %v", err)
72+
}
73+
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
74+
if err != nil {
75+
t.Fatalf("Failed to create persistent freezer database: %v", err)
7476
}
7577
// Initialize a fresh chain
7678
var (
@@ -255,12 +257,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
255257
chain.triedb.Close()
256258

257259
// Start a new blockchain back up and see where the repair leads us
258-
newdb, err := rawdb.Open(rawdb.OpenOptions{
259-
Directory: snaptest.datadir,
260-
AncientsDirectory: snaptest.ancient,
261-
})
260+
pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false)
261+
if err != nil {
262+
t.Fatalf("Failed to create persistent key-value database: %v", err)
263+
}
264+
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
262265
if err != nil {
263-
t.Fatalf("Failed to reopen persistent database: %v", err)
266+
t.Fatalf("Failed to create persistent freezer database: %v", err)
264267
}
265268
defer newdb.Close()
266269

‎core/blockchain_test.go‎

Lines changed: 7 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929
"time"
3030

3131
"github.com/ethereum/go-ethereum/common"
32-
"github.com/ethereum/go-ethereum/common/math"
3332
"github.com/ethereum/go-ethereum/consensus"
3433
"github.com/ethereum/go-ethereum/consensus/beacon"
3534
"github.com/ethereum/go-ethereum/consensus/ethash"
@@ -40,6 +39,7 @@ import (
4039
"github.com/ethereum/go-ethereum/crypto"
4140
"github.com/ethereum/go-ethereum/eth/tracers/logger"
4241
"github.com/ethereum/go-ethereum/ethdb"
42+
"github.com/ethereum/go-ethereum/ethdb/pebble"
4343
"github.com/ethereum/go-ethereum/params"
4444
"github.com/ethereum/go-ethereum/trie"
4545
"github.com/holiman/uint256"
@@ -2663,12 +2663,13 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
26632663
datadir := t.TempDir()
26642664
ancient := path.Join(datadir, "ancient")
26652665

2666-
db, err := rawdb.Open(rawdb.OpenOptions{
2667-
Directory: datadir,
2668-
AncientsDirectory: ancient,
2669-
})
2666+
pdb, err := pebble.New(datadir, 0, 0, "", false)
2667+
if err != nil {
2668+
t.Fatalf("Failed to create persistent key-value database: %v", err)
2669+
}
2670+
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
26702671
if err != nil {
2671-
t.Fatalf("Failed to create persistent database: %v", err)
2672+
t.Fatalf("Failed to create persistent freezer database: %v", err)
26722673
}
26732674
defer db.Close()
26742675

@@ -4231,36 +4232,3 @@ func TestPragueRequests(t *testing.T) {
42314232
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
42324233
}
42334234
}
4234-
4235-
func BenchmarkReorg(b *testing.B) {
4236-
chainLength := b.N
4237-
4238-
dir := b.TempDir()
4239-
db, err := rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
4240-
if err != nil {
4241-
b.Fatalf("cannot create temporary database: %v", err)
4242-
}
4243-
defer db.Close()
4244-
gspec := &Genesis{
4245-
Config: params.TestChainConfig,
4246-
Alloc: types.GenesisAlloc{benchRootAddr: {Balance: math.BigPow(2, 254)}},
4247-
}
4248-
blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
4249-
defer blockchain.Stop()
4250-
4251-
// Insert an easy and a difficult chain afterwards
4252-
easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000))
4253-
diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000))
4254-
4255-
if _, err := blockchain.InsertChain(easyBlocks); err != nil {
4256-
b.Fatalf("failed to insert easy chain: %v", err)
4257-
}
4258-
b.ResetTimer()
4259-
if _, err := blockchain.InsertChain(diffBlocks); err != nil {
4260-
b.Fatalf("failed to insert difficult chain: %v", err)
4261-
}
4262-
}
4263-
4264-
// Master: BenchmarkReorg-8 10000 899591 ns/op 820154 B/op 1440 allocs/op 1549443072 bytes of heap used
4265-
// WithoutOldChain: BenchmarkReorg-8 10000 1147281 ns/op 943163 B/op 1564 allocs/op 1163870208 bytes of heap used
4266-
// WithoutNewChain: BenchmarkReorg-8 10000 1018922 ns/op 943580 B/op 1564 allocs/op 1171890176 bytes of heap used

0 commit comments

Comments
 (0)