Skip to content

Commit 4f6eea4

Browse files
authored
Merge pull request #2377 from CortexFoundation/dev
integrate eradb backend
2 parents 8f0fbd0 + bae9f0b commit 4f6eea4

38 files changed

Lines changed: 938 additions & 341 deletions

cmd/cortex/main.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ var (
6666
utils.BootnodesV4Flag,
6767
// utils.BootnodesV5Flag,
6868
utils.DataDirFlag,
69+
utils.EraFlag,
6970
utils.DBEngineFlag,
7071
utils.AncientFlag,
7172
utils.MinFreeDiskSpaceFlag,
@@ -219,7 +220,7 @@ func init() {
219220
// Initialize the CLI app and start Ctxc
220221
app.Action = cortex
221222
app.HideVersion = true // we have a command to print the version
222-
app.Copyright = "Copyright 2018-2019 The go-ethereum Authors"
223+
app.Copyright = "Copyright 2018-2025 The cortex Authors"
223224
app.Commands = []cli.Command{
224225
// See chaincmd.go:
225226
initCommand,
@@ -264,6 +265,8 @@ func init() {
264265
app.Flags = append(app.Flags, cvmFlags...)
265266
app.Flags = append(app.Flags, storageFlags...)
266267
app.Flags = append(app.Flags, inferFlags...)
268+
//app.Flags = append(app.Flags, utils.NetworkFlags...)
269+
//app.Flags = append(app.Flags, utils.DatabaseFlags...)
267270

268271
//flags.AutoEnvVars(app.Flags, "CORTEX")
269272
app.Before = func(ctx *cli.Context) error {

cmd/cortex/usage.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
var AppHelpTemplate = `NAME:
3434
{{.App.Name}} - {{.App.Usage}}
3535
36-
Copyright 2018-2019 The go-ethereum Authors
36+
Copyright 2018-2025 The cortex Authors
3737
3838
USAGE:
3939
{{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -71,6 +71,7 @@ var AppHelpFlagGroups = []flagGroup{
7171
utils.DataDirFlag,
7272
utils.DBEngineFlag,
7373
utils.AncientFlag,
74+
utils.EraFlag,
7475
utils.MinFreeDiskSpaceFlag,
7576
utils.KeyStoreDirFlag,
7677
// utils.NoUSBFlag,

cmd/utils/flags.go

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,10 @@ var (
145145
Usage: "Data directory for the databases and keystore",
146146
Value: DirectoryString{node.DefaultDataDir()},
147147
}
148+
RemoteDBFlag = &cli.StringFlag{
149+
Name: "remotedb",
150+
Usage: "URL for remote database",
151+
}
148152
DBEngineFlag = &cli.StringFlag{
149153
Name: "db.engine",
150154
Usage: "Backing database implementation to use ('pebble' or 'leveldb')",
@@ -154,6 +158,10 @@ var (
154158
Name: "datadir.ancient",
155159
Usage: "Data directory for ancient chain segments (default = inside chaindata)",
156160
}
161+
EraFlag = DirectoryFlag{
162+
Name: "datadir.era",
163+
Usage: "Root directory for era1 history (default = inside ancient/chain)",
164+
}
157165
MinFreeDiskSpaceFlag = DirectoryFlag{
158166
Name: "datadir.minfreedisk",
159167
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@@ -866,6 +874,8 @@ var (
866874
DatabaseFlags = []cli.Flag{
867875
DataDirFlag,
868876
AncientFlag,
877+
EraFlag,
878+
RemoteDBFlag,
869879
DBEngineFlag,
870880
}
871881
)
@@ -1506,6 +1516,9 @@ func SetCortexConfig(ctx *cli.Context, stack *node.Node, cfg *ctxc.Config) {
15061516
if ctx.GlobalIsSet(AncientFlag.Name) {
15071517
cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name)
15081518
}
1519+
if ctx.IsSet(EraFlag.Name) {
1520+
cfg.DatabaseEra = ctx.String(EraFlag.Name)
1521+
}
15091522

15101523
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
15111524
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@@ -1889,9 +1902,18 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ctxcdb
18891902
cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
18901903
handles = makeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name))
18911904
)
1892-
name := "chaindata"
1905+
//name := "chaindata"
1906+
options := node.DatabaseOptions{
1907+
ReadOnly: readonly,
1908+
Cache: cache,
1909+
Handles: handles,
1910+
AncientsDirectory: ctx.String(AncientFlag.Name),
1911+
MetricsNamespace: "ctxc/db/chaindata/",
1912+
EraDirectory: ctx.String(EraFlag.Name),
1913+
}
18931914
//chainDb, err := stack.OpenDatabase(name, cache, handles)
1894-
chainDb, err := stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
1915+
//chainDb, err := stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
1916+
chainDb, err := stack.OpenDatabaseWithOptions("chaindata", options)
18951917
if err != nil {
18961918
Fatalf("Could not open database: %v", err)
18971919
}

common/lru/basiclru.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,30 +47,37 @@ func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] {
4747

4848
// Add adds a value to the cache. Returns true if an item was evicted to store the new item.
4949
func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
50+
_, _, evicted = c.Add3(key, value)
51+
return evicted
52+
}
53+
54+
// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item.
55+
func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) {
5056
item, ok := c.items[key]
5157
if ok {
52-
// Already exists in cache.
5358
item.value = value
5459
c.items[key] = item
5560
c.list.moveToFront(item.elem)
56-
return false
61+
return ek, ev, false
5762
}
5863

5964
var elem *listElem[K]
6065
if c.Len() >= c.cap {
6166
elem = c.list.removeLast()
62-
delete(c.items, elem.v)
6367
evicted = true
68+
ek = elem.v
69+
ev = c.items[ek].value
70+
delete(c.items, ek)
6471
} else {
6572
elem = new(listElem[K])
6673
}
6774

6875
// Store the new item.
69-
// Note that, if another item was evicted, we re-use its list element here.
76+
// Note that if another item was evicted, we re-use its list element here.
7077
elem.v = key
7178
c.items[key] = cacheItem[K, V]{elem, value}
7279
c.list.pushElem(elem)
73-
return evicted
80+
return ek, ev, evicted
7481
}
7582

7683
// Contains reports whether the given key exists in the cache.

core/rawdb/accessors_chain_test.go

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"encoding/hex"
2222
"fmt"
2323
"math/big"
24-
"os"
2524
"reflect"
2625
"testing"
2726

@@ -363,13 +362,8 @@ func checkReceiptsRLP(have, want types.Receipts) error {
363362

364363
func TestAncientStorage(t *testing.T) {
365364
// Freezer style fast import the chain.
366-
frdir, err := os.MkdirTemp("", "")
367-
if err != nil {
368-
t.Fatalf("failed to create temp freezer dir: %v", err)
369-
}
370-
defer os.RemoveAll(frdir)
371-
372-
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
365+
frdir := t.TempDir()
366+
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: frdir})
373367
if err != nil {
374368
t.Fatalf("failed to create database with ancient backend")
375369
}

core/rawdb/ancient_scheme.go

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,24 @@ const (
4040
ChainFreezerDifficultyTable = "diffs"
4141
)
4242

43+
// chainFreezerTableConfigs configures the settings for tables in the chain freezer.
44+
// Compression is disabled for hashes as they don't compress well. Additionally,
45+
// tail truncation is disabled for the header and hash tables, as these are intended
46+
// to be retained long-term.
47+
var chainFreezerTableConfigs = map[string]freezerTableConfig{
48+
ChainFreezerHeaderTable: {noSnappy: false, prunable: false},
49+
ChainFreezerHashTable: {noSnappy: true, prunable: false},
50+
ChainFreezerBodiesTable: {noSnappy: false, prunable: true},
51+
ChainFreezerReceiptTable: {noSnappy: false, prunable: true},
52+
ChainFreezerDifficultyTable: {noSnappy: true, prunable: false},
53+
}
54+
55+
// freezerTableConfig contains the settings for a freezer table.
56+
type freezerTableConfig struct {
57+
noSnappy bool // disables item compression
58+
prunable bool // true for tables that can be pruned by TruncateTail
59+
}
60+
4361
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
4462
// Hashes and difficulties don't compress well.
4563
var chainFreezerNoSnappy = map[string]bool{
@@ -71,6 +89,15 @@ var stateFreezerNoSnappy = map[string]bool{
7189
stateHistoryStorageData: false,
7290
}
7391

92+
// stateFreezerTableConfigs configures the settings for tables in the state freezer.
93+
var stateFreezerTableConfigs = map[string]freezerTableConfig{
94+
stateHistoryMeta: {noSnappy: true, prunable: true},
95+
stateHistoryAccountIndex: {noSnappy: false, prunable: true},
96+
stateHistoryStorageIndex: {noSnappy: false, prunable: true},
97+
stateHistoryAccountData: {noSnappy: false, prunable: true},
98+
stateHistoryStorageData: {noSnappy: false, prunable: true},
99+
}
100+
74101
// The list of identifiers of ancient stores.
75102
var (
76103
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
@@ -88,7 +115,7 @@ var freezers = []string{ChainFreezerName, StateFreezerName}
88115
// state freezer.
89116
func NewStateFreezer(ancientDir string, readOnly bool) (ctxcdb.ResettableAncientStore, error) {
90117
if ancientDir == "" {
91-
return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil
118+
return NewMemoryFreezer(readOnly, stateFreezerTableConfigs), nil
92119
}
93-
return newResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
120+
return newResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "ctxc/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs)
94121
}

core/rawdb/ancient_utils.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,13 +97,13 @@ func inspectFreezers(db ctxcdb.Database) ([]freezerInfo, error) {
9797
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
9898
var (
9999
path string
100-
tables map[string]bool
100+
tables map[string]freezerTableConfig
101101
)
102102
switch freezerName {
103103
case ChainFreezerName:
104-
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
104+
path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs
105105
case StateFreezerName:
106-
path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy
106+
path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs
107107
default:
108108
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
109109
}

core/rawdb/chain_freezer.go

Lines changed: 97 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"time"
2424

2525
"github.com/CortexFoundation/CortexTheseus/common"
26+
"github.com/CortexFoundation/CortexTheseus/core/rawdb/eradb"
2627
"github.com/CortexFoundation/CortexTheseus/ctxcdb"
2728
"github.com/CortexFoundation/CortexTheseus/log"
2829
"github.com/CortexFoundation/CortexTheseus/params"
@@ -43,7 +44,10 @@ const (
4344
// feature. The background thread will keep moving ancient chain segments from
4445
// key-value database to flat files for saving space on live database.
4546
type chainFreezer struct {
46-
ctxcdb.AncientStore // Ancient store for storing cold chain segment
47+
ancients ctxcdb.AncientStore // Ancient store for storing cold chain segment
48+
49+
// Optional Era database used as a backup for the pruned chain.
50+
eradb *eradb.Store
4751

4852
quit chan struct{}
4953
wg sync.WaitGroup
@@ -56,23 +60,27 @@ type chainFreezer struct {
5660
// state freezer (e.g. dev mode).
5761
// - if non-empty directory is given, initializes the regular file-based
5862
// state freezer.
59-
func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFreezer, error) {
60-
var (
61-
err error
62-
freezer ctxcdb.AncientStore
63-
)
63+
func newChainFreezer(datadir string, eraDir string, namespace string, readonly bool) (*chainFreezer, error) {
6464
if datadir == "" {
65-
freezer = NewMemoryFreezer(readonly, chainFreezerNoSnappy)
66-
} else {
67-
freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerNoSnappy)
65+
return &chainFreezer{
66+
ancients: NewMemoryFreezer(readonly, chainFreezerTableConfigs),
67+
quit: make(chan struct{}),
68+
trigger: make(chan chan struct{}),
69+
}, nil
70+
}
71+
freezer, err := NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs)
72+
if err != nil {
73+
return nil, err
6874
}
75+
edb, err := eradb.New(resolveChainEraDir(datadir, eraDir))
6976
if err != nil {
7077
return nil, err
7178
}
7279
return &chainFreezer{
73-
AncientStore: freezer,
74-
quit: make(chan struct{}),
75-
trigger: make(chan chan struct{}),
80+
ancients: freezer,
81+
eradb: edb,
82+
quit: make(chan struct{}),
83+
trigger: make(chan chan struct{}),
7684
}, nil
7785
}
7886

@@ -84,7 +92,11 @@ func (f *chainFreezer) Close() error {
8492
close(f.quit)
8593
}
8694
f.wg.Wait()
87-
return f.AncientStore.Close()
95+
96+
if f.eradb != nil {
97+
f.eradb.Close()
98+
}
99+
return f.ancients.Close()
88100
}
89101

90102
// readHeadNumber returns the number of chain head block. 0 is returned if the
@@ -342,3 +354,75 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
342354
})
343355
return hashes, err
344356
}
357+
358+
// Ancient retrieves an ancient binary blob from the append-only immutable files.
359+
func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
360+
// Lookup the entry in the underlying ancient store, assuming that
361+
// headers and hashes are always available.
362+
if kind == ChainFreezerHeaderTable || kind == ChainFreezerHashTable {
363+
return f.ancients.Ancient(kind, number)
364+
}
365+
tail, err := f.ancients.Tail()
366+
if err != nil {
367+
return nil, err
368+
}
369+
// Lookup the entry in the underlying ancient store if it's not pruned
370+
if number >= tail {
371+
return f.ancients.Ancient(kind, number)
372+
}
373+
// Lookup the entry in the optional era backend
374+
if f.eradb == nil {
375+
return nil, errOutOfBounds
376+
}
377+
switch kind {
378+
case ChainFreezerBodiesTable:
379+
return f.eradb.GetRawBody(number)
380+
case ChainFreezerReceiptTable:
381+
return f.eradb.GetRawReceipts(number)
382+
}
383+
return nil, errUnknownTable
384+
}
385+
386+
// ReadAncients executes an operation while preventing mutations to the freezer,
387+
// i.e. if fn performs multiple reads, they will be consistent with each other.
388+
func (f *chainFreezer) ReadAncients(fn func(ctxcdb.AncientReaderOp) error) (err error) {
389+
if store, ok := f.ancients.(*Freezer); ok {
390+
store.writeLock.Lock()
391+
defer store.writeLock.Unlock()
392+
}
393+
return fn(f)
394+
}
395+
396+
// Methods below are just pass-through to the underlying ancient store.
397+
398+
func (f *chainFreezer) Ancients() (uint64, error) {
399+
return f.ancients.Ancients()
400+
}
401+
402+
func (f *chainFreezer) Tail() (uint64, error) {
403+
return f.ancients.Tail()
404+
}
405+
406+
func (f *chainFreezer) AncientSize(kind string) (uint64, error) {
407+
return f.ancients.AncientSize(kind)
408+
}
409+
410+
func (f *chainFreezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
411+
return f.ancients.AncientRange(kind, start, count, maxBytes)
412+
}
413+
414+
func (f *chainFreezer) ModifyAncients(fn func(ctxcdb.AncientWriteOp) error) (int64, error) {
415+
return f.ancients.ModifyAncients(fn)
416+
}
417+
418+
func (f *chainFreezer) TruncateHead(items uint64) (uint64, error) {
419+
return f.ancients.TruncateHead(items)
420+
}
421+
422+
func (f *chainFreezer) TruncateTail(items uint64) (uint64, error) {
423+
return f.ancients.TruncateTail(items)
424+
}
425+
426+
func (f *chainFreezer) SyncAncient() error {
427+
return f.ancients.SyncAncient()
428+
}

0 commit comments

Comments
 (0)