From c260e36ba0a54a1f83cd07ac8e8b0ac743f36adb Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 27 Jan 2025 20:52:02 +0100 Subject: [PATCH 01/99] add basic eip7732 (epbs) support --- clients/consensus/chainspec.go | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- handlers/index.go | 8 ++++++++ indexer/beacon/block_helper.go | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 3bb2b347a..786aa76f6 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -36,6 +36,8 @@ type ChainSpec struct { ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH"` Eip7594ForkVersion phase0.Version `yaml:"EIP7594_FORK_VERSION" check-if-fork:"Eip7594ForkEpoch"` Eip7594ForkEpoch *uint64 `yaml:"EIP7594_FORK_EPOCH"` + Eip7732ForkVersion phase0.Version `yaml:"EIP7732_FORK_VERSION" check-if-fork:"Eip7732ForkEpoch"` + Eip7732ForkEpoch *uint64 `yaml:"EIP7732_FORK_EPOCH"` SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"` SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"` EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"` @@ -64,6 +66,11 @@ type ChainSpec struct { DataColumnSidecarSubnetCount *uint64 `yaml:"DATA_COLUMN_SIDECAR_SUBNET_COUNT" check-if-fork:"Eip7594ForkEpoch"` CustodyRequirement *uint64 `yaml:"CUSTODY_REQUIREMENT" check-if-fork:"Eip7594ForkEpoch"` + // EIP7732: ePBS + PtcSize uint64 `yaml:"PTC_SIZE" check-if-fork:"Eip7732ForkEpoch"` + MaxPayloadAttestations uint64 `yaml:"MAX_PAYLOAD_ATTESTATIONS" check-if-fork:"Eip7732ForkEpoch"` + DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"Eip7732ForkEpoch"` + // additional dora specific specs WhiskForkEpoch *uint64 } diff --git a/go.mod b/go.mod index 52eedc4f8..cc0004d3c 100644 --- a/go.mod +++ b/go.mod @@ -152,6 +152,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b54ab8ba1..1d161da4d 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43 h1:QVr4VkeZGzR7Mxtg1PKZ6n97aQU4VfFbTVD5hZauNUM= -github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= @@ -348,6 +346,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= +github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd h1:xty/DNk1Y9vwj2Enx/D/eFBoSL0zf4V6wkqCRHsmliY= +github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/handlers/index.go b/handlers/index.go index 1005ef354..24a52efab 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -216,6 +216,14 @@ func buildIndexPageData() (*models.IndexPageData, time.Duration) { Active: uint64(currentEpoch) >= *specs.Eip7594ForkEpoch, }) } + if specs.Eip7732ForkEpoch != nil && *specs.Eip7732ForkEpoch < uint64(18446744073709551615) { + pageData.NetworkForks = append(pageData.NetworkForks, &models.IndexPageDataForks{ + Name: "eip7732", + Epoch: *specs.Eip7732ForkEpoch, + Version: specs.Eip7732ForkVersion[:], + Active: uint64(currentEpoch) >= *specs.Eip7732ForkEpoch, + }) + } // load recent epochs buildIndexPageRecentEpochsData(pageData, currentEpoch, finalizedEpoch, justifiedEpoch, recentEpochCount) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 843a11003..34f03b06d 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -9,6 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" @@ -44,6 +45,9 @@ func marshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, block *spec.Ver case spec.DataVersionElectra: version = uint64(block.Version) ssz, err = dynSsz.MarshalSSZ(block.Electra) + case spec.DataVersionEIP7732: + version = uint64(block.Version) + ssz, err = dynSsz.MarshalSSZ(block.EIP7732) default: err = fmt.Errorf("unknown block version") } @@ -110,6 +114,11 @@ func unmarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, version uint6 if err := dynSsz.UnmarshalSSZ(block.Electra, ssz); err != nil { return nil, fmt.Errorf("failed to decode electra signed beacon block: %v", err) } + case spec.DataVersionEIP7732: + block.EIP7732 = &eip7732.SignedBeaconBlock{} + if err := dynSsz.UnmarshalSSZ(block.EIP7732, ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -137,6 +146,9 @@ func marshalVersionedSignedBeaconBlockJson(block *spec.VersionedSignedBeaconBloc case spec.DataVersionElectra: version = uint64(block.Version) jsonRes, err = block.Electra.MarshalJSON() + case spec.DataVersionEIP7732: + version = uint64(block.Version) + jsonRes, err = block.EIP7732.MarshalJSON() default: err = fmt.Errorf("unknown block version") } @@ -185,6 +197,11 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. if err := block.Electra.UnmarshalJSON(ssz); err != nil { return nil, fmt.Errorf("failed to decode electra signed beacon block: %v", err) } + case spec.DataVersionEIP7732: + block.EIP7732 = &eip7732.SignedBeaconBlock{} + if err := block.EIP7732.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -218,6 +235,8 @@ func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, err } return v.Electra.Message.Body.ExecutionPayload.ExtraData, nil + case spec.DataVersionEIP7732: + return nil, nil default: return nil, errors.New("unknown version") } @@ -262,6 +281,12 @@ func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { } return v.Electra.RANDAOMixes, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.RANDAOMixes == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.RANDAOMixes, nil default: return nil, errors.New("unknown version") } @@ -282,6 +307,8 @@ func getStateDepositIndex(state *spec.VersionedBeaconState) uint64 { return state.Deneb.ETH1DepositIndex case spec.DataVersionElectra: return state.Electra.ETH1DepositIndex + case spec.DataVersionEIP7732: + return state.EIP7732.ETH1DepositIndex } return 0 } @@ -321,6 +348,12 @@ func getStateCurrentSyncCommittee(v *spec.VersionedBeaconState) ([]phase0.BLSPub } return v.Electra.CurrentSyncCommittee.Pubkeys, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.CurrentSyncCommittee == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.CurrentSyncCommittee.Pubkeys, nil default: return nil, errors.New("unknown version") } From 32a348f87aebb67dd119cf9317214d27106eca61 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 27 Jan 2025 21:07:06 +0100 Subject: [PATCH 02/99] trigger CI From 311e4af4e15e7c3339da9d98f91edc1a74042820 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 5 Feb 2025 11:43:20 +0100 Subject: [PATCH 03/99] bump go-eth2-client --- go.mod | 13 +++++++------ go.sum | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index cc0004d3c..ab4fc2540 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( ) require ( + github.com/emicklei/dot v1.6.4 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/minio/highwayhash v1.0.2 // indirect @@ -83,8 +84,8 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.2 // indirect github.com/ethereum/go-verkle v0.2.2 // indirect - github.com/fatih/color v1.17.0 // indirect - github.com/ferranbt/fastssz v0.1.3 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/ferranbt/fastssz v0.1.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -106,8 +107,8 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -140,7 +141,7 @@ require ( golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/Knetic/govaluate.v3 v3.0.0 gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect @@ -152,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 1d161da4d..9b23c21e5 100644 --- a/go.sum +++ b/go.sum @@ -74,16 +74,18 @@ github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeM github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/dot v1.6.4 h1:cG9ycT67d9Yw22G+mAb4XiuUz6E6H1S0zePp/5Cwe/c= +github.com/emicklei/dot v1.6.4/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/ethereum/c-kzg-4844 v1.0.2 h1:8tV84BCEiPeOkiVgW9mpYBeBUir2bkCNVqxPwwVeO+s= github.com/ethereum/c-kzg-4844 v1.0.2/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ethpandaops/ethwallclock v0.3.0 h1:xF5fwtBf+bHFHZKBnwiPFEuelW3sMM7SD3ZNFq1lJY4= github.com/ethpandaops/ethwallclock v0.3.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= -github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= +github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -245,8 +247,8 @@ github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhd github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -278,8 +280,9 @@ github.com/mashingan/smapping v0.1.19 h1:SsEtuPn2UcM1croIupPtGLgWgpYRuS0rSQMvKD9 github.com/mashingan/smapping v0.1.19/go.mod h1:FjfiwFxGOuNxL/OT1WcrNAwTPx0YJeg5JiXwBB1nyig= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -346,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd h1:xty/DNk1Y9vwj2Enx/D/eFBoSL0zf4V6wkqCRHsmliY= -github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= +github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 h1:cKV3IQs3pPhKbcxTGftrKvKjM+QWH+yIeJq7oXFCej8= +github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -450,8 +453,6 @@ github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZ github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= -github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= -github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10/go.mod h1:x/Pa0FF5Te9kdrlZKJK82YmAkvL8+f989USgz6Jiw7M= github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= @@ -600,8 +601,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= From 3c911bedb7b258708ac435ab78828d35d7777a45 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 5 Feb 2025 13:24:24 +0100 Subject: [PATCH 04/99] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab4fc2540..3fe8724cb 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 9b23c21e5..b53bb516f 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 h1:cKV3IQs3pPhKbcxTGftrKvKjM+QWH+yIeJq7oXFCej8= -github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= +github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 0167d8b0034415b9a27caf691e5f8a9eb632d234 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 6 Feb 2025 00:44:31 +0100 Subject: [PATCH 05/99] subscribe to execution_payload events & load execution payload envelopes --- clients/consensus/client.go | 65 ++++++------ clients/consensus/clientlogic.go | 18 +++- clients/consensus/rpc/beaconapi.go | 17 ++++ clients/consensus/rpc/beaconstream.go | 34 ++++++- go.mod | 2 +- go.sum | 4 +- indexer/beacon/block.go | 140 +++++++++++++++++++++----- indexer/beacon/client.go | 53 +++++++++- indexer/beacon/requests.go | 17 ++++ 9 files changed, 285 insertions(+), 65 deletions(-) diff --git a/clients/consensus/client.go b/clients/consensus/client.go index c91131de6..ef4d55236 100644 --- a/clients/consensus/client.go +++ b/clients/consensus/client.go @@ -22,36 +22,37 @@ type ClientConfig struct { } type Client struct { - pool *Pool - clientIdx uint16 - endpointConfig *ClientConfig - clientCtx context.Context - clientCtxCancel context.CancelFunc - rpcClient *rpc.BeaconClient - logger *logrus.Entry - isOnline bool - isSyncing bool - isOptimistic bool - versionStr string - nodeIdentity *rpc.NodeIdentity - clientType ClientType - lastEvent time.Time - retryCounter uint64 - lastError error - headMutex sync.RWMutex - headRoot phase0.Root - headSlot phase0.Slot - justifiedRoot phase0.Root - justifiedEpoch phase0.Epoch - finalizedRoot phase0.Root - finalizedEpoch phase0.Epoch - lastFinalityUpdateEpoch phase0.Epoch - lastPeerUpdateEpoch phase0.Epoch - lastSyncUpdateEpoch phase0.Epoch - peers []*v1.Peer - blockDispatcher Dispatcher[*v1.BlockEvent] - headDispatcher Dispatcher[*v1.HeadEvent] - checkpointDispatcher Dispatcher[*v1.Finality] + pool *Pool + clientIdx uint16 + endpointConfig *ClientConfig + clientCtx context.Context + clientCtxCancel context.CancelFunc + rpcClient *rpc.BeaconClient + logger *logrus.Entry + isOnline bool + isSyncing bool + isOptimistic bool + versionStr string + nodeIdentity *rpc.NodeIdentity + clientType ClientType + lastEvent time.Time + retryCounter uint64 + lastError error + headMutex sync.RWMutex + headRoot phase0.Root + headSlot phase0.Slot + justifiedRoot phase0.Root + justifiedEpoch phase0.Epoch + finalizedRoot phase0.Root + finalizedEpoch phase0.Epoch + lastFinalityUpdateEpoch phase0.Epoch + lastPeerUpdateEpoch phase0.Epoch + lastSyncUpdateEpoch phase0.Epoch + peers []*v1.Peer + blockDispatcher Dispatcher[*v1.BlockEvent] + headDispatcher Dispatcher[*v1.HeadEvent] + checkpointDispatcher Dispatcher[*v1.Finality] + executionPayloadDispatcher Dispatcher[*v1.ExecutionPayloadEvent] } func (pool *Pool) newPoolClient(clientIdx uint16, endpoint *ClientConfig) (*Client, error) { @@ -96,6 +97,10 @@ func (client *Client) SubscribeFinalizedEvent(capacity int) *Subscription[*v1.Fi return client.checkpointDispatcher.Subscribe(capacity, false) } +func (client *Client) SubscribeExecutionPayloadEvent(capacity int, blocking bool) *Subscription[*v1.ExecutionPayloadEvent] { + return client.executionPayloadDispatcher.Subscribe(capacity, blocking) +} + func (client *Client) GetPool() *Pool { return client.pool } diff --git a/clients/consensus/clientlogic.go b/clients/consensus/clientlogic.go index 09a9a2853..5a163be76 100644 --- a/clients/consensus/clientlogic.go +++ b/clients/consensus/clientlogic.go @@ -133,7 +133,11 @@ func (client *Client) runClientLogic() error { } // start event stream - blockStream := client.rpcClient.NewBlockStream(client.clientCtx, client.logger, rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent) + blockStream := client.rpcClient.NewBlockStream( + client.clientCtx, + client.logger, + rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent|rpc.StreamExecutionPayloadEvent, + ) defer blockStream.Close() // process events @@ -171,6 +175,12 @@ func (client *Client) runClientLogic() error { if err != nil { client.logger.Warnf("failed processing finalized event: %v", err) } + + case rpc.StreamExecutionPayloadEvent: + err := client.processExecutionPayloadEvent(evt.Data.(*v1.ExecutionPayloadEvent)) + if err != nil { + client.logger.Warnf("failed processing execution payload event: %v", err) + } } client.logger.Tracef("event (%v) processing time: %v ms", evt.Event, time.Since(now).Milliseconds()) @@ -392,3 +402,9 @@ func (client *Client) pollClientHead() error { return nil } + +func (client *Client) processExecutionPayloadEvent(evt *v1.ExecutionPayloadEvent) error { + client.executionPayloadDispatcher.Fire(evt) + + return nil +} diff --git a/clients/consensus/rpc/beaconapi.go b/clients/consensus/rpc/beaconapi.go index a4ef43d07..70bbce3cc 100644 --- a/clients/consensus/rpc/beaconapi.go +++ b/clients/consensus/rpc/beaconapi.go @@ -19,6 +19,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" @@ -406,6 +407,22 @@ func (bc *BeaconClient) GetBlockBodyByBlockroot(ctx context.Context, blockroot p return result.Data, nil } +func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + provider, isProvider := bc.clientSvc.(eth2client.ExecutionPayloadProvider) + if !isProvider { + return nil, fmt.Errorf("get execution payload not supported") + } + + result, err := provider.SignedExecutionPayloadEnvelope(ctx, &api.SignedExecutionPayloadEnvelopeOpts{ + Block: fmt.Sprintf("0x%x", blockroot), + }) + if err != nil { + return nil, err + } + + return result.Data, nil +} + func (bc *BeaconClient) GetState(ctx context.Context, stateRef string) (*spec.VersionedBeaconState, error) { provider, isProvider := bc.clientSvc.(eth2client.BeaconStateProvider) if !isProvider { diff --git a/clients/consensus/rpc/beaconstream.go b/clients/consensus/rpc/beaconstream.go index be6fd92c9..5020b5ccc 100644 --- a/clients/consensus/rpc/beaconstream.go +++ b/clients/consensus/rpc/beaconstream.go @@ -17,9 +17,10 @@ import ( ) const ( - StreamBlockEvent uint16 = 0x01 - StreamHeadEvent uint16 = 0x02 - StreamFinalizedEvent uint16 = 0x04 + StreamBlockEvent uint16 = 0x01 + StreamHeadEvent uint16 = 0x02 + StreamFinalizedEvent uint16 = 0x04 + StreamExecutionPayloadEvent uint16 = 0x08 ) type BeaconStreamEvent struct { @@ -87,6 +88,8 @@ func (bs *BeaconStream) startStream() { bs.processHeadEvent(evt) case "finalized_checkpoint": bs.processFinalizedEvent(evt) + case "execution_payload": + bs.processExecutionPayloadEvent(evt) } case <-stream.Ready: bs.ReadyChan <- &BeaconStreamStatus{ @@ -148,6 +151,16 @@ func (bs *BeaconStream) subscribeStream(endpoint string, events uint16) *eventst topicsCount++ } + if events&StreamExecutionPayloadEvent > 0 { + if topicsCount > 0 { + fmt.Fprintf(&topics, ",") + } + + fmt.Fprintf(&topics, "execution_payload") + + topicsCount++ + } + if topicsCount == 0 { return nil } @@ -225,6 +238,21 @@ func (bs *BeaconStream) processFinalizedEvent(evt eventsource.Event) { } } +func (bs *BeaconStream) processExecutionPayloadEvent(evt eventsource.Event) { + var parsed v1.ExecutionPayloadEvent + + err := json.Unmarshal([]byte(evt.Data()), &parsed) + if err != nil { + bs.logger.Warnf("beacon block stream failed to decode execution_payload event: %v", err) + return + } + + bs.EventChan <- &BeaconStreamEvent{ + Event: StreamExecutionPayloadEvent, + Data: &parsed, + } +} + func getRedactedURL(requrl string) string { var logurl string diff --git a/go.mod b/go.mod index 3fe8724cb..f7cf45157 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b53bb516f..80a84d2e7 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2 h1:oHm1KoDCyQH3/YhGRdwgCAAKIPjxAkhAvc88aoHCAIw= +github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 9b1562ed2..ec38c125d 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -8,6 +8,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -16,26 +17,30 @@ import ( // Block represents a beacon block. type Block struct { - Root phase0.Root - Slot phase0.Slot - dynSsz *dynssz.DynSsz - parentRoot *phase0.Root - dependentRoot *phase0.Root - forkId ForkKey - forkChecked bool - headerMutex sync.Mutex - headerChan chan bool - header *phase0.SignedBeaconBlockHeader - blockMutex sync.Mutex - blockChan chan bool - block *spec.VersionedSignedBeaconBlock - blockIndex *BlockBodyIndex - isInFinalizedDb bool // block is in finalized table (slots) - isInUnfinalizedDb bool // block is in unfinalized table (unfinalized_blocks) - processingStatus dbtypes.UnfinalizedBlockStatus - seenMutex sync.RWMutex - seenMap map[uint16]*Client - processedActivity uint8 + Root phase0.Root + Slot phase0.Slot + dynSsz *dynssz.DynSsz + parentRoot *phase0.Root + dependentRoot *phase0.Root + forkId ForkKey + forkChecked bool + headerMutex sync.Mutex + headerChan chan bool + header *phase0.SignedBeaconBlockHeader + blockMutex sync.Mutex + blockChan chan bool + block *spec.VersionedSignedBeaconBlock + executionPayloadMutex sync.Mutex + executionPayloadChan chan bool + executionPayload *eip7732.SignedExecutionPayloadEnvelope + blockIndex *BlockBodyIndex + isInFinalizedDb bool // block is in finalized table (slots) + isInUnfinalizedDb bool // block is in unfinalized table (unfinalized_blocks) + hasExecutionPayload bool // block has an execution payload (either in cache or db) + processingStatus dbtypes.UnfinalizedBlockStatus + seenMutex sync.RWMutex + seenMap map[uint16]*Client + processedActivity uint8 } // BlockBodyIndex holds important block propoerties that are used as index for cache lookups. @@ -50,12 +55,13 @@ type BlockBodyIndex struct { // newBlock creates a new Block instance. func newBlock(dynSsz *dynssz.DynSsz, root phase0.Root, slot phase0.Slot) *Block { return &Block{ - Root: root, - Slot: slot, - dynSsz: dynSsz, - seenMap: make(map[uint16]*Client), - headerChan: make(chan bool), - blockChan: make(chan bool), + Root: root, + Slot: slot, + dynSsz: dynSsz, + seenMap: make(map[uint16]*Client), + headerChan: make(chan bool), + blockChan: make(chan bool), + executionPayloadChan: make(chan bool), } } @@ -142,6 +148,42 @@ func (block *Block) AwaitBlock(ctx context.Context, timeout time.Duration) *spec return block.block } +// GetExecutionPayload returns the execution payload of this block. +func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelope { + if block.executionPayload != nil { + return block.executionPayload + } + + if block.hasExecutionPayload && block.isInUnfinalizedDb { + /* TODO: add execution payload to unfinalized blocks table + dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + if dbBlock != nil { + blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + if err == nil { + return blockBody + } + } + */ + } + + return nil +} + +// AwaitExecutionPayload waits for the execution payload of this block to be available. +func (block *Block) AwaitExecutionPayload(ctx context.Context, timeout time.Duration) *eip7732.SignedExecutionPayloadEnvelope { + if ctx == nil { + ctx = context.Background() + } + + select { + case <-block.executionPayloadChan: + case <-time.After(timeout): + case <-ctx.Done(): + } + + return block.executionPayload +} + // GetParentRoot returns the parent root of this block. func (block *Block) GetParentRoot() *phase0.Root { if block.parentRoot != nil { @@ -234,6 +276,52 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo return true, nil } +// SetExecutionPayload sets the execution payload of this block. +func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { + block.executionPayload = payload + + if block.executionPayloadChan != nil { + close(block.executionPayloadChan) + block.executionPayloadChan = nil + } +} + +// EnsureExecutionPayload ensures that the execution payload of this block is available. +func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732.SignedExecutionPayloadEnvelope, error)) (bool, error) { + if block.executionPayload != nil { + return false, nil + } + + if block.isInUnfinalizedDb || block.isInFinalizedDb { + return false, nil + } + + block.executionPayloadMutex.Lock() + defer block.executionPayloadMutex.Unlock() + + if block.executionPayload != nil { + return false, nil + } + + payload, err := loadExecutionPayload() + if err != nil { + return false, err + } + + if payload == nil { + return false, nil + } + + block.executionPayload = payload + block.hasExecutionPayload = true + if block.executionPayloadChan != nil { + close(block.executionPayloadChan) + block.executionPayloadChan = nil + } + + return true, nil +} + // setBlockIndex sets the block index of this block. func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock) { blockIndex := &BlockBodyIndex{} diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index c37330513..5720c4f6c 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -10,6 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -30,8 +31,9 @@ type Client struct { archive bool skipValidators bool - blockSubscription *consensus.Subscription[*v1.BlockEvent] - headSubscription *consensus.Subscription[*v1.HeadEvent] + blockSubscription *consensus.Subscription[*v1.BlockEvent] + headSubscription *consensus.Subscription[*v1.HeadEvent] + executionPayloadSubscription *consensus.Subscription[*v1.ExecutionPayloadEvent] headRoot phase0.Root } @@ -79,6 +81,7 @@ func (c *Client) startIndexing() { // blocking block subscription with a buffer to ensure no blocks are missed c.blockSubscription = c.client.SubscribeBlockEvent(100, true) c.headSubscription = c.client.SubscribeHeadEvent(100, true) + c.executionPayloadSubscription = c.client.SubscribeExecutionPayloadEvent(100, true) go c.startClientLoop() } @@ -177,6 +180,11 @@ func (c *Client) runClientLoop() error { if err != nil { c.logger.Errorf("failed processing head %v (%v): %v", headEvent.Slot, headEvent.Block.String(), err) } + case executionPayloadEvent := <-c.executionPayloadSubscription.Channel(): + err := c.processExecutionPayloadEvent(executionPayloadEvent) + if err != nil { + c.logger.Errorf("failed processing execution payload %v (%v): %v", executionPayloadEvent.Slot, executionPayloadEvent.BlockRoot.String(), err) + } } } @@ -513,3 +521,44 @@ func (c *Client) backfillParentBlocks(headBlock *Block) error { } return nil } + +// processExecutionPayloadEvent processes an execution payload event from the event stream. +func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.ExecutionPayloadEvent) error { + if c.client.GetStatus() != consensus.ClientStatusOnline && c.client.GetStatus() != consensus.ClientStatusOptimistic { + // client is not ready, skip + return nil + } + + chainState := c.client.GetPool().GetChainState() + finalizedSlot := chainState.GetFinalizedSlot() + + var block *Block + + if executionPayloadEvent.Slot < finalizedSlot { + // block is in finalized epoch + // known block or a new orphaned block + + // don't add to cache, process this block right after loading the details + block = newBlock(c.indexer.dynSsz, executionPayloadEvent.BlockRoot, executionPayloadEvent.Slot) + + dbBlockHead := db.GetBlockHeadByRoot(executionPayloadEvent.BlockRoot[:]) + if dbBlockHead != nil { + block.isInFinalizedDb = true + block.parentRoot = (*phase0.Root)(dbBlockHead.ParentRoot) + } + + } else { + block = c.indexer.blockCache.getBlockByRoot(executionPayloadEvent.BlockRoot) + } + + if block == nil { + c.logger.Warnf("execution payload event for unknown block %v:%v [0x%x]", chainState.EpochOfSlot(executionPayloadEvent.Slot), executionPayloadEvent.Slot, executionPayloadEvent.BlockRoot) + return nil + } + + _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) + }) + + return err +} diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index df6ec6fb3..585be13cf 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -6,6 +6,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -18,6 +19,9 @@ const beaconBodyRequestTimeout time.Duration = 30 * time.Second // BeaconStateRequestTimeout is the timeout duration for beacon state requests. const beaconStateRequestTimeout time.Duration = 600 * time.Second +// ExecutionPayloadRequestTimeout is the timeout duration for execution payload requests. +const executionPayloadRequestTimeout time.Duration = 30 * time.Second + const beaconStateRetryCount = 10 // LoadBeaconHeader loads the block header from the client. @@ -75,3 +79,16 @@ func LoadBeaconState(ctx context.Context, client *Client, root phase0.Root) (*sp return resState, nil } + +// LoadExecutionPayload loads the execution payload from the client. +func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + ctx, cancel := context.WithTimeout(ctx, executionPayloadRequestTimeout) + defer cancel() + + payload, err := client.client.GetRPCClient().GetExecutionPayloadByBlockroot(ctx, root) + if err != nil { + return nil, err + } + + return payload, nil +} From edd68ac381be1773072d13a95346510ff9e452e8 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 6 Feb 2025 20:24:09 +0100 Subject: [PATCH 06/99] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3fe8724cb..2ffed078c 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b53bb516f..711578891 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf h1:HaW86VhMTHp3I2C21Z+mkEEkwtaDAwj3iXopobENlUA= +github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From f820feb2b7c37885dc102d0bdf275d83451189e4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sat, 8 Feb 2025 23:59:54 +0100 Subject: [PATCH 07/99] epbs payload handling & db persistence --- clients/consensus/chainstate.go | 8 ++ db/orphaned_blocks.go | 12 +- .../pgsql/20250208225212_epbs-payload.sql | 16 +++ .../sqlite/20250208225212_epbs-payload.sql | 16 +++ db/unfinalized_blocks.go | 47 +++++--- dbtypes/dbtypes.go | 30 ++--- handlers/slot.go | 34 +++++- indexer/beacon/block.go | 61 +++++++--- indexer/beacon/block_helper.go | 76 ++++++++++++ indexer/beacon/client.go | 27 ++++- indexer/beacon/finalization.go | 14 +++ indexer/beacon/indexer.go | 16 ++- indexer/beacon/indexer_getter.go | 8 ++ indexer/beacon/pruning.go | 3 +- indexer/beacon/synchronizer.go | 23 +++- indexer/beacon/writedb.go | 110 +++++++++++++----- services/chainservice_blocks.go | 48 ++++++++ 17 files changed, 460 insertions(+), 89 deletions(-) create mode 100644 db/schema/pgsql/20250208225212_epbs-payload.sql create mode 100644 db/schema/sqlite/20250208225212_epbs-payload.sql diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index abc240a40..c0b403110 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -265,3 +265,11 @@ func (cs *ChainState) GetValidatorChurnLimit(validatorCount uint64) uint64 { return adaptable } + +func (cs *ChainState) IsEip7732Enabled(epoch phase0.Epoch) bool { + if cs.specs == nil { + return false + } + + return cs.specs.Eip7732ForkEpoch != nil && phase0.Epoch(*cs.specs.Eip7732ForkEpoch) <= epoch +} diff --git a/db/orphaned_blocks.go b/db/orphaned_blocks.go index 798b476eb..92f44a95b 100644 --- a/db/orphaned_blocks.go +++ b/db/orphaned_blocks.go @@ -9,15 +9,15 @@ func InsertOrphanedBlock(block *dbtypes.OrphanedBlock, tx *sqlx.Tx) error { _, err := tx.Exec(EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` INSERT INTO orphaned_blocks ( - root, header_ver, header_ssz, block_ver, block_ssz - ) VALUES ($1, $2, $3, $4, $5) + root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz + ) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (root) DO NOTHING`, dbtypes.DBEngineSqlite: ` INSERT OR IGNORE INTO orphaned_blocks ( - root, header_ver, header_ssz, block_ver, block_ssz - ) VALUES ($1, $2, $3, $4, $5)`, + root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, }), - block.Root, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ) + block.Root, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.PayloadVer, block.PayloadSSZ) if err != nil { return err } @@ -27,7 +27,7 @@ func InsertOrphanedBlock(block *dbtypes.OrphanedBlock, tx *sqlx.Tx) error { func GetOrphanedBlock(root []byte) *dbtypes.OrphanedBlock { block := dbtypes.OrphanedBlock{} err := ReaderDb.Get(&block, ` - SELECT root, header_ver, header_ssz, block_ver, block_ssz + SELECT root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz FROM orphaned_blocks WHERE root = $1 `, root) diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql new file mode 100644 index 000000000..4db3ab6c9 --- /dev/null +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin + +ALTER TABLE public."unfinalized_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" bytea NULL; + +ALTER TABLE public."orphaned_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" bytea NULL; + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd \ No newline at end of file diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql new file mode 100644 index 000000000..1b20a8be8 --- /dev/null +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin + +ALTER TABLE "unfinalized_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" BLOB NULL; + +ALTER TABLE "orphaned_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" BLOB NULL; + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd \ No newline at end of file diff --git a/db/unfinalized_blocks.go b/db/unfinalized_blocks.go index 329b5c47b..dcdc23c73 100644 --- a/db/unfinalized_blocks.go +++ b/db/unfinalized_blocks.go @@ -12,15 +12,15 @@ func InsertUnfinalizedBlock(block *dbtypes.UnfinalizedBlock, tx *sqlx.Tx) error _, err := tx.Exec(EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` INSERT INTO unfinalized_blocks ( - root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (root) DO NOTHING`, dbtypes.DBEngineSqlite: ` INSERT OR IGNORE INTO unfinalized_blocks ( - root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, }), - block.Root, block.Slot, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.Status, block.ForkId) + block.Root, block.Slot, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.PayloadVer, block.PayloadSSZ, block.Status, block.ForkId) if err != nil { return err } @@ -77,6 +77,14 @@ func UpdateUnfinalizedBlockForkId(roots [][]byte, forkId uint64, tx *sqlx.Tx) er return nil } +func UpdateUnfinalizedBlockPayload(root []byte, payloadVer uint64, payloadSSZ []byte, tx *sqlx.Tx) error { + _, err := tx.Exec(`UPDATE unfinalized_blocks SET payload_ver = $1, payload_ssz = $2 WHERE root = $3`, payloadVer, payloadSSZ, root) + if err != nil { + return err + } + return nil +} + func GetUnfinalizedBlocks(filter *dbtypes.UnfinalizedBlockFilter) []*dbtypes.UnfinalizedBlock { blockRefs := []*dbtypes.UnfinalizedBlock{} @@ -120,7 +128,7 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc var sql strings.Builder args := []any{slot} - fmt.Fprint(&sql, `SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id FROM unfinalized_blocks WHERE slot >= $1`) + fmt.Fprint(&sql, `SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id FROM unfinalized_blocks WHERE slot >= $1`) rows, err := ReaderDb.Query(sql.String(), args...) if err != nil { @@ -130,7 +138,7 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc for rows.Next() { block := dbtypes.UnfinalizedBlock{} - err := rows.Scan(&block.Root, &block.Slot, &block.HeaderVer, &block.HeaderSSZ, &block.BlockVer, &block.BlockSSZ, &block.Status, &block.ForkId) + err := rows.Scan(&block.Root, &block.Slot, &block.HeaderVer, &block.HeaderSSZ, &block.BlockVer, &block.BlockSSZ, &block.PayloadVer, &block.PayloadSSZ, &block.Status, &block.ForkId) if err != nil { logger.Errorf("Error while scanning unfinalized block: %v", err) return err @@ -141,13 +149,26 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc return nil } -func GetUnfinalizedBlock(root []byte) *dbtypes.UnfinalizedBlock { +func GetUnfinalizedBlock(root []byte, withHeader bool, withBody bool, withPayload bool) *dbtypes.UnfinalizedBlock { + var sql strings.Builder + fmt.Fprint(&sql, `SELECT root, slot`) + + if withHeader { + fmt.Fprint(&sql, `, header_ver, header_ssz`) + } + + if withBody { + fmt.Fprint(&sql, `, block_ver, block_ssz`) + } + + if withPayload { + fmt.Fprint(&sql, `, payload_ver, payload_ssz`) + } + + fmt.Fprint(&sql, `FROM unfinalized_blocks WHERE root = $1`) + block := dbtypes.UnfinalizedBlock{} - err := ReaderDb.Get(&block, ` - SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - FROM unfinalized_blocks - WHERE root = $1 - `, root) + err := ReaderDb.Get(&block, sql.String(), root) if err != nil { logger.Errorf("Error while fetching unfinalized block 0x%x: %v", root, err) return nil diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 49c78504f..b38705604 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -73,11 +73,13 @@ type Epoch struct { } type OrphanedBlock struct { - Root []byte `db:"root"` - HeaderVer uint64 `db:"header_ver"` - HeaderSSZ []byte `db:"header_ssz"` - BlockVer uint64 `db:"block_ver"` - BlockSSZ []byte `db:"block_ssz"` + Root []byte `db:"root"` + HeaderVer uint64 `db:"header_ver"` + HeaderSSZ []byte `db:"header_ssz"` + BlockVer uint64 `db:"block_ver"` + BlockSSZ []byte `db:"block_ssz"` + PayloadVer uint64 `db:"payload_ver"` + PayloadSSZ []byte `db:"payload_ssz"` } type SlotAssignment struct { @@ -100,14 +102,16 @@ const ( ) type UnfinalizedBlock struct { - Root []byte `db:"root"` - Slot uint64 `db:"slot"` - HeaderVer uint64 `db:"header_ver"` - HeaderSSZ []byte `db:"header_ssz"` - BlockVer uint64 `db:"block_ver"` - BlockSSZ []byte `db:"block_ssz"` - Status UnfinalizedBlockStatus `db:"status"` - ForkId uint64 `db:"fork_id"` + Root []byte `db:"root"` + Slot uint64 `db:"slot"` + HeaderVer uint64 `db:"header_ver"` + HeaderSSZ []byte `db:"header_ssz"` + BlockVer uint64 `db:"block_ver"` + BlockSSZ []byte `db:"block_ssz"` + PayloadVer uint64 `db:"payload_ver"` + PayloadSSZ []byte `db:"payload_ssz"` + Status UnfinalizedBlockStatus `db:"status"` + ForkId uint64 `db:"fork_id"` } type UnfinalizedEpoch struct { diff --git a/handlers/slot.go b/handlers/slot.go index db9482c0c..26318e269 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -681,6 +681,28 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV BlockNumber: uint64(executionPayload.BlockNumber), } getSlotPageTransactions(pageData, executionPayload.Transactions) + case spec.DataVersionEIP7732: + if blockData.Payload == nil { + break + } + executionPayload := blockData.Payload.Message.Payload + pageData.ExecutionData = &models.SlotPageExecutionData{ + ParentHash: executionPayload.ParentHash[:], + FeeRecipient: executionPayload.FeeRecipient[:], + StateRoot: executionPayload.StateRoot[:], + ReceiptsRoot: executionPayload.ReceiptsRoot[:], + LogsBloom: executionPayload.LogsBloom[:], + Random: executionPayload.PrevRandao[:], + GasLimit: uint64(executionPayload.GasLimit), + GasUsed: uint64(executionPayload.GasUsed), + Timestamp: uint64(executionPayload.Timestamp), + Time: time.Unix(int64(executionPayload.Timestamp), 0), + ExtraData: executionPayload.ExtraData, + BaseFeePerGas: executionPayload.BaseFeePerGas.Uint64(), + BlockHash: executionPayload.BlockHash[:], + BlockNumber: uint64(executionPayload.BlockNumber), + } + getSlotPageTransactions(pageData, executionPayload.Transactions) } } @@ -723,8 +745,16 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } if specs.ElectraForkEpoch != nil && uint64(epoch) >= *specs.ElectraForkEpoch { - requests, err := blockData.Block.ExecutionRequests() - if err == nil && requests != nil { + var requests *electra.ExecutionRequests + if blockData.Block.Version >= spec.DataVersionEIP7732 { + if blockData.Payload != nil { + requests = blockData.Payload.Message.ExecutionRequests + } + } else { + requests, _ = blockData.Block.ExecutionRequests() + } + + if requests != nil { getSlotPageDepositRequests(pageData, requests.Deposits) getSlotPageWithdrawalRequests(pageData, requests.Withdrawals) getSlotPageConsolidationRequests(pageData, requests.Consolidations) diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index ec38c125d..cc8aca8eb 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -121,7 +121,7 @@ func (block *Block) GetBlock() *spec.VersionedSignedBeaconBlock { } if block.isInUnfinalizedDb { - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, true, false) if dbBlock != nil { blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) if err == nil { @@ -155,15 +155,13 @@ func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelop } if block.hasExecutionPayload && block.isInUnfinalizedDb { - /* TODO: add execution payload to unfinalized blocks table - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, false, true) if dbBlock != nil { - blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err == nil { - return blockBody + return payload } } - */ } return nil @@ -235,7 +233,7 @@ func (block *Block) EnsureHeader(loadHeader func() (*phase0.SignedBeaconBlockHea // SetBlock sets the versioned signed beacon block of this block. func (block *Block) SetBlock(body *spec.VersionedSignedBeaconBlock) { - block.setBlockIndex(body) + block.setBlockIndex(body, nil) block.block = body if block.blockChan != nil { @@ -266,7 +264,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo return false, err } - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, nil) block.block = blockBody if block.blockChan != nil { close(block.blockChan) @@ -278,6 +276,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo // SetExecutionPayload sets the execution payload of this block. func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { + block.setBlockIndex(block.block, payload) block.executionPayload = payload if block.executionPayloadChan != nil { @@ -312,6 +311,7 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 return false, nil } + block.setBlockIndex(block.block, payload) block.executionPayload = payload block.hasExecutionPayload = true if block.executionPayloadChan != nil { @@ -323,12 +323,23 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 } // setBlockIndex sets the block index of this block. -func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock) { - blockIndex := &BlockBodyIndex{} - blockIndex.Graffiti, _ = body.Graffiti() - blockIndex.ExecutionExtraData, _ = getBlockExecutionExtraData(body) - blockIndex.ExecutionHash, _ = body.ExecutionBlockHash() - blockIndex.ExecutionNumber, _ = body.ExecutionBlockNumber() +func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload *eip7732.SignedExecutionPayloadEnvelope) { + blockIndex := block.blockIndex + if blockIndex == nil { + blockIndex = &BlockBodyIndex{} + } + + if body != nil { + blockIndex.Graffiti, _ = body.Graffiti() + blockIndex.ExecutionExtraData, _ = getBlockExecutionExtraData(body) + blockIndex.ExecutionHash, _ = body.ExecutionBlockHash() + if execNumber, err := body.ExecutionBlockNumber(); err == nil { + blockIndex.ExecutionNumber = uint64(execNumber) + } + } + if payload != nil { + blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) + } block.blockIndex = blockIndex } @@ -341,7 +352,7 @@ func (block *Block) GetBlockIndex() *BlockBodyIndex { blockBody := block.GetBlock() if blockBody != nil { - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, block.GetExecutionPayload()) } return block.blockIndex @@ -383,13 +394,24 @@ func (block *Block) buildOrphanedBlock(compress bool) (*dbtypes.OrphanedBlock, e return nil, fmt.Errorf("marshal block ssz failed: %v", err) } - return &dbtypes.OrphanedBlock{ + orphanedBlock := &dbtypes.OrphanedBlock{ Root: block.Root[:], HeaderVer: 1, HeaderSSZ: headerSSZ, BlockVer: blockVer, BlockSSZ: blockSSZ, - }, nil + } + + if block.executionPayload != nil { + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) + if err != nil { + return nil, fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + orphanedBlock.PayloadVer = payloadVer + orphanedBlock.PayloadSSZ = payloadSSZ + } + + return orphanedBlock, nil } // unpruneBlockBody retrieves the block body from the database if it is not already present. @@ -398,9 +420,12 @@ func (block *Block) unpruneBlockBody() { return } - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, true, true) if dbBlock != nil { block.block, _ = unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + if len(dbBlock.PayloadSSZ) > 0 { + block.executionPayload, _ = unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + } } } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 34f03b06d..a6dcd3079 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -208,6 +208,82 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. return block, nil } +// marshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. +func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { + if utils.Config.KillSwitch.DisableSSZEncoding { + // SSZ encoding disabled, use json instead + version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) + } else { + // SSZ encoding + version = uint64(spec.DataVersionEIP7732) + ssz, err = dynSsz.MarshalSSZ(payload) + } + + if compress { + ssz = compressBytes(ssz) + version |= compressionFlag + } + + return +} + +// unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. +func unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { + if (version & compressionFlag) != 0 { + // decompress + if d, err := decompressBytes(ssz); err != nil { + return nil, fmt.Errorf("failed to decompress: %v", err) + } else { + ssz = d + version &= ^compressionFlag + } + } + + if (version & jsonVersionFlag) != 0 { + // JSON encoding + return unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version, ssz) + } + + if version != uint64(spec.DataVersionEIP7732) { + return nil, fmt.Errorf("unknown version") + } + + // SSZ encoding + payload := &eip7732.SignedExecutionPayloadEnvelope{} + if err := dynSsz.UnmarshalSSZ(payload, ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed execution payload envelope: %v", err) + } + + return payload, nil +} + +// marshalVersionedSignedExecutionPayloadEnvelopeJson marshals a versioned signed execution payload envelope using JSON encoding. +func marshalVersionedSignedExecutionPayloadEnvelopeJson(payload *eip7732.SignedExecutionPayloadEnvelope) (version uint64, jsonRes []byte, err error) { + version = uint64(spec.DataVersionEIP7732) + jsonRes, err = payload.MarshalJSON() + + version |= jsonVersionFlag + + return +} + +// unmarshalVersionedSignedExecutionPayloadEnvelopeJson unmarshals a versioned signed execution payload envelope using JSON encoding. +func unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { + if version&jsonVersionFlag == 0 { + return nil, fmt.Errorf("no json encoding") + } + + if version-jsonVersionFlag != uint64(spec.DataVersionEIP7732) { + return nil, fmt.Errorf("unknown version") + } + + payload := &eip7732.SignedExecutionPayloadEnvelope{} + if err := payload.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed execution payload envelope: %v", err) + } + return payload, nil +} + // getBlockExecutionExtraData returns the extra data from the execution payload of a versioned signed beacon block. func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, error) { switch v.Version { diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 5720c4f6c..ee7685250 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -556,9 +556,32 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio return nil } - _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) }) + if err != nil { + return err + } - return err + if newPayload { + // write payload to db + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + if err != nil { + return fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + + err = db.RunDBTransaction(func(tx *sqlx.Tx) error { + err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + } + + return nil } diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 9aae391c6..7f56668be 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -8,6 +8,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -145,6 +146,19 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R if block.block == nil { return true, fmt.Errorf("missing block body for canonical block %v (%v)", block.Slot, block.Root.String()) } + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + if _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + return LoadExecutionPayload(client.getContext(), client, block.Root) + }); err != nil { + client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) + } + + if block.executionPayload == nil && !lastTry { + return false, fmt.Errorf("missing execution payload for canonical block %v (%v)", block.Slot, block.Root.String()) + } + } + canonicalBlocks = append(canonicalBlocks, block) } else { if block.block == nil { diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index a66bf6327..6b580c00f 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -303,6 +303,7 @@ func (indexer *Indexer) StartIndexer() { // restore unfinalized blocks from db restoredBlockCount := 0 restoredBodyCount := 0 + restoredPayloadCount := 0 t1 = time.Now() err = db.StreamUnfinalizedBlocks(uint64(finalizedSlot), func(dbBlock *dbtypes.UnfinalizedBlock) { @@ -334,10 +335,23 @@ func (indexer *Indexer) StartIndexer() { block.SetBlock(blockBody) restoredBodyCount++ } else { - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, nil) block.isInFinalizedDb = true } + if len(dbBlock.PayloadSSZ) > 0 { + blockPayload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + if err != nil { + indexer.logger.Warnf("could not restore unfinalized block payload %v [%x] from db: %v", dbBlock.Slot, dbBlock.Root, err) + } else if block.processingStatus == 0 { + block.SetExecutionPayload(blockPayload) + restoredPayloadCount++ + } else { + block.setBlockIndex(blockBody, blockPayload) + block.isInFinalizedDb = true + } + } + indexer.blockCache.addBlockToExecBlockMap(block) blockFork := indexer.forkCache.getForkById(block.forkId) diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index c595e9d02..a0ce297ea 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -192,6 +192,14 @@ func (indexer *Indexer) GetOrphanedBlockByRoot(blockRoot phase0.Root) (*Block, e block.SetHeader(header) block.SetBlock(blockBody) + if len(orphanedBlock.PayloadSSZ) > 0 { + payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) + if err != nil { + return nil, fmt.Errorf("could not restore orphaned block payload %v [%x] from db: %v", header.Message.Slot, orphanedBlock.Root, err) + } + block.SetExecutionPayload(payload) + } + return block, nil } diff --git a/indexer/beacon/pruning.go b/indexer/beacon/pruning.go index 65033d7ff..8209ba6a3 100644 --- a/indexer/beacon/pruning.go +++ b/indexer/beacon/pruning.go @@ -257,8 +257,9 @@ func (indexer *Indexer) processEpochPruning(pruneEpoch phase0.Epoch) (uint64, ui for _, block := range pruningBlocks { block.isInFinalizedDb = true block.processingStatus = dbtypes.UnfinalizedBlockStatusPruned - block.setBlockIndex(block.block) + block.setBlockIndex(block.block, block.executionPayload) block.block = nil + block.executionPayload = nil } // clean up epoch stats cache diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 6d6e7e5ee..a62fdadca 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -9,6 +9,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -262,11 +263,17 @@ func (sync *synchronizer) loadBlockHeader(client *Client, slot phase0.Slot) (*ph } func (sync *synchronizer) loadBlockBody(client *Client, root phase0.Root) (*spec.VersionedSignedBeaconBlock, error) { - ctx, cancel := context.WithTimeout(sync.syncCtx, beaconHeaderRequestTimeout) + ctx, cancel := context.WithTimeout(sync.syncCtx, beaconBodyRequestTimeout) defer cancel() return LoadBeaconBlock(ctx, client, root) } +func (sync *synchronizer) loadBlockPayload(client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + ctx, cancel := context.WithTimeout(sync.syncCtx, executionPayloadRequestTimeout) + defer cancel() + return LoadExecutionPayload(ctx, client, root) +} + func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry bool) (bool, error) { if !utils.Config.Indexer.ResyncForceUpdate && db.IsEpochSynchronized(uint64(syncEpoch)) { return true, nil @@ -312,6 +319,20 @@ func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, last block.SetBlock(blockBody) } + if slot > 0 && chainState.IsEip7732Enabled(chainState.EpochOfSlot(slot)) { + blockPayload, err := sync.loadBlockPayload(client, phase0.Root(blockRoot)) + if err != nil && !lastTry { + return false, fmt.Errorf("error fetching slot %v execution payload: %v", slot, err) + } + if blockPayload == nil && !lastTry { + return false, fmt.Errorf("error fetching slot %v execution payload: not found", slot) + } + + if blockPayload != nil { + block.SetExecutionPayload(blockPayload) + } + } + sync.cachedBlocks[slot] = block } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 3742f4aea..04dd8fed7 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -4,6 +4,8 @@ import ( "fmt" "math" + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" @@ -231,6 +233,8 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override epochStatsValues = epochStats.GetValues(true) } + chainState := dbw.indexer.consensusPool.GetChainState() + graffiti, _ := blockBody.Graffiti() attestations, _ := blockBody.Attestations() deposits, _ := blockBody.Deposits() @@ -239,17 +243,33 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - executionBlockNumber, _ := blockBody.ExecutionBlockNumber() executionBlockHash, _ := blockBody.ExecutionBlockHash() - executionExtraData, _ := getBlockExecutionExtraData(blockBody) - executionTransactions, _ := blockBody.ExecutionTransactions() - executionWithdrawals, _ := blockBody.Withdrawals() + var executionBlockNumber uint64 + var executionExtraData []byte + var executionTransactions []bellatrix.Transaction + var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - executionRequests, _ := blockBody.ExecutionRequests() - if executionRequests != nil { - depositRequests = executionRequests.Deposits + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + executionBlockNumber = blockPayload.Message.Payload.BlockNumber + executionExtraData = blockPayload.Message.Payload.ExtraData + executionTransactions = blockPayload.Message.Payload.Transactions + executionWithdrawals = blockPayload.Message.Payload.Withdrawals + depositRequests = blockPayload.Message.ExecutionRequests.Deposits + } + } else { + executionBlockNumber, _ = blockBody.ExecutionBlockNumber() + + executionExtraData, _ = getBlockExecutionExtraData(blockBody) + executionTransactions, _ = blockBody.ExecutionTransactions() + executionWithdrawals, _ = blockBody.Withdrawals() + executionRequests, _ := blockBody.ExecutionRequests() + if executionRequests != nil { + depositRequests = executionRequests.Deposits + } } dbBlock := dbtypes.Slot{ @@ -495,14 +515,22 @@ func (dbw *dbWriter) persistBlockDepositRequests(tx *sqlx.Tx, block *Block, orph } func (dbw *dbWriter) buildDbDepositRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.Deposit { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() } deposits := requests.Deposits @@ -685,14 +713,25 @@ func (dbw *dbWriter) persistBlockConsolidationRequests(tx *sqlx.Tx, block *Block } func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.ConsolidationRequest { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + var blockNumber uint64 + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + blockNumber = payload.Message.Payload.BlockNumber + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() + blockNumber, _ = blockBody.ExecutionBlockNumber() } consolidations := requests.Consolidations @@ -701,8 +740,6 @@ func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, o return []*dbtypes.ConsolidationRequest{} } - blockNumber, _ := blockBody.ExecutionBlockNumber() - dbConsolidations := make([]*dbtypes.ConsolidationRequest, len(consolidations)) for idx, consolidation := range consolidations { dbConsolidation := &dbtypes.ConsolidationRequest{ @@ -749,14 +786,25 @@ func (dbw *dbWriter) persistBlockWithdrawalRequests(tx *sqlx.Tx, block *Block, o } func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.WithdrawalRequest { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + var blockNumber uint64 + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + blockNumber = payload.Message.Payload.BlockNumber + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() + blockNumber, _ = blockBody.ExecutionBlockNumber() } withdrawalRequests := requests.Withdrawals @@ -765,8 +813,6 @@ func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, over return []*dbtypes.WithdrawalRequest{} } - blockNumber, _ := blockBody.ExecutionBlockNumber() - dbWithdrawalRequests := make([]*dbtypes.WithdrawalRequest, len(withdrawalRequests)) for idx, withdrawalRequest := range withdrawalRequests { dbWithdrawalRequest := &dbtypes.WithdrawalRequest{ diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 3dc8197bd..3c58b6690 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -9,6 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" @@ -21,6 +22,7 @@ type CombinedBlockResponse struct { Root phase0.Root Header *phase0.SignedBeaconBlockHeader Block *spec.VersionedSignedBeaconBlock + Payload *eip7732.SignedExecutionPayloadEnvelope Orphaned bool } @@ -67,6 +69,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot Root: blockInfo.Root, Header: blockInfo.GetHeader(), Block: blockInfo.GetBlock(), + Payload: blockInfo.GetExecutionPayload(), Orphaned: !bs.beaconIndexer.IsCanonicalBlock(blockInfo, nil), } } else if blockInfo, err := bs.beaconIndexer.GetOrphanedBlockByRoot(blockroot); blockInfo != nil || err != nil { @@ -77,6 +80,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot Root: blockInfo.Root, Header: blockInfo.GetHeader(), Block: blockInfo.GetBlock(), + Payload: blockInfo.GetExecutionPayload(), Orphaned: true, } } else { @@ -125,10 +129,32 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot if err != nil || block == nil { return nil, err } + + var payload *eip7732.SignedExecutionPayloadEnvelope + if block.Version >= spec.DataVersionEIP7732 { + for retry := headRetry; retry < headRetry+3; retry++ { + client := clients[headRetry%len(clients)] + payload, err = beacon.LoadExecutionPayload(ctx, client, blockroot) + if payload != nil { + break + } else if err != nil { + log := logrus.WithError(err) + if client != nil { + log = log.WithField("client", client.GetClient().GetName()) + } + log.Warnf("Error loading block payload for root 0x%x", blockroot) + } + } + if err != nil || payload == nil { + return nil, err + } + } + result = &CombinedBlockResponse{ Root: blockroot, Header: header, Block: block, + Payload: payload, Orphaned: false, } } @@ -160,6 +186,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl Root: cachedBlock.Root, Header: cachedBlock.GetHeader(), Block: cachedBlock.GetBlock(), + Payload: cachedBlock.GetExecutionPayload(), Orphaned: isOrphaned, } } else { @@ -210,10 +237,31 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl return nil, err } + var payload *eip7732.SignedExecutionPayloadEnvelope + if block.Version >= spec.DataVersionEIP7732 { + for retry := headRetry; retry < headRetry+3; retry++ { + client := clients[headRetry%len(clients)] + payload, err = beacon.LoadExecutionPayload(ctx, client, blockRoot) + if payload != nil { + break + } else if err != nil { + log := logrus.WithError(err) + if client != nil { + log = log.WithField("client", client.GetClient().GetName()) + } + log.Warnf("Error loading block payload for root 0x%x", blockRoot) + } + } + if err != nil || payload == nil { + return nil, err + } + } + result = &CombinedBlockResponse{ Root: blockRoot, Header: header, Block: block, + Payload: payload, Orphaned: orphaned, } } From 30a4d6dc118f34da1e813ef08ad1888fd44d608d Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 00:15:04 +0100 Subject: [PATCH 08/99] fix schema upgrade sql for sqlite --- db/schema/sqlite/20250208225212_epbs-payload.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 1b20a8be8..5cb7841e5 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -1,13 +1,11 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE "unfinalized_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" BLOB NULL; +ALTER TABLE "unfinalized_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; +ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; -ALTER TABLE "orphaned_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" BLOB NULL; +ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; +ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; -- +goose StatementEnd -- +goose Down From efb9454417bb472d7efd2374e98c81244c651633 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 00:34:43 +0100 Subject: [PATCH 09/99] various small fixes for epbs implementation --- go.mod | 2 +- go.sum | 4 ++-- indexer/beacon/block.go | 3 ++- indexer/beacon/indexer.go | 2 +- indexer/beacon/writedb.go | 12 ++++++++++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2ffed078c..bf9a07ec1 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 711578891..7dda2cd21 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf h1:HaW86VhMTHp3I2C21Z+mkEEkwtaDAwj3iXopobENlUA= -github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae h1:yTGoTjLjdtmYdZ4lntYD3rAEKglSlqZEALMK5NcGfig= +github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index cc8aca8eb..b113c19e7 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -278,6 +278,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { block.setBlockIndex(block.block, payload) block.executionPayload = payload + block.hasExecutionPayload = true if block.executionPayloadChan != nil { close(block.executionPayloadChan) @@ -291,7 +292,7 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 return false, nil } - if block.isInUnfinalizedDb || block.isInFinalizedDb { + if block.hasExecutionPayload { return false, nil } diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 6b580c00f..e82c7129a 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -348,7 +348,7 @@ func (indexer *Indexer) StartIndexer() { restoredPayloadCount++ } else { block.setBlockIndex(blockBody, blockPayload) - block.isInFinalizedDb = true + block.hasExecutionPayload = true } } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 04dd8fed7..318b99498 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -533,6 +533,10 @@ func (dbw *dbWriter) buildDbDepositRequests(block *Block, orphaned bool, overrid requests, _ = blockBody.ExecutionRequests() } + if requests == nil { + return []*dbtypes.Deposit{} + } + deposits := requests.Deposits dbDeposits := make([]*dbtypes.Deposit, len(deposits)) @@ -734,6 +738,10 @@ func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, o blockNumber, _ = blockBody.ExecutionBlockNumber() } + if requests == nil { + return []*dbtypes.ConsolidationRequest{} + } + consolidations := requests.Consolidations if len(consolidations) == 0 { @@ -807,6 +815,10 @@ func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, over blockNumber, _ = blockBody.ExecutionBlockNumber() } + if requests == nil { + return []*dbtypes.WithdrawalRequest{} + } + withdrawalRequests := requests.Withdrawals if len(withdrawalRequests) == 0 { From 4cc1d38e69e27ed33d1b64794f44526a3b7d6966 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 01:35:14 +0100 Subject: [PATCH 10/99] fixes for epbs payload loading --- go.mod | 2 +- go.sum | 4 +-- indexer/beacon/client.go | 54 ++++++++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index bf9a07ec1..7da64df96 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 7dda2cd21..064b0ee7f 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae h1:yTGoTjLjdtmYdZ4lntYD3rAEKglSlqZEALMK5NcGfig= -github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff h1:8JGDEupV2yBL9fMWfjY9p54W51bfIKC7u39BXHeDxhY= +github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index ee7685250..4f25f5ab6 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -146,7 +146,7 @@ func (c *Client) runClientLoop() error { c.headRoot = headRoot - headBlock, isNew, processingTimes, err := c.processBlock(headSlot, headRoot, nil) + headBlock, isNew, processingTimes, err := c.processBlock(headSlot, headRoot, nil, true) if err != nil { return fmt.Errorf("failed processing head block: %v", err) } @@ -294,7 +294,7 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { // processStreamBlock processes a block received from the stream (either via block or head events). func (c *Client) processStreamBlock(slot phase0.Slot, root phase0.Root) (*Block, error) { - block, isNew, processingTimes, err := c.processBlock(slot, root, nil) + block, isNew, processingTimes, err := c.processBlock(slot, root, nil, false) if err != nil { return nil, err } @@ -348,7 +348,7 @@ func (c *Client) processReorg(oldHead *Block, newHead *Block) error { } // processBlock processes a block (from stream & polling). -func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0.SignedBeaconBlockHeader) (block *Block, isNew bool, processingTimes []time.Duration, err error) { +func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0.SignedBeaconBlockHeader, loadPayload bool) (block *Block, isNew bool, processingTimes []time.Duration, err error) { chainState := c.client.GetPool().GetChainState() finalizedSlot := chainState.GetFinalizedSlot() processingTimes = make([]time.Duration, 3) @@ -387,7 +387,6 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 } isNew, err = block.EnsureBlock(func() (*spec.VersionedSignedBeaconBlock, error) { - t1 := time.Now() defer func() { processingTimes[0] += time.Since(t1) @@ -399,6 +398,25 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 return } + if loadPayload { + newPayload, _ := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + t1 := time.Now() + defer func() { + processingTimes[0] += time.Since(t1) + }() + + return LoadExecutionPayload(c.getContext(), c, root) + }) + + if !isNew && newPayload { + // write payload to db + err = c.persistExecutionPayload(block) + if err != nil { + return + } + } + } + if slot >= finalizedSlot && isNew { c.indexer.blockCache.addBlockToParentMap(block) c.indexer.blockCache.addBlockToExecBlockMap(block) @@ -494,7 +512,7 @@ func (c *Client) backfillParentBlocks(headBlock *Block) error { if parentBlock == nil { var err error - parentBlock, isNewBlock, processingTimes, err = c.processBlock(parentSlot, parentRoot, parentHead) + parentBlock, isNewBlock, processingTimes, err = c.processBlock(parentSlot, parentRoot, parentHead, true) if err != nil { return fmt.Errorf("could not process block [0x%x]: %v", parentRoot, err) } @@ -565,23 +583,27 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio if newPayload { // write payload to db - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + err = c.persistExecutionPayload(block) if err != nil { - return fmt.Errorf("marshal execution payload ssz failed: %v", err) + return err } + } - err = db.RunDBTransaction(func(tx *sqlx.Tx) error { - err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) - if err != nil { - return err - } + return nil +} - return nil - }) +func (c *Client) persistExecutionPayload(block *Block) error { + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + if err != nil { + return fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + + return db.RunDBTransaction(func(tx *sqlx.Tx) error { + err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) if err != nil { return err } - } - return nil + return nil + }) } From 804d4b8237f30f9ce181331617860e5ce89de7e8 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 16:07:38 +0100 Subject: [PATCH 11/99] fetch execution payloads and show execution payload status on UI --- db/epochs.go | 15 +++++------ .../pgsql/20250208225212_epbs-payload.sql | 13 ++++++++++ .../sqlite/20250208225212_epbs-payload.sql | 8 ++++++ db/slots.go | 25 ++++++++++--------- db/unfinalized_epochs.go | 20 +++++++++------ dbtypes/dbtypes.go | 3 +++ handlers/epoch.go | 1 + handlers/index.go | 9 +++++-- handlers/slots.go | 4 ++- handlers/slots_filtered.go | 6 +++-- handlers/validator_slots.go | 6 +++-- indexer/beacon/finalization.go | 4 --- indexer/beacon/synchronizer.go | 3 --- indexer/beacon/writedb.go | 25 +++++++++++++++---- static/css/layout.css | 10 ++++++++ templates/epoch/epoch.html | 6 ++--- templates/index/recentBlocks.html | 12 ++++----- templates/index/recentSlots.html | 12 ++++----- templates/slots/slots.html | 6 ++--- templates/slots_filtered/slots_filtered.html | 6 ++--- templates/validator_slots/slots.html | 10 ++++---- types/models/epoch.go | 1 + types/models/indexPage.go | 2 ++ types/models/slots.go | 1 + types/models/slots_filtered.go | 1 + types/models/validator_slots.go | 1 + 26 files changed, 139 insertions(+), 71 deletions(-) diff --git a/db/epochs.go b/db/epochs.go index cf3a1d6f6..238851925 100644 --- a/db/epochs.go +++ b/db/epochs.go @@ -11,8 +11,8 @@ func InsertEpoch(epoch *dbtypes.Epoch, tx *sqlx.Tx) error { INSERT INTO epochs ( epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (epoch) DO UPDATE SET validator_count = excluded.validator_count, validator_balance = excluded.validator_balance, @@ -31,17 +31,18 @@ func InsertEpoch(epoch *dbtypes.Epoch, tx *sqlx.Tx) error { proposer_slashing_count = excluded.proposer_slashing_count, bls_change_count = excluded.bls_change_count, eth_transaction_count = excluded.eth_transaction_count, - sync_participation = excluded.sync_participation`, + sync_participation = excluded.sync_participation, + payload_count = excluded.payload_count`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO epochs ( epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`, + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)`, }), epoch.Epoch, epoch.ValidatorCount, epoch.ValidatorBalance, epoch.Eligible, epoch.VotedTarget, epoch.VotedHead, epoch.VotedTotal, epoch.BlockCount, epoch.OrphanedCount, epoch.AttestationCount, epoch.DepositCount, epoch.ExitCount, epoch.WithdrawCount, epoch.WithdrawAmount, epoch.AttesterSlashingCount, epoch.ProposerSlashingCount, - epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation) + epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation, epoch.PayloadCount) if err != nil { return err } @@ -63,7 +64,7 @@ func GetEpochs(firstEpoch uint64, limit uint32) []*dbtypes.Epoch { SELECT epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count FROM epochs WHERE epoch <= $1 ORDER BY epoch DESC diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql index 4db3ab6c9..527e5645b 100644 --- a/db/schema/pgsql/20250208225212_epbs-payload.sql +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -9,6 +9,19 @@ ALTER TABLE public."orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0, "payload_ssz" bytea NULL; +ALTER TABLE public."slots" ADD + "has_payload" boolean NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" + ON public."slots" + ("has_payload" ASC NULLS LAST); + +ALTER TABLE public."epochs" ADD + "payload_count" int NOT NULL DEFAULT 0; + +ALTER TABLE public."unfinalized_epochs" ADD + "payload_count" int NOT NULL DEFAULT 0; + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 5cb7841e5..9bc45c060 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -7,6 +7,14 @@ ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; +ALTER TABLE "slots" ADD "has_payload" boolean NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" ON "slots" ("has_payload" ASC); + +ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; + +ALTER TABLE "unfinalized_epochs" ADD "payload_count" int NOT NULL DEFAULT 0; + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/slots.go b/db/slots.go index c96b16880..8786cd5a4 100644 --- a/db/slots.go +++ b/db/slots.go @@ -19,25 +19,26 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, eth_block_extra_text = excluded.eth_block_extra_text, - fork_id = excluded.fork_id`, + fork_id = excluded.fork_id, + has_payload = excluded.has_payload`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO slots ( slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)`, + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, - slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId) + slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.HasPayload) if err != nil { return err } @@ -92,7 +93,7 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -124,7 +125,7 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -143,7 +144,7 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE root = $1 `, root) @@ -169,7 +170,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -212,7 +213,7 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -272,7 +273,7 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) diff --git a/db/unfinalized_epochs.go b/db/unfinalized_epochs.go index cc4230c8d..50be75511 100644 --- a/db/unfinalized_epochs.go +++ b/db/unfinalized_epochs.go @@ -11,8 +11,9 @@ func InsertUnfinalizedEpoch(epoch *dbtypes.UnfinalizedEpoch, tx *sqlx.Tx) error INSERT INTO unfinalized_epochs ( epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) ON CONFLICT (epoch, dependent_root, epoch_head_root) DO UPDATE SET epoch_head_fork_id = excluded.epoch_head_fork_id, validator_count = excluded.validator_count, @@ -32,17 +33,20 @@ func InsertUnfinalizedEpoch(epoch *dbtypes.UnfinalizedEpoch, tx *sqlx.Tx) error proposer_slashing_count = excluded.proposer_slashing_count, bls_change_count = excluded.bls_change_count, eth_transaction_count = excluded.eth_transaction_count, - sync_participation = excluded.sync_participation`, + sync_participation = excluded.sync_participation, + payload_count = excluded.payload_count`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO unfinalized_epochs ( epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22)`, + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)`, }), epoch.Epoch, epoch.DependentRoot, epoch.EpochHeadRoot, epoch.EpochHeadForkId, epoch.ValidatorCount, epoch.ValidatorBalance, epoch.Eligible, epoch.VotedTarget, epoch.VotedHead, epoch.VotedTotal, epoch.BlockCount, epoch.OrphanedCount, epoch.AttestationCount, epoch.DepositCount, epoch.ExitCount, epoch.WithdrawCount, epoch.WithdrawAmount, epoch.AttesterSlashingCount, epoch.ProposerSlashingCount, epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation, + epoch.PayloadCount, ) if err != nil { return err @@ -55,7 +59,8 @@ func StreamUnfinalizedEpochs(epoch uint64, cb func(duty *dbtypes.UnfinalizedEpoc SELECT epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count FROM unfinalized_epochs WHERE epoch >= $1`, epoch) if err != nil { @@ -69,6 +74,7 @@ func StreamUnfinalizedEpochs(epoch uint64, cb func(duty *dbtypes.UnfinalizedEpoc &e.Epoch, &e.DependentRoot, &e.EpochHeadRoot, &e.EpochHeadForkId, &e.ValidatorCount, &e.ValidatorBalance, &e.Eligible, &e.VotedTarget, &e.VotedHead, &e.VotedTotal, &e.BlockCount, &e.OrphanedCount, &e.AttestationCount, &e.DepositCount, &e.ExitCount, &e.WithdrawCount, &e.WithdrawAmount, &e.AttesterSlashingCount, &e.ProposerSlashingCount, &e.BLSChangeCount, &e.EthTransactionCount, &e.SyncParticipation, + &e.PayloadCount, ) if err != nil { logger.Errorf("Error while scanning unfinalized epoch: %v", err) @@ -86,7 +92,7 @@ func GetUnfinalizedEpoch(epoch uint64, headRoot []byte) *dbtypes.UnfinalizedEpoc SELECT epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count FROM unfinalized_epochs WHERE epoch = $1 AND epoch_head_root = $2 `, epoch, headRoot) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index b38705604..a92231e70 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -48,6 +48,7 @@ type Slot struct { EthBlockExtraText string `db:"eth_block_extra_text"` SyncParticipation float32 `db:"sync_participation"` ForkId uint64 `db:"fork_id"` + HasPayload bool `db:"has_payload"` } type Epoch struct { @@ -70,6 +71,7 @@ type Epoch struct { BLSChangeCount uint64 `db:"bls_change_count"` EthTransactionCount uint64 `db:"eth_transaction_count"` SyncParticipation float32 `db:"sync_participation"` + PayloadCount uint64 `db:"payload_count"` } type OrphanedBlock struct { @@ -137,6 +139,7 @@ type UnfinalizedEpoch struct { BLSChangeCount uint64 `db:"bls_change_count"` EthTransactionCount uint64 `db:"eth_transaction_count"` SyncParticipation float32 `db:"sync_participation"` + PayloadCount uint64 `db:"payload_count"` } type Fork struct { diff --git a/handlers/epoch.go b/handlers/epoch.go index a7bb32694..b1c6b28cd 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -173,6 +173,7 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { Ts: chainState.SlotToTime(phase0.Slot(slot)), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Status: uint8(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(phase0.Epoch(epoch)), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), AttestationCount: dbSlot.AttestationCount, diff --git a/handlers/index.go b/handlers/index.go index 24a52efab..27c07e9d6 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -284,13 +284,16 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockC if blockData == nil { continue } + + epoch := chainState.EpochOfSlot(phase0.Slot(blockData.Slot)) blockModel := &models.IndexPageDataBlocks{ - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(blockData.Slot))), + Epoch: uint64(epoch), Slot: blockData.Slot, Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), Proposer: blockData.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), Status: uint64(blockData.Status), + NoPayload: !blockData.HasPayload && chainState.IsEip7732Enabled(epoch), BlockRoot: blockData.Root, } if blockData.EthBlockNumber != nil { @@ -329,11 +332,13 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot pha dbSlot := dbSlots[dbIdx] dbIdx++ + epoch := chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot)) slotData := &models.IndexPageDataSlots{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Status: uint64(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), BlockRoot: dbSlot.Root, diff --git a/handlers/slots.go b/handlers/slots.go index 1de7f1b72..67bb151d4 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -142,12 +142,14 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat dbSlot := dbSlots[dbIdx] dbIdx++ + epoch := chainState.EpochOfSlot(phase0.Slot(slot)) slotData := &models.SlotsPageDataSlot{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Finalized: finalized, Status: uint8(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Synchronized: dbSlot.SyncParticipation != -1, Proposer: dbSlot.Proposer, diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index dcb8ce793..fd17ba992 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -234,12 +234,13 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string break } slot := phase0.Slot(dbBlock.Slot) + epoch := chainState.EpochOfSlot(slot) slotData := &models.SlotsFilteredPageDataSlot{ Slot: uint64(slot), - Epoch: uint64(chainState.EpochOfSlot(slot)), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(slot), - Finalized: finalizedEpoch >= chainState.EpochOfSlot(slot), + Finalized: finalizedEpoch >= epoch, Synchronized: true, Scheduled: slot >= currentSlot, Proposer: dbBlock.Proposer, @@ -265,6 +266,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber } + slotData.NoPayload = !dbBlock.Block.HasPayload && chainState.IsEip7732Enabled(epoch) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index c80a7abd1..dad03dc40 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -112,12 +112,13 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint break } slot := blockAssignment.Slot + epoch := chainState.EpochOfSlot(phase0.Slot(slot)) slotData := &models.ValidatorSlotsPageDataSlot{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), - Finalized: finalizedEpoch >= chainState.EpochOfSlot(phase0.Slot(slot)), + Finalized: finalizedEpoch >= epoch, Status: uint8(0), Proposer: validator, ProposerName: pageData.Name, @@ -139,6 +140,7 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.EthBlockNumber } + slotData.NoPayload = !dbBlock.HasPayload && chainState.IsEip7732Enabled(epoch) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 7f56668be..9428db6c3 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -153,10 +153,6 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R }); err != nil { client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) } - - if block.executionPayload == nil && !lastTry { - return false, fmt.Errorf("missing execution payload for canonical block %v (%v)", block.Slot, block.Root.String()) - } } canonicalBlocks = append(canonicalBlocks, block) diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index a62fdadca..1af16be13 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -324,9 +324,6 @@ func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, last if err != nil && !lastTry { return false, fmt.Errorf("error fetching slot %v execution payload: %v", slot, err) } - if blockPayload == nil && !lastTry { - return false, fmt.Errorf("error fetching slot %v execution payload: not found", slot) - } if blockPayload != nil { block.SetExecutionPayload(blockPayload) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 318b99498..79614be46 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -250,6 +250,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var hasPayload bool if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -259,6 +260,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + hasPayload = true } } else { executionBlockNumber, _ = blockBody.ExecutionBlockNumber() @@ -288,6 +290,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override AttesterSlashingCount: uint64(len(attesterSlashings)), ProposerSlashingCount: uint64(len(proposerSlashings)), BLSChangeCount: uint64(len(blsToExecChanges)), + HasPayload: hasPayload, } if overrideForkId != nil { @@ -394,14 +397,26 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - executionTransactions, _ := blockBody.ExecutionTransactions() - executionWithdrawals, _ := blockBody.Withdrawals() + var executionTransactions []bellatrix.Transaction + var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - executionRequests, _ := blockBody.ExecutionRequests() - if executionRequests != nil { - depositRequests = executionRequests.Deposits + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + dbEpoch.PayloadCount++ + executionTransactions = blockPayload.Message.Payload.Transactions + executionWithdrawals = blockPayload.Message.Payload.Withdrawals + depositRequests = blockPayload.Message.ExecutionRequests.Deposits + } + } else { + executionTransactions, _ = blockBody.ExecutionTransactions() + executionWithdrawals, _ = blockBody.Withdrawals() + executionRequests, _ := blockBody.ExecutionRequests() + if executionRequests != nil { + depositRequests = executionRequests.Deposits + } } dbEpoch.AttestationCount += uint64(len(attestations)) diff --git a/static/css/layout.css b/static/css/layout.css index 8947b1d9f..c10a43314 100644 --- a/static/css/layout.css +++ b/static/css/layout.css @@ -322,3 +322,13 @@ span.validator-label { .table-sm>:not(caption)>*>* { padding: 1px .25rem; } + +.badge.split-warning { + background: linear-gradient( + 90deg, + rgba(255,255,255,0) 0%, + rgba(255,255,255,0) 50%, + rgba(255,193,7,1) 50%, + rgba(255,193,7,1) 100% + ); +} diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html index 3957ba1fa..c400c2bfd 100644 --- a/templates/epoch/epoch.html +++ b/templates/epoch/epoch.html @@ -176,15 +176,15 @@

{{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Orphaned {{ else if not $epoch.Synchronized }} ? {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index ab59debcd..9e78baeb4 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@

Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -74,11 +74,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $block.Ts }} diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index 118619238..3be0f7fff 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -97,11 +97,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Orphaned + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots/slots.html b/templates/slots/slots.html index b847031f3..2a4a6d148 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -92,9 +92,9 @@

Slots

{{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if not $slot.Synchronized }} @@ -102,7 +102,7 @@

Slots

{{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index 7d4ed6b4d..8400972e1 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -183,9 +183,9 @@

Filtered Slots

{{- if eq $slot.Slot 0 }} Genesis {{- else if eq $slot.Status 1 }} - Proposed + Proposed {{- else if eq $slot.Status 2 }} - Orphaned + Missed (Orphaned) {{- else if $slot.Scheduled }} Scheduled {{- else if not $slot.Synchronized }} @@ -193,7 +193,7 @@

Filtered Slots

{{- else if eq $slot.Status 0 }} Missed {{- else }} - Unknown + Unknown {{- end }} {{- end }} diff --git a/templates/validator_slots/slots.html b/templates/validator_slots/slots.html index f8a027311..a89f09d83 100644 --- a/templates/validator_slots/slots.html +++ b/templates/validator_slots/slots.html @@ -71,16 +71,16 @@

Validator {{ format {{ if eq $slot.Slot 0 }} Genesis + {{ else if eq $slot.Status 1 }} + Proposed + {{ else if eq $slot.Status 2 }} + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 0 }} Missed - {{ else if eq $slot.Status 1 }} - Proposed - {{ else if eq $slot.Status 2 }} - Orphaned {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/types/models/epoch.go b/types/models/epoch.go index e7a8d263c..c15be304c 100644 --- a/types/models/epoch.go +++ b/types/models/epoch.go @@ -44,6 +44,7 @@ type EpochPageDataSlot struct { Ts time.Time `json:"ts"` Scheduled bool `json:"scheduled"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` diff --git a/types/models/indexPage.go b/types/models/indexPage.go index c0a64898f..af5d5c4de 100644 --- a/types/models/indexPage.go +++ b/types/models/indexPage.go @@ -65,6 +65,7 @@ type IndexPageDataBlocks struct { Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` Status uint64 `json:"status"` + NoPayload bool `json:"no_payload"` BlockRoot []byte `json:"block_root"` } @@ -76,6 +77,7 @@ type IndexPageDataSlots struct { Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` Status uint64 `json:"status"` + NoPayload bool `json:"no_payload"` BlockRoot []byte `json:"block_root"` ParentRoot []byte `json:"-"` ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` diff --git a/types/models/slots.go b/types/models/slots.go index 075277e37..a3fcf9d41 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -31,6 +31,7 @@ type SlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index b084733da..38fd10d48 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -56,6 +56,7 @@ type SlotsFilteredPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/validator_slots.go b/types/models/validator_slots.go index 22b33e1e3..0f4ade491 100644 --- a/types/models/validator_slots.go +++ b/types/models/validator_slots.go @@ -34,6 +34,7 @@ type ValidatorSlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` From 3c637f9f597a0b2c04e14ec98a154aaabfdc5764 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 11 Feb 2025 14:15:48 +0100 Subject: [PATCH 12/99] fix payload badges on knockout rendered startpage --- templates/index/recentBlocks.html | 6 +++--- templates/index/recentSlots.html | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index 9e78baeb4..51b639fcc 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index 3be0f7fff..bc348763a 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown From 6da622da310c9e52e9a786239c4ae8bb11c2a688 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 11 Feb 2025 14:56:45 +0100 Subject: [PATCH 13/99] show payload header data on slot details page --- handlers/slot.go | 16 ++++++ templates/slot/overview.html | 95 ++++++++++++++++++++++++++++++++---- types/models/slot.go | 18 ++++++- 3 files changed, 118 insertions(+), 11 deletions(-) diff --git a/handlers/slot.go b/handlers/slot.go index 26318e269..9defa093b 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -682,10 +682,26 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } getSlotPageTransactions(pageData, executionPayload.Transactions) case spec.DataVersionEIP7732: + payloadHeader := blockData.Block.EIP7732.Message.Body.SignedExecutionPayloadHeader + pageData.PayloadHeader = &models.SlotPagePayloadHeader{ + PayloadStatus: 0, + ParentBlockHash: payloadHeader.Message.ParentBlockHash[:], + ParentBlockRoot: payloadHeader.Message.ParentBlockRoot[:], + BlockHash: payloadHeader.Message.BlockHash[:], + GasLimit: uint64(payloadHeader.Message.GasLimit), + BuilderIndex: uint64(payloadHeader.Message.BuilderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadHeader.Message.BuilderIndex)), + Slot: uint64(payloadHeader.Message.Slot), + Value: uint64(payloadHeader.Message.Value), + BlobKzgCommitmentsRoot: payloadHeader.Message.BlobKZGCommitmentsRoot[:], + Signature: payloadHeader.Signature[:], + } + if blockData.Payload == nil { break } executionPayload := blockData.Payload.Message.Payload + pageData.PayloadHeader.PayloadStatus = 1 pageData.ExecutionData = &models.SlotPageExecutionData{ ParentHash: executionPayload.ParentHash[:], FeeRecipient: executionPayload.FeeRecipient[:], diff --git a/templates/slot/overview.html b/templates/slot/overview.html index ce631bcb5..38a004dfd 100644 --- a/templates/slot/overview.html +++ b/templates/slot/overview.html @@ -152,30 +152,103 @@ {{ end }} - {{ if .Block.ExecutionData }} + + {{ if .Block.PayloadHeader }} {{ $block := .Block }} - {{ with .Block.ExecutionData }} + {{ with .Block.PayloadHeader }}
-
Execution Payload:
+
Payload Header:
+
-
Block Number:
-
{{ ethBlockLink .BlockNumber }}
+
Payload Status:
+
+ {{ if eq .PayloadStatus 0 }} + Missing + {{ else if eq .PayloadStatus 1 }} + Proposed + {{ else if eq .PayloadStatus 2 }} + Orphaned + {{ else }} + Unknown + {{ end }} +
Block Hash:
{{ ethBlockHashLink .BlockHash }} +
Parent Hash:
- 0x{{ printf "%x" .ParentHash }} + {{ ethBlockHashLink .ParentBlockHash }} + +
+
+ +
+
Builder Index:
+
+ {{ formatValidator .BuilderIndex .BuilderName }} +
+
+ +
+
Block Value:
+
+ {{ formatEthFromGwei .Value }} +
+
+ +
+
Gas Limit:
+
+ {{ .GasLimit }} +
+
+ +
+
Blob KZG Root:
+
+ 0x{{ printf "%x" .BlobKzgCommitmentsRoot }} +
+
+
+ {{ end }} + {{ end }} + {{ if .Block.ExecutionData }} + {{ $block := .Block }} + {{ with .Block.ExecutionData }} +
+
Execution Payload:
+
+
+
Block Number:
+
{{ ethBlockLink .BlockNumber }}
+
+ + {{ if not $block.PayloadHeader }} +
+
Block Hash:
+
+ {{ ethBlockHashLink .BlockHash }} +
+
+ +
+
Parent Hash:
+
+ 0x{{ printf "%x" .ParentHash }} +
+
+ {{ end }}
Fee Recipient:
@@ -189,10 +262,12 @@
{{ .GasUsed }}
-
-
Gas Limit:
-
{{ .GasLimit }}
-
+ {{ if not $block.PayloadHeader }} +
+
Gas Limit:
+
{{ .GasLimit }}
+
+ {{ end }}
Base fee per gas:
diff --git a/types/models/slot.go b/types/models/slot.go index 7ea14080e..34901472b 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -66,7 +66,9 @@ type SlotPageBlockData struct { WithdrawalRequestsCount uint64 `json:"withdrawal_requests_count"` ConsolidationRequestsCount uint64 `json:"consolidation_requests_count"` - ExecutionData *SlotPageExecutionData `json:"execution_data"` + PayloadHeader *SlotPagePayloadHeader `json:"payload_header"` + ExecutionData *SlotPageExecutionData `json:"execution_data"` + Attestations []*SlotPageAttestation `json:"attestations"` // Attestations included in this block Deposits []*SlotPageDeposit `json:"deposits"` // Deposits included in this block VoluntaryExits []*SlotPageVoluntaryExit `json:"voluntary_exits"` // Voluntary Exits included in this block @@ -98,6 +100,20 @@ type SlotPageExecutionData struct { BlockNumber uint64 `json:"block_number"` } +type SlotPagePayloadHeader struct { + PayloadStatus uint16 `json:"payload_status"` + ParentBlockHash []byte `json:"parent_block_hash"` + ParentBlockRoot []byte `json:"parent_block_root"` + BlockHash []byte `json:"block_hash"` + GasLimit uint64 `json:"gas_limit"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` + Slot uint64 `json:"slot"` + Value uint64 `json:"value"` + BlobKzgCommitmentsRoot []byte `json:"blob_kzg_commitments_root"` + Signature []byte `json:"signature"` +} + type SlotPageAttestation struct { Slot uint64 `json:"slot"` CommitteeIndex []uint64 `json:"committeeindex"` From 4fe1f72c207d043e31940a5f28029136d1e6e393 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 12 Feb 2025 11:18:46 +0100 Subject: [PATCH 14/99] add payload_status & remove has_payload to allow additional payload status (orphaned) --- .../pgsql/20250208225212_epbs-payload.sql | 6 +- .../sqlite/20250208225212_epbs-payload.sql | 4 +- db/slots.go | 20 +++---- dbtypes/dbtypes.go | 56 +++++++++++-------- handlers/epoch.go | 7 ++- handlers/index.go | 48 ++++++++++------ handlers/slots.go | 7 ++- handlers/slots_filtered.go | 7 ++- handlers/validator_slots.go | 7 ++- indexer/beacon/writedb.go | 9 ++- static/css/layout.css | 10 ++++ templates/epoch/epoch.html | 6 +- templates/index/recentBlocks.html | 12 ++-- templates/index/recentSlots.html | 12 ++-- templates/slots/slots.html | 6 +- templates/slots_filtered/slots_filtered.html | 6 +- templates/validator_slots/slots.html | 6 +- types/models/epoch.go | 2 +- types/models/indexPage.go | 44 +++++++-------- types/models/slots.go | 2 +- types/models/slots_filtered.go | 2 +- types/models/validator_slots.go | 2 +- 22 files changed, 167 insertions(+), 114 deletions(-) diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql index 527e5645b..bccb6ba96 100644 --- a/db/schema/pgsql/20250208225212_epbs-payload.sql +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -10,11 +10,11 @@ ALTER TABLE public."orphaned_blocks" ADD "payload_ssz" bytea NULL; ALTER TABLE public."slots" ADD - "has_payload" boolean NOT NULL DEFAULT false; + "payload_status" smallint NOT NULL DEFAULT 0; -CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" +CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON public."slots" - ("has_payload" ASC NULLS LAST); + ("payload_status" ASC NULLS LAST); ALTER TABLE public."epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 9bc45c060..8dc13a74f 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -7,9 +7,9 @@ ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; -ALTER TABLE "slots" ADD "has_payload" boolean NOT NULL DEFAULT false; +ALTER TABLE "slots" ADD "payload_status" smallint NOT NULL DEFAULT 0; -CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" ON "slots" ("has_payload" ASC); +CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON "slots" ("payload_status" ASC); ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/slots.go b/db/slots.go index 8786cd5a4..c842f4486 100644 --- a/db/slots.go +++ b/db/slots.go @@ -19,26 +19,26 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, eth_block_extra_text = excluded.eth_block_extra_text, fork_id = excluded.fork_id, - has_payload = excluded.has_payload`, + payload_status = excluded.payload_status`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO slots ( slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, - slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.HasPayload) + slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.PayloadStatus) if err != nil { return err } @@ -93,7 +93,7 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "payload_status", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -125,7 +125,7 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -144,7 +144,7 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE root = $1 `, root) @@ -170,7 +170,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -213,7 +213,7 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -273,7 +273,7 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "payload_status", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index a92231e70..8a6127cf5 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -18,6 +18,14 @@ const ( Orphaned ) +type PayloadStatus uint8 + +const ( + PayloadStatusMissing PayloadStatus = iota + PayloadStatusCanonical + PayloadStatusOrphaned +) + type SlotHeader struct { Slot uint64 `db:"slot"` Proposer uint64 `db:"proposer"` @@ -25,30 +33,30 @@ type SlotHeader struct { } type Slot struct { - Slot uint64 `db:"slot"` - Proposer uint64 `db:"proposer"` - Status SlotStatus `db:"status"` - Root []byte `db:"root"` - ParentRoot []byte `db:"parent_root"` - StateRoot []byte `db:"state_root"` - Graffiti []byte `db:"graffiti"` - GraffitiText string `db:"graffiti_text"` - AttestationCount uint64 `db:"attestation_count"` - DepositCount uint64 `db:"deposit_count"` - ExitCount uint64 `db:"exit_count"` - WithdrawCount uint64 `db:"withdraw_count"` - WithdrawAmount uint64 `db:"withdraw_amount"` - AttesterSlashingCount uint64 `db:"attester_slashing_count"` - ProposerSlashingCount uint64 `db:"proposer_slashing_count"` - BLSChangeCount uint64 `db:"bls_change_count"` - EthTransactionCount uint64 `db:"eth_transaction_count"` - EthBlockNumber *uint64 `db:"eth_block_number"` - EthBlockHash []byte `db:"eth_block_hash"` - EthBlockExtra []byte `db:"eth_block_extra"` - EthBlockExtraText string `db:"eth_block_extra_text"` - SyncParticipation float32 `db:"sync_participation"` - ForkId uint64 `db:"fork_id"` - HasPayload bool `db:"has_payload"` + Slot uint64 `db:"slot"` + Proposer uint64 `db:"proposer"` + Status SlotStatus `db:"status"` + Root []byte `db:"root"` + ParentRoot []byte `db:"parent_root"` + StateRoot []byte `db:"state_root"` + Graffiti []byte `db:"graffiti"` + GraffitiText string `db:"graffiti_text"` + AttestationCount uint64 `db:"attestation_count"` + DepositCount uint64 `db:"deposit_count"` + ExitCount uint64 `db:"exit_count"` + WithdrawCount uint64 `db:"withdraw_count"` + WithdrawAmount uint64 `db:"withdraw_amount"` + AttesterSlashingCount uint64 `db:"attester_slashing_count"` + ProposerSlashingCount uint64 `db:"proposer_slashing_count"` + BLSChangeCount uint64 `db:"bls_change_count"` + EthTransactionCount uint64 `db:"eth_transaction_count"` + EthBlockNumber *uint64 `db:"eth_block_number"` + EthBlockHash []byte `db:"eth_block_hash"` + EthBlockExtra []byte `db:"eth_block_extra"` + EthBlockExtraText string `db:"eth_block_extra_text"` + SyncParticipation float32 `db:"sync_participation"` + ForkId uint64 `db:"fork_id"` + PayloadStatus PayloadStatus `db:"payload_status"` } type Epoch struct { diff --git a/handlers/epoch.go b/handlers/epoch.go index b1c6b28cd..0df58eff8 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -167,13 +167,18 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { pageData.MissedCount++ } + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.EpochPageDataSlot{ Slot: slot, Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), Ts: chainState.SlotToTime(phase0.Slot(slot)), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Status: uint8(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(phase0.Epoch(epoch)), + PayloadStatus: uint8(payloadStatus), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), AttestationCount: dbSlot.AttestationCount, diff --git a/handlers/index.go b/handlers/index.go index 27c07e9d6..8c02e4e0a 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -286,15 +286,21 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockC } epoch := chainState.EpochOfSlot(phase0.Slot(blockData.Slot)) + + payloadStatus := blockData.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + blockModel := &models.IndexPageDataBlocks{ - Epoch: uint64(epoch), - Slot: blockData.Slot, - Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), - Proposer: blockData.Proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), - Status: uint64(blockData.Status), - NoPayload: !blockData.HasPayload && chainState.IsEip7732Enabled(epoch), - BlockRoot: blockData.Root, + Epoch: uint64(epoch), + Slot: blockData.Slot, + Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), + Proposer: blockData.Proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), + Status: uint64(blockData.Status), + PayloadStatus: uint8(payloadStatus), + BlockRoot: blockData.Root, } if blockData.EthBlockNumber != nil { blockModel.WithEthBlock = true @@ -333,17 +339,23 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot pha dbIdx++ epoch := chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot)) + + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.IndexPageDataSlots{ - Slot: slot, - Epoch: uint64(epoch), - Ts: chainState.SlotToTime(phase0.Slot(slot)), - Status: uint64(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), - Proposer: dbSlot.Proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), - BlockRoot: dbSlot.Root, - ParentRoot: dbSlot.ParentRoot, - ForkGraph: make([]*models.IndexPageDataForkGraph, 0), + Slot: slot, + Epoch: uint64(epoch), + Ts: chainState.SlotToTime(phase0.Slot(slot)), + Status: uint64(dbSlot.Status), + PayloadStatus: uint8(payloadStatus), + Proposer: dbSlot.Proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), + BlockRoot: dbSlot.Root, + ParentRoot: dbSlot.ParentRoot, + ForkGraph: make([]*models.IndexPageDataForkGraph, 0), } pageData.RecentSlots = append(pageData.RecentSlots, slotData) blockCount++ diff --git a/handlers/slots.go b/handlers/slots.go index 67bb151d4..330d2bff3 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -143,13 +143,18 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat dbIdx++ epoch := chainState.EpochOfSlot(phase0.Slot(slot)) + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.SlotsPageDataSlot{ Slot: slot, Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Finalized: finalized, Status: uint8(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), + PayloadStatus: uint8(payloadStatus), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Synchronized: dbSlot.SyncParticipation != -1, Proposer: dbSlot.Proposer, diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index fd17ba992..3acdaa764 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -266,7 +266,12 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber } - slotData.NoPayload = !dbBlock.Block.HasPayload && chainState.IsEip7732Enabled(epoch) + + payloadStatus := dbBlock.Block.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData.PayloadStatus = uint8(payloadStatus) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index dad03dc40..c5f70b600 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -140,7 +140,12 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.EthBlockNumber } - slotData.NoPayload = !dbBlock.HasPayload && chainState.IsEip7732Enabled(epoch) + + payloadStatus := dbBlock.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData.PayloadStatus = uint8(payloadStatus) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 79614be46..d1c7905c7 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -250,7 +250,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - var hasPayload bool + var payloadStatus dbtypes.PayloadStatus if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -260,9 +260,12 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits - hasPayload = true + payloadStatus = dbtypes.PayloadStatusCanonical + } else { + payloadStatus = dbtypes.PayloadStatusMissing } } else { + payloadStatus = dbtypes.PayloadStatusCanonical executionBlockNumber, _ = blockBody.ExecutionBlockNumber() executionExtraData, _ = getBlockExecutionExtraData(blockBody) @@ -290,7 +293,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override AttesterSlashingCount: uint64(len(attesterSlashings)), ProposerSlashingCount: uint64(len(proposerSlashings)), BLSChangeCount: uint64(len(blsToExecChanges)), - HasPayload: hasPayload, + PayloadStatus: payloadStatus, } if overrideForkId != nil { diff --git a/static/css/layout.css b/static/css/layout.css index c10a43314..ddb8442ae 100644 --- a/static/css/layout.css +++ b/static/css/layout.css @@ -332,3 +332,13 @@ span.validator-label { rgba(255,193,7,1) 100% ); } + +.badge.split-info { + background: linear-gradient( + 90deg, + rgba(255,255,255,0) 0%, + rgba(255,255,255,0) 50%, + rgba(13,202,240,1) 50%, + rgba(13,202,240,1) 100% + ); +} diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html index c400c2bfd..d625d1704 100644 --- a/templates/epoch/epoch.html +++ b/templates/epoch/epoch.html @@ -176,15 +176,15 @@

{{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Orphaned {{ else if not $epoch.Synchronized }} ? {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index 51b639fcc..27a2950d4 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@

Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -74,11 +74,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $block.Ts }} diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index bc348763a..7a47f7cd3 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -97,11 +97,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots/slots.html b/templates/slots/slots.html index 2a4a6d148..b075653f0 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -92,9 +92,9 @@

Slots

{{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if not $slot.Synchronized }} @@ -102,7 +102,7 @@

Slots

{{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index 8400972e1..875741745 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -183,9 +183,9 @@

Filtered Slots

{{- if eq $slot.Slot 0 }} Genesis {{- else if eq $slot.Status 1 }} - Proposed + Proposed {{- else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{- else if $slot.Scheduled }} Scheduled {{- else if not $slot.Synchronized }} @@ -193,7 +193,7 @@

Filtered Slots

{{- else if eq $slot.Status 0 }} Missed {{- else }} - Unknown + Unknown {{- end }} {{- end }} diff --git a/templates/validator_slots/slots.html b/templates/validator_slots/slots.html index a89f09d83..26aa83f09 100644 --- a/templates/validator_slots/slots.html +++ b/templates/validator_slots/slots.html @@ -72,15 +72,15 @@

Validator {{ format {{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/types/models/epoch.go b/types/models/epoch.go index c15be304c..5c97fa218 100644 --- a/types/models/epoch.go +++ b/types/models/epoch.go @@ -44,7 +44,7 @@ type EpochPageDataSlot struct { Ts time.Time `json:"ts"` Scheduled bool `json:"scheduled"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` diff --git a/types/models/indexPage.go b/types/models/indexPage.go index af5d5c4de..d262f2231 100644 --- a/types/models/indexPage.go +++ b/types/models/indexPage.go @@ -56,31 +56,31 @@ type IndexPageDataEpochs struct { } type IndexPageDataBlocks struct { - Epoch uint64 `json:"epoch"` - Slot uint64 `json:"slot"` - WithEthBlock bool `json:"has_block"` - EthBlock uint64 `json:"eth_block"` - EthBlockLink string `json:"eth_link"` - Ts time.Time `json:"ts"` - Proposer uint64 `json:"proposer"` - ProposerName string `json:"proposer_name"` - Status uint64 `json:"status"` - NoPayload bool `json:"no_payload"` - BlockRoot []byte `json:"block_root"` + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + WithEthBlock bool `json:"has_block"` + EthBlock uint64 `json:"eth_block"` + EthBlockLink string `json:"eth_link"` + Ts time.Time `json:"ts"` + Proposer uint64 `json:"proposer"` + ProposerName string `json:"proposer_name"` + Status uint64 `json:"status"` + PayloadStatus uint8 `json:"payload_status"` + BlockRoot []byte `json:"block_root"` } type IndexPageDataSlots struct { - Epoch uint64 `json:"epoch"` - Slot uint64 `json:"slot"` - EthBlock uint64 `json:"eth_block"` - Ts time.Time `json:"ts"` - Proposer uint64 `json:"proposer"` - ProposerName string `json:"proposer_name"` - Status uint64 `json:"status"` - NoPayload bool `json:"no_payload"` - BlockRoot []byte `json:"block_root"` - ParentRoot []byte `json:"-"` - ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + EthBlock uint64 `json:"eth_block"` + Ts time.Time `json:"ts"` + Proposer uint64 `json:"proposer"` + ProposerName string `json:"proposer_name"` + Status uint64 `json:"status"` + PayloadStatus uint8 `json:"payload_status"` + BlockRoot []byte `json:"block_root"` + ParentRoot []byte `json:"-"` + ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` } type IndexPageDataForkGraph struct { diff --git a/types/models/slots.go b/types/models/slots.go index a3fcf9d41..651e91f20 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -31,7 +31,7 @@ type SlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index 38fd10d48..483d32112 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -56,7 +56,7 @@ type SlotsFilteredPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/validator_slots.go b/types/models/validator_slots.go index 0f4ade491..6d61408db 100644 --- a/types/models/validator_slots.go +++ b/types/models/validator_slots.go @@ -34,7 +34,7 @@ type ValidatorSlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` From 241771c0ba5594cf560a6e7916e48b92d156887e Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 17 Feb 2025 23:06:00 +0100 Subject: [PATCH 15/99] fix missing getters for eip7732 states --- indexer/beacon/block_helper.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 656494410..211d43092 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -454,6 +454,12 @@ func getStatePendingWithdrawals(v *spec.VersionedBeaconState) ([]*electra.Pendin } return v.Electra.PendingPartialWithdrawals, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingPartialWithdrawals == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingPartialWithdrawals, nil default: return nil, errors.New("unknown version") } @@ -478,6 +484,12 @@ func getStatePendingConsolidations(v *spec.VersionedBeaconState) ([]*electra.Pen } return v.Electra.PendingConsolidations, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingConsolidations == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingConsolidations, nil default: return nil, errors.New("unknown version") } From 0380f1366782dfce938d919ead8a6b2950f80f24 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 7 Apr 2025 22:30:36 +0200 Subject: [PATCH 16/99] save execution payload to block store --- blockdb/blockdb.go | 16 +++++---- blockdb/pebble/pebble.go | 62 ++++++++++++++++++++++++++++++-- blockdb/s3/s3store.go | 40 ++++++++++++++++----- blockdb/types/engine.go | 15 ++++---- cmd/dora-utils/blockdb_sync.go | 26 +++++++++++--- indexer/beacon/block.go | 6 ++-- indexer/beacon/block_helper.go | 8 ++--- indexer/beacon/client.go | 2 +- indexer/beacon/indexer.go | 2 +- indexer/beacon/indexer_getter.go | 2 +- services/chainservice_blocks.go | 4 +++ 11 files changed, 145 insertions(+), 38 deletions(-) diff --git a/blockdb/blockdb.go b/blockdb/blockdb.go index 6ef50d237..2e13cb6ce 100644 --- a/blockdb/blockdb.go +++ b/blockdb/blockdb.go @@ -45,17 +45,19 @@ func (db *BlockDb) Close() error { return db.engine.Close() } -func (db *BlockDb) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { - return db.engine.GetBlock(ctx, slot, root, parseBlock) +func (db *BlockDb) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { + return db.engine.GetBlock(ctx, slot, root, parseBlock, parsePayload) } -func (db *BlockDb) AddBlock(ctx context.Context, slot uint64, root []byte, header_ver uint64, header_data []byte, body_ver uint64, body_data []byte) (bool, error) { +func (db *BlockDb) AddBlock(ctx context.Context, slot uint64, root []byte, header_ver uint64, header_data []byte, body_ver uint64, body_data []byte, payload_ver uint64, payload_data []byte) (bool, error) { return db.engine.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { return &types.BlockData{ - HeaderVersion: header_ver, - HeaderData: header_data, - BodyVersion: body_ver, - BodyData: body_data, + HeaderVersion: header_ver, + HeaderData: header_data, + BodyVersion: body_ver, + BodyData: body_data, + PayloadVersion: payload_ver, + PayloadData: payload_data, }, nil }) } diff --git a/blockdb/pebble/pebble.go b/blockdb/pebble/pebble.go index d8619201e..7bea79c25 100644 --- a/blockdb/pebble/pebble.go +++ b/blockdb/pebble/pebble.go @@ -14,8 +14,9 @@ const ( ) const ( - BlockTypeHeader uint16 = 1 - BlockTypeBody uint16 = 2 + BlockTypeHeader uint16 = 1 + BlockTypeBody uint16 = 2 + BlockTypePayload uint16 = 3 ) type PebbleEngine struct { @@ -97,7 +98,34 @@ func (e *PebbleEngine) getBlockBody(root []byte, parser func(uint64, []byte) (in return body, version, nil } -func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { +func (e *PebbleEngine) getBlockPayload(root []byte, parser func(uint64, []byte) (interface{}, error)) (interface{}, uint64, error) { + key := make([]byte, 2+len(root)+2) + binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) + copy(key[2:], root) + binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) + + res, closer, err := e.db.Get(key) + if err != nil && err != pebble.ErrNotFound { + return nil, 0, err + } + defer closer.Close() + + if err == pebble.ErrNotFound || len(res) == 0 { + return nil, 0, nil + } + + version := binary.BigEndian.Uint64(res[:8]) + block := res[8:] + + body, err := parser(version, block) + if err != nil { + return nil, 0, err + } + + return body, version, nil +} + +func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { header, header_ver, err := e.getBlockHeader(root) if err != nil { return nil, err @@ -124,6 +152,14 @@ func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseB blockData.Body = body blockData.BodyVersion = body_ver + payload, payload_ver, err := e.getBlockPayload(root, parsePayload) + if err != nil { + return nil, err + } + + blockData.Payload = payload + blockData.PayloadVersion = payload_ver + return blockData, nil } @@ -157,6 +193,19 @@ func (e *PebbleEngine) addBlockBody(root []byte, version uint64, block []byte) e return e.db.Set(key, data, nil) } +func (e *PebbleEngine) addBlockPayload(root []byte, version uint64, payload []byte) error { + key := make([]byte, 2+len(root)+2) + binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) + copy(key[2:], root) + binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) + + data := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint64(data[:8], version) + copy(data[8:], payload) + + return e.db.Set(key, data, nil) +} + func (e *PebbleEngine) AddBlock(_ context.Context, _ uint64, root []byte, dataCb func() (*types.BlockData, error)) (bool, error) { key := make([]byte, 2+len(root)+2) binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) @@ -182,5 +231,12 @@ func (e *PebbleEngine) AddBlock(_ context.Context, _ uint64, root []byte, dataCb return false, err } + if blockData.PayloadVersion != 0 { + err = e.addBlockPayload(root, blockData.PayloadVersion, blockData.PayloadData) + if err != nil { + return false, err + } + } + return true, nil } diff --git a/blockdb/s3/s3store.go b/blockdb/s3/s3store.go index 1110b4a60..e3aec098d 100644 --- a/blockdb/s3/s3store.go +++ b/blockdb/s3/s3store.go @@ -60,10 +60,12 @@ func (e *S3Engine) getObjectKey(root []byte, slot uint64) string { } type objectMetadata struct { - objVersion uint32 - headerLength uint32 - bodyVersion uint32 - bodyLength uint32 + objVersion uint32 + headerLength uint32 + bodyVersion uint32 + bodyLength uint32 + payloadVersion uint32 + payloadLength uint32 } func (e *S3Engine) readObjectMetadata(data []byte) (*objectMetadata, int, error) { @@ -78,6 +80,13 @@ func (e *S3Engine) readObjectMetadata(data []byte) (*objectMetadata, int, error) metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) metadataLength += 12 + case 2: + metadata.headerLength = binary.BigEndian.Uint32(data[4:8]) + metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) + metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) + metadata.payloadVersion = binary.BigEndian.Uint32(data[16:20]) + metadata.payloadLength = binary.BigEndian.Uint32(data[20:24]) + metadataLength += 20 } return metadata, metadataLength, nil @@ -92,12 +101,18 @@ func (e *S3Engine) writeObjectMetadata(metadata *objectMetadata) []byte { data = binary.BigEndian.AppendUint32(data, metadata.headerLength) data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) + case 2: + data = binary.BigEndian.AppendUint32(data, metadata.headerLength) + data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) + data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) + data = binary.BigEndian.AppendUint32(data, metadata.payloadVersion) + data = binary.BigEndian.AppendUint32(data, metadata.payloadLength) } return data } -func (e *S3Engine) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { +func (e *S3Engine) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { key := e.getObjectKey(root, slot) obj, err := e.client.GetObject(ctx, e.bucket, key, minio.GetObjectOptions{}) @@ -184,20 +199,29 @@ func (e *S3Engine) AddBlock(ctx context.Context, slot uint64, root []byte, dataC } metadata := &objectMetadata{ - objVersion: uint32(blockData.HeaderVersion), + objVersion: 1, headerLength: uint32(len(blockData.HeaderData)), bodyVersion: uint32(blockData.BodyVersion), bodyLength: uint32(len(blockData.BodyData)), } + if blockData.PayloadVersion != 0 { + metadata.objVersion = 2 + metadata.payloadVersion = uint32(blockData.PayloadVersion) + metadata.payloadLength = uint32(len(blockData.PayloadData)) + } + metadataBytes := e.writeObjectMetadata(metadata) metadataLength := len(metadataBytes) // Prepare data with header and body versions and lengths - data := make([]byte, metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)) + data := make([]byte, metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)) copy(data[:metadataLength], metadataBytes) copy(data[metadataLength:metadataLength+int(metadata.headerLength)], blockData.HeaderData) - copy(data[metadataLength+int(metadata.headerLength):], blockData.BodyData) + copy(data[metadataLength+int(metadata.headerLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)], blockData.BodyData) + if metadata.objVersion == 2 { + copy(data[metadataLength+int(metadata.headerLength)+int(metadata.bodyLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)], blockData.PayloadData) + } // Upload object _, err = e.client.PutObject( diff --git a/blockdb/types/engine.go b/blockdb/types/engine.go index 8152b5013..33d3c7fc0 100644 --- a/blockdb/types/engine.go +++ b/blockdb/types/engine.go @@ -3,14 +3,17 @@ package types import "context" type BlockData struct { - HeaderVersion uint64 - HeaderData []byte - BodyVersion uint64 - BodyData []byte - Body interface{} + HeaderVersion uint64 + HeaderData []byte + BodyVersion uint64 + BodyData []byte + Body interface{} + PayloadVersion uint64 + PayloadData []byte + Payload interface{} } type BlockDbEngine interface { Close() error - GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*BlockData, error) + GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*BlockData, error) AddBlock(ctx context.Context, slot uint64, root []byte, dataCb func() (*BlockData, error)) (bool, error) } diff --git a/cmd/dora-utils/blockdb_sync.go b/cmd/dora-utils/blockdb_sync.go index 6dbd9c426..936647029 100644 --- a/cmd/dora-utils/blockdb_sync.go +++ b/cmd/dora-utils/blockdb_sync.go @@ -272,11 +272,29 @@ func processSlot(ctx context.Context, pool *consensus.Pool, dynSsz *dynssz.DynSs return nil, fmt.Errorf("failed to marshal block body for slot %d: %v", slot, err) } + var payloadVersion uint64 + var payloadBytes []byte + + chainState := pool.GetChainState() + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(phase0.Slot(slot))) { + blockPayload, err := client.GetRPCClient().GetExecutionPayloadByBlockroot(ctx, blockHeader.Root) + if err != nil { + return nil, fmt.Errorf("failed to get block execution payload for slot %d: %v", slot, err) + } + + payloadVersion, payloadBytes, err = beacon.MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz, blockPayload, true) + if err != nil { + return nil, fmt.Errorf("failed to marshal block execution payload for slot %d: %v", slot, err) + } + } + return &btypes.BlockData{ - HeaderVersion: 1, - HeaderData: headerBytes, - BodyVersion: version, - BodyData: bodyBytes, + HeaderVersion: 1, + HeaderData: headerBytes, + BodyVersion: version, + BodyData: bodyBytes, + PayloadVersion: payloadVersion, + PayloadData: payloadBytes, }, nil }) if err != nil { diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 4c5b36824..d28a15795 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -190,7 +190,7 @@ func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelop if block.hasExecutionPayload && block.isInUnfinalizedDb { dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, false, true) if dbBlock != nil { - payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + payload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err == nil { return payload } @@ -461,7 +461,7 @@ func (block *Block) buildOrphanedBlock(compress bool) (*dbtypes.OrphanedBlock, e } if block.executionPayload != nil { - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) + payloadVer, payloadSSZ, err := MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) if err != nil { return nil, fmt.Errorf("marshal execution payload ssz failed: %v", err) } @@ -512,7 +512,7 @@ func (block *Block) unpruneBlockBody() { if dbBlock != nil { block.block, _ = UnmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) if len(dbBlock.PayloadSSZ) > 0 { - block.executionPayload, _ = unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + block.executionPayload, _ = UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) } } } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index da4b6c74e..9679256b6 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -208,8 +208,8 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. return block, nil } -// marshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. -func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { +// MarshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. +func MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { if utils.Config.KillSwitch.DisableSSZEncoding { // SSZ encoding disabled, use json instead version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) @@ -227,8 +227,8 @@ func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, pa return } -// unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. -func unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { +// UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. +func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { if (version & compressionFlag) != 0 { // decompress if d, err := decompressBytes(ssz); err != nil { diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 4f25f5ab6..b41dd446b 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -593,7 +593,7 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio } func (c *Client) persistExecutionPayload(block *Block) error { - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + payloadVer, payloadSSZ, err := MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) if err != nil { return fmt.Errorf("marshal execution payload ssz failed: %v", err) } diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 5d5513797..623a93bf0 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -361,7 +361,7 @@ func (indexer *Indexer) StartIndexer() { } if len(dbBlock.PayloadSSZ) > 0 { - blockPayload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + blockPayload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err != nil { indexer.logger.Warnf("could not restore unfinalized block payload %v [%x] from db: %v", dbBlock.Slot, dbBlock.Root, err) } else if block.processingStatus == 0 { diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 80b5f12e0..7c80dd844 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -201,7 +201,7 @@ func (indexer *Indexer) GetOrphanedBlockByRoot(blockRoot phase0.Root) (*Block, e block.SetBlock(blockBody) if len(orphanedBlock.PayloadSSZ) > 0 { - payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) + payload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) if err != nil { return nil, fmt.Errorf("could not restore orphaned block payload %v [%x] from db: %v", header.Message.Slot, orphanedBlock.Root, err) } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index b49deec5a..a3f8391e2 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -177,6 +177,8 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot if result == nil && header != nil && blockdb.GlobalBlockDb != nil { blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(header.Message.Slot), blockroot[:], func(version uint64, block []byte) (interface{}, error) { return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (interface{}, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) }) if err == nil && blockData != nil { result = &CombinedBlockResponse{ @@ -309,6 +311,8 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl if result == nil && header != nil && blockdb.GlobalBlockDb != nil { blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(slot), blockRoot[:], func(version uint64, block []byte) (interface{}, error) { return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (interface{}, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) }) if err == nil && blockData != nil { header := &phase0.SignedBeaconBlockHeader{} From 499ef5aca49863608c6437992798880a72bfe90f Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 15 Jul 2025 16:33:44 +0200 Subject: [PATCH 17/99] fix merge issue with blob kzg blob kzg commitments --- indexer/beacon/writedb.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 90170e556..fccbaba49 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -7,6 +7,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" @@ -248,13 +249,13 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() executionBlockHash, _ := blockBody.ExecutionBlockHash() - blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionBlockNumber uint64 var executionExtraData []byte var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var blobKzgCommitments []deneb.KZGCommitment var payloadStatus dbtypes.PayloadStatus if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { @@ -265,6 +266,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + blobKzgCommitments = blockPayload.Message.BlobKZGCommitments payloadStatus = dbtypes.PayloadStatusCanonical } else { payloadStatus = dbtypes.PayloadStatusMissing @@ -276,6 +278,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionExtraData, _ = getBlockExecutionExtraData(blockBody) executionTransactions, _ = blockBody.ExecutionTransactions() executionWithdrawals, _ = blockBody.Withdrawals() + blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits @@ -478,11 +481,11 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var blobKzgCommitments []deneb.KZGCommitment if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -491,10 +494,12 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + blobKzgCommitments = blockPayload.Message.BlobKZGCommitments } } else { executionTransactions, _ = blockBody.ExecutionTransactions() executionWithdrawals, _ = blockBody.Withdrawals() + blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits From eef1a3dafb44cba8a9637dddf681dabf2e307357 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 15 Jul 2025 16:57:04 +0200 Subject: [PATCH 18/99] add missing block helpers for eip7732 --- indexer/beacon/block_helper.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 234ec54e3..763a67a95 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -454,6 +454,12 @@ func getStateDepositBalanceToConsume(v *spec.VersionedBeaconState) (phase0.Gwei, } return v.Electra.DepositBalanceToConsume, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil { + return 0, errors.New("no eip7732 block") + } + + return v.EIP7732.DepositBalanceToConsume, nil default: return 0, errors.New("unknown version") } @@ -478,6 +484,12 @@ func getStatePendingDeposits(v *spec.VersionedBeaconState) ([]*electra.PendingDe } return v.Electra.PendingDeposits, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingDeposits == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingDeposits, nil default: return nil, errors.New("unknown version") } @@ -558,6 +570,8 @@ func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) return dynSsz.SizeSSZ(block.Deneb) case spec.DataVersionElectra: return dynSsz.SizeSSZ(block.Electra) + case spec.DataVersionEIP7732: + return dynSsz.SizeSSZ(block.EIP7732) default: return 0, errors.New("unknown version") } From 60104583b81f7e048785adcacc6f8a67f2d2bfba Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 16:32:26 +0200 Subject: [PATCH 19/99] add gloas fork badge and internal getters --- clients/consensus/chainspec.go | 7 +++- clients/consensus/chainstate.go | 28 ++++++++++++++ handlers/api/network_forks_v1.go | 4 +- handlers/index.go | 13 +++++++ indexer/beacon/block_helper.go | 63 ++++++++++++++++++++++++++++++++ indexer/beacon/writedb.go | 16 ++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 22fc057b4..04f9b7c99 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -40,8 +40,10 @@ type ChainSpec struct { DenebForkEpoch *uint64 `yaml:"DENEB_FORK_EPOCH"` ElectraForkVersion phase0.Version `yaml:"ELECTRA_FORK_VERSION" check-if-fork:"ElectraForkEpoch"` ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH" check-if-fork:"ElectraForkEpoch"` - FuluForkVersion phase0.Version `yaml:"FULU_FORK_VERSION" check-if-fork:"FuluForkEpoch"` - FuluForkEpoch *uint64 `yaml:"FULU_FORK_EPOCH"` + FuluForkVersion phase0.Version `yaml:"FULU_FORK_VERSION" check-if-fork:"FuluForkEpoch"` + FuluForkEpoch *uint64 `yaml:"FULU_FORK_EPOCH" check-if-fork:"FuluForkEpoch"` + GloasForkVersion phase0.Version `yaml:"GLOAS_FORK_VERSION" check-if-fork:"GloasForkEpoch"` + GloasForkEpoch *uint64 `yaml:"GLOAS_FORK_EPOCH" check-if-fork:"GloasForkEpoch"` SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"` SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"` EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"` @@ -65,6 +67,7 @@ type ChainSpec struct { MinEpochsForDataColumnSidecars uint64 `yaml:"MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"` DepositChainId uint64 `yaml:"DEPOSIT_CHAIN_ID"` MinActivationBalance uint64 `yaml:"MIN_ACTIVATION_BALANCE"` + MaxBlobsPerBlock uint64 `yaml:"MAX_BLOBS_PER_BLOCK" check-if-fork:"DenebForkEpoch"` MaxPendingPartialsPerWithdrawalsSweep uint64 `yaml:"MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP" check-if-fork:"ElectraForkEpoch"` MaxPendingDepositsPerEpoch uint64 `yaml:"MAX_PENDING_DEPOSITS_PER_EPOCH" check-if-fork:"ElectraForkEpoch"` PendingPartialWithdrawalsLimit uint64 `yaml:"PENDING_PARTIAL_WITHDRAWALS_LIMIT" check-if-fork:"ElectraForkEpoch"` diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index f4945e16b..a9d1457f7 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -282,6 +282,34 @@ func (cs *ChainState) GetForkDigestForEpoch(epoch phase0.Epoch) phase0.ForkDiges return cs.GetForkDigest(currentForkVersion, currentBlobParams) } +func (cs *ChainState) GetBlobScheduleForEpoch(epoch phase0.Epoch) *BlobScheduleEntry { + if cs.specs == nil { + return nil + } + + var blobSchedule *BlobScheduleEntry + + if cs.specs.ElectraForkEpoch != nil && epoch >= phase0.Epoch(*cs.specs.ElectraForkEpoch) { + blobSchedule = &BlobScheduleEntry{ + Epoch: *cs.specs.ElectraForkEpoch, + MaxBlobsPerBlock: cs.specs.MaxBlobsPerBlockElectra, + } + } else if cs.specs.DenebForkEpoch != nil && epoch >= phase0.Epoch(*cs.specs.DenebForkEpoch) { + blobSchedule = &BlobScheduleEntry{ + Epoch: *cs.specs.DenebForkEpoch, + MaxBlobsPerBlock: cs.specs.MaxBlobsPerBlock, + } + } + + for i, blobScheduleEntry := range cs.specs.BlobSchedule { + if blobScheduleEntry.Epoch <= uint64(epoch) { + blobSchedule = &cs.specs.BlobSchedule[i] + } + } + + return blobSchedule +} + func (cs *ChainState) GetForkDigest(forkVersion phase0.Version, blobParams *BlobScheduleEntry) phase0.ForkDigest { if cs.specs == nil || cs.genesis == nil { return phase0.ForkDigest{} diff --git a/handlers/api/network_forks_v1.go b/handlers/api/network_forks_v1.go index a3391be88..21493d59e 100644 --- a/handlers/api/network_forks_v1.go +++ b/handlers/api/network_forks_v1.go @@ -111,7 +111,8 @@ func buildNetworkForks(chainState *consensus.ChainState) []*APINetworkForkInfo { // Helper function to add consensus fork addConsensusFork := func(name string, forkEpoch *uint64, forkVersion phase0.Version) { if forkEpoch != nil && *forkEpoch < uint64(18446744073709551615) { - forkDigest := chainState.GetForkDigest(forkVersion, nil) + blobParams := chainState.GetBlobScheduleForEpoch(phase0.Epoch(*forkEpoch)) + forkDigest := chainState.GetForkDigest(forkVersion, blobParams) version := fmt.Sprintf("0x%x", forkVersion) epoch := *forkEpoch forks = append(forks, &APINetworkForkInfo{ @@ -134,6 +135,7 @@ func buildNetworkForks(chainState *consensus.ChainState) []*APINetworkForkInfo { addConsensusFork("Deneb", specs.DenebForkEpoch, specs.DenebForkVersion) addConsensusFork("Electra", specs.ElectraForkEpoch, specs.ElectraForkVersion) addConsensusFork("Fulu", specs.FuluForkEpoch, specs.FuluForkVersion) + addConsensusFork("Gloas", specs.GloasForkEpoch, specs.GloasForkVersion) // Add BPO forks from BLOB_SCHEDULE for i, blobSchedule := range specs.BlobSchedule { diff --git a/handlers/index.go b/handlers/index.go index 9c39294e1..055d5f44b 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -289,6 +289,19 @@ func buildIndexPageData() (*models.IndexPageData, time.Duration) { ForkDigest: forkDigest[:], }) } + if specs.GloasForkEpoch != nil && *specs.GloasForkEpoch < uint64(18446744073709551615) { + blobParams := chainState.GetBlobScheduleForEpoch(phase0.Epoch(*specs.GloasForkEpoch)) + forkDigest := chainState.GetForkDigest(specs.GloasForkVersion, blobParams) + pageData.NetworkForks = append(pageData.NetworkForks, &models.IndexPageDataForks{ + Name: "Gloas", + Epoch: *specs.GloasForkEpoch, + Version: specs.GloasForkVersion[:], + Time: uint64(chainState.EpochToTime(phase0.Epoch(*specs.GloasForkEpoch)).Unix()), + Active: uint64(currentEpoch) >= *specs.GloasForkEpoch, + Type: "consensus", + ForkDigest: forkDigest[:], + }) + } // Add BPO forks from BLOB_SCHEDULE for i, blobSchedule := range specs.BlobSchedule { diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index c943ede1c..8fa7069aa 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -10,6 +10,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" dynssz "github.com/pk910/dynamic-ssz" @@ -47,6 +48,9 @@ func MarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, block *spec.Ver case spec.DataVersionFulu: version = uint64(block.Version) ssz, err = dynSsz.MarshalSSZ(block.Fulu) + case spec.DataVersionGloas: + version = uint64(block.Version) + ssz, err = dynSsz.MarshalSSZ(block.Gloas) default: err = fmt.Errorf("unknown block version") } @@ -118,6 +122,11 @@ func UnmarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, version uint6 if err := dynSsz.UnmarshalSSZ(block.Fulu, ssz); err != nil { return nil, fmt.Errorf("failed to decode fulu signed beacon block: %v", err) } + case spec.DataVersionGloas: + block.Gloas = &gloas.SignedBeaconBlock{} + if err := dynSsz.UnmarshalSSZ(block.Gloas, ssz); err != nil { + return nil, fmt.Errorf("failed to decode gloas signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -148,6 +157,9 @@ func MarshalVersionedSignedBeaconBlockJson(block *spec.VersionedSignedBeaconBloc case spec.DataVersionFulu: version = uint64(block.Version) jsonRes, err = block.Fulu.MarshalJSON() + case spec.DataVersionGloas: + version = uint64(block.Version) + jsonRes, err = block.Gloas.MarshalJSON() default: err = fmt.Errorf("unknown block version") } @@ -201,6 +213,11 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. if err := block.Fulu.UnmarshalJSON(ssz); err != nil { return nil, fmt.Errorf("failed to decode fulu signed beacon block: %v", err) } + case spec.DataVersionGloas: + block.Gloas = &gloas.SignedBeaconBlock{} + if err := block.Gloas.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode gloas signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -252,6 +269,12 @@ func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { } return v.Fulu.RANDAOMixes, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.RANDAOMixes == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.RANDAOMixes, nil default: return nil, errors.New("unknown version") } @@ -274,6 +297,8 @@ func getStateDepositIndex(state *spec.VersionedBeaconState) uint64 { return state.Electra.ETH1DepositIndex case spec.DataVersionFulu: return state.Fulu.ETH1DepositIndex + case spec.DataVersionGloas: + return state.Gloas.ETH1DepositIndex } return 0 } @@ -319,6 +344,12 @@ func getStateCurrentSyncCommittee(v *spec.VersionedBeaconState) ([]phase0.BLSPub } return v.Fulu.CurrentSyncCommittee.Pubkeys, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.CurrentSyncCommittee == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.CurrentSyncCommittee.Pubkeys, nil default: return nil, errors.New("unknown version") } @@ -349,6 +380,12 @@ func getStateDepositBalanceToConsume(v *spec.VersionedBeaconState) (phase0.Gwei, } return v.Fulu.DepositBalanceToConsume, nil + case spec.DataVersionGloas: + if v.Gloas == nil { + return 0, errors.New("no gloas block") + } + + return v.Gloas.DepositBalanceToConsume, nil default: return 0, errors.New("unknown version") } @@ -379,6 +416,12 @@ func getStatePendingDeposits(v *spec.VersionedBeaconState) ([]*electra.PendingDe } return v.Fulu.PendingDeposits, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingDeposits == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingDeposits, nil default: return nil, errors.New("unknown version") } @@ -409,6 +452,12 @@ func getStatePendingWithdrawals(v *spec.VersionedBeaconState) ([]*electra.Pendin } return v.Fulu.PendingPartialWithdrawals, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingPartialWithdrawals == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingPartialWithdrawals, nil default: return nil, errors.New("unknown version") } @@ -439,6 +488,12 @@ func getStatePendingConsolidations(v *spec.VersionedBeaconState) ([]*electra.Pen } return v.Fulu.PendingConsolidations, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingConsolidations == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingConsolidations, nil default: return nil, errors.New("unknown version") } @@ -465,6 +520,12 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator } return v.Fulu.ProposerLookahead, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.ProposerLookahead == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.ProposerLookahead, nil default: return nil, errors.New("unknown version") } @@ -487,6 +548,8 @@ func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) return dynSsz.SizeSSZ(block.Electra) case spec.DataVersionFulu: return dynSsz.SizeSSZ(block.Fulu) + case spec.DataVersionGloas: + return dynSsz.SizeSSZ(block.Gloas) default: return 0, errors.New("unknown version") } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index cfc9f257a..ff87a1421 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -400,6 +400,15 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) dbBlock.EthFeeRecipient = payload.FeeRecipient[:] } + case spec.DataVersionGloas: + if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && + blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { + payload := blockBody.Gloas.Message.Body.ExecutionPayload + dbBlock.EthGasUsed = payload.GasUsed + dbBlock.EthGasLimit = payload.GasLimit + dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) + dbBlock.EthFeeRecipient = payload.FeeRecipient[:] + } } } @@ -555,6 +564,13 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat dbEpoch.EthGasUsed += payload.GasUsed dbEpoch.EthGasLimit += payload.GasLimit } + case spec.DataVersionGloas: + if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && + blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { + payload := blockBody.Gloas.Message.Body.ExecutionPayload + dbEpoch.EthGasUsed += payload.GasUsed + dbEpoch.EthGasLimit += payload.GasLimit + } } } } From e35dc1a11c5d58ff72b49de2c6ab811ea84412eb Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 23:08:01 +0200 Subject: [PATCH 20/99] fix merge conflicts --- clients/consensus/chainstate.go | 2 +- clients/consensus/rpc/beaconapi.go | 4 ++-- go.sum | 10 ++++++++++ handlers/slot.go | 2 +- indexer/beacon/block.go | 10 ++++++++-- indexer/beacon/block_helper.go | 10 +++++----- indexer/beacon/client.go | 6 +++--- indexer/beacon/finalization.go | 4 ++-- indexer/beacon/requests.go | 4 ++-- indexer/beacon/writedb.go | 22 ++++++++++++---------- services/chainservice_blocks.go | 12 ++++++------ 11 files changed, 52 insertions(+), 34 deletions(-) diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index e0ef4ca65..68476bd8d 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -398,7 +398,7 @@ func (cs *ChainState) IsEip7732Enabled(epoch phase0.Epoch) bool { return false } - return cs.specs.Eip7732ForkEpoch != nil && phase0.Epoch(*cs.specs.Eip7732ForkEpoch) <= epoch + return cs.specs.GloasForkEpoch != nil && phase0.Epoch(*cs.specs.GloasForkEpoch) <= epoch } func (cs *ChainState) GetBalanceChurnLimit(totalActiveBalance uint64) uint64 { diff --git a/clients/consensus/rpc/beaconapi.go b/clients/consensus/rpc/beaconapi.go index 70bbce3cc..6d4f05aec 100644 --- a/clients/consensus/rpc/beaconapi.go +++ b/clients/consensus/rpc/beaconapi.go @@ -19,7 +19,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" @@ -407,7 +407,7 @@ func (bc *BeaconClient) GetBlockBodyByBlockroot(ctx context.Context, blockroot p return result.Data, nil } -func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { +func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*gloas.SignedExecutionPayloadEnvelope, error) { provider, isProvider := bc.clientSvc.(eth2client.ExecutionPayloadProvider) if !isProvider { return nil, fmt.Errorf("get execution payload not supported") diff --git a/go.sum b/go.sum index f15da993b..0cdc5b6a1 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,14 @@ github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7 h1:0r1HjEx github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7/go.mod h1:b07+cRZs+eAR8TR57CB9TQlt5Gnl/06Xs76xt/1wq0M= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= @@ -589,6 +597,8 @@ github.com/pion/webrtc/v4 v4.0.10 h1:Hq/JLjhqLxi+NmCtE8lnRPDr8H4LcNvwg8OxVcdv56Q github.com/pion/webrtc/v4 v4.0.10/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck= github.com/pk910/dynamic-ssz v1.0.2 h1:X3CKAz0Q9wgj+zZYoatkGOsd123F65ShNqPiArFpV7Q= github.com/pk910/dynamic-ssz v1.0.2/go.mod h1:J7iiJ98fRx1KBceJK/K4fGWV2l5ITVbrnjUgGZM4ths= +github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c h1:XwF7W6QFd/a2sOoMuw7VTnLh8Ts6cb2UmmuHnNh5ewM= +github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/handlers/slot.go b/handlers/slot.go index b5eb12a3b..34e89bc71 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -753,7 +753,7 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV if specs.ElectraForkEpoch != nil && uint64(epoch) >= *specs.ElectraForkEpoch { var requests *electra.ExecutionRequests - if blockData.Block.Version >= spec.DataVersionEIP7732 { + if blockData.Block.Version >= spec.DataVersionGloas { if blockData.Payload != nil { requests = blockData.Payload.Message.ExecutionRequests } diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index d5818b187..e5acef257 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -397,16 +397,22 @@ func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload if execNumber, err := body.ExecutionBlockNumber(); err == nil { blockIndex.ExecutionNumber = uint64(execNumber) } + if transactions, err := body.ExecutionTransactions(); err == nil { + blockIndex.EthTransactionCount = uint64(len(transactions)) + } + if blobKzgCommitments, err := body.BlobKZGCommitments(); err == nil { + blockIndex.BlobCount = uint64(len(blobKzgCommitments)) + } } if payload != nil { blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) // Calculate transaction count - executionTransactions, _ := payload.Transactions() + executionTransactions := payload.Message.Payload.Transactions blockIndex.EthTransactionCount = uint64(len(executionTransactions)) // Calculate blob count - blobKzgCommitments, _ := body.BlobKZGCommitments() + blobKzgCommitments := payload.Message.BlobKZGCommitments blockIndex.BlobCount = uint64(len(blobKzgCommitments)) } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 1fcd53639..bafec4128 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -231,7 +231,7 @@ func MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, pa version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) } else { // SSZ encoding - version = uint64(spec.DataVersionEIP7732) + version = uint64(spec.DataVersionGloas) ssz, err = dynSsz.MarshalSSZ(payload) } @@ -260,7 +260,7 @@ func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, return unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version, ssz) } - if version != uint64(spec.DataVersionEIP7732) { + if version != uint64(spec.DataVersionGloas) { return nil, fmt.Errorf("unknown version") } @@ -275,7 +275,7 @@ func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, // marshalVersionedSignedExecutionPayloadEnvelopeJson marshals a versioned signed execution payload envelope using JSON encoding. func marshalVersionedSignedExecutionPayloadEnvelopeJson(payload *gloas.SignedExecutionPayloadEnvelope) (version uint64, jsonRes []byte, err error) { - version = uint64(spec.DataVersionEIP7732) + version = uint64(spec.DataVersionGloas) jsonRes, err = payload.MarshalJSON() version |= jsonVersionFlag @@ -289,7 +289,7 @@ func unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version uint64, ssz [] return nil, fmt.Errorf("no json encoding") } - if version-jsonVersionFlag != uint64(spec.DataVersionEIP7732) { + if version-jsonVersionFlag != uint64(spec.DataVersionGloas) { return nil, fmt.Errorf("unknown version") } @@ -327,7 +327,7 @@ func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, err } return v.Electra.Message.Body.ExecutionPayload.ExtraData, nil - case spec.DataVersionEIP7732: + case spec.DataVersionGloas: return nil, nil default: return nil, errors.New("unknown version") diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 57f4793ae..a96d77323 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -10,7 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" @@ -412,7 +412,7 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 } if loadPayload { - newPayload, _ := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, _ := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { t1 := time.Now() defer func() { processingTimes[0] += time.Since(t1) @@ -614,7 +614,7 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio return nil } - newPayload, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, err := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) }) if err != nil { diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index ce67ebed4..3bcc34869 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -10,7 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" "github.com/ethpandaops/dora/db" @@ -153,7 +153,7 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R } if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { - if _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + if _, err := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(client.getContext(), client, block.Root) }); err != nil { client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index 585be13cf..f8182a79c 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -6,7 +6,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -81,7 +81,7 @@ func LoadBeaconState(ctx context.Context, client *Client, root phase0.Root) (*sp } // LoadExecutionPayload loads the execution payload from the client. -func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { +func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*gloas.SignedExecutionPayloadEnvelope, error) { ctx, cancel := context.WithTimeout(ctx, executionPayloadRequestTimeout) defer cancel() diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 8c6f0e504..6da1e9919 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -274,10 +274,12 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override } else { payloadStatus = dbtypes.PayloadStatusCanonical executionBlockNumber, _ = blockBody.ExecutionBlockNumber() - - executionExtraData, _ = getBlockExecutionExtraData(blockBody) - executionTransactions, _ = blockBody.ExecutionTransactions() - executionWithdrawals, _ = blockBody.Withdrawals() + executionPayload, _ := blockBody.ExecutionPayload() + if executionPayload != nil { + executionExtraData, _ = executionPayload.ExtraData() + executionTransactions, _ = executionPayload.Transactions() + executionWithdrawals, _ = executionPayload.Withdrawals() + } blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { @@ -418,9 +420,9 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override dbBlock.EthFeeRecipient = payload.FeeRecipient[:] } case spec.DataVersionGloas: - if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && - blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { - payload := blockBody.Gloas.Message.Body.ExecutionPayload + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + payload := blockPayload.Message.Payload dbBlock.EthGasUsed = payload.GasUsed dbBlock.EthGasLimit = payload.GasLimit dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) @@ -596,9 +598,9 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat dbEpoch.EthGasLimit += payload.GasLimit } case spec.DataVersionGloas: - if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && - blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { - payload := blockBody.Gloas.Message.Body.ExecutionPayload + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + payload := blockPayload.Message.Payload dbEpoch.EthGasUsed += payload.GasUsed dbEpoch.EthGasLimit += payload.GasLimit } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 6379f7ea4..db5a3617d 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -9,7 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" @@ -23,7 +23,7 @@ type CombinedBlockResponse struct { Root phase0.Root Header *phase0.SignedBeaconBlockHeader Block *spec.VersionedSignedBeaconBlock - Payload *eip7732.SignedExecutionPayloadEnvelope + Payload *gloas.SignedExecutionPayloadEnvelope Orphaned bool } @@ -136,7 +136,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot } var block *spec.VersionedSignedBeaconBlock - var payload *eip7732.SignedExecutionPayloadEnvelope + var payload *gloas.SignedExecutionPayloadEnvelope bodyRetry := 0 for ; bodyRetry < 3; bodyRetry++ { client := clients[bodyRetry%len(clients)] @@ -151,7 +151,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot } } - if block.Version >= spec.DataVersionEIP7732 { + if block.Version >= spec.DataVersionGloas { payload, err = beacon.LoadExecutionPayload(ctx, client, blockroot) if payload != nil { break @@ -277,7 +277,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl var err error var block *spec.VersionedSignedBeaconBlock - var payload *eip7732.SignedExecutionPayloadEnvelope + var payload *gloas.SignedExecutionPayloadEnvelope bodyRetry := 0 for ; bodyRetry < 3; bodyRetry++ { client := clients[bodyRetry%len(clients)] @@ -290,7 +290,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl log.Warnf("Error loading block body for slot %v", slot) } - if block.Version >= spec.DataVersionEIP7732 { + if block.Version >= spec.DataVersionGloas { payload, err = beacon.LoadExecutionPayload(ctx, client, blockRoot) if payload != nil { break From b4982aef71effedbbf42d755462dff68332b3219 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 23:09:32 +0200 Subject: [PATCH 21/99] trigger CI From dce16531dcb20d00bddb80194302396f574134ff Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 8 Jan 2026 20:20:22 +0100 Subject: [PATCH 22/99] bump `go-eth2-client` --- go.mod | 3 ++- go.sum | 6 ++++-- indexer/beacon/client.go | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5944e48f6..872d6b852 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( require ( github.com/DataDog/zstd v1.5.7 // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/OffchainLabs/go-bitfield v0.0.0-20251031151322-f427d04d8506 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/VictoriaMetrics/fastcache v1.13.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -259,4 +260,4 @@ require ( modernc.org/sqlite v1.38.2 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c diff --git a/go.sum b/go.sum index 758160f03..94585ab4f 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/OffchainLabs/go-bitfield v0.0.0-20251031151322-f427d04d8506 h1:d/SJkN8/9Ca+1YmuDiUJxAiV4w/a9S8NcsG7GMQSrVI= +github.com/OffchainLabs/go-bitfield v0.0.0-20251031151322-f427d04d8506/go.mod h1:6TZI4FU6zT8x6ZfWa1J8YQ2NgW0wLV/W3fHRca8ISBo= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMGMsNAoA/+4G0= @@ -564,8 +566,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.0 h1:25Kb7CQKKkh8r8mlj/exNJY5vSoe3wGm/8QpsfayRtM= github.com/pk910/dynamic-ssz v1.2.0/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c h1:XwF7W6QFd/a2sOoMuw7VTnLh8Ts6cb2UmmuHnNh5ewM= -github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= +github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c h1:1ZlGOxqx5QOTus5NGPZH/dVqQ7tnGrqituiQYnOh0eY= +github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 32aa898bd..68b099925 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -598,7 +598,7 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio // known block or a new orphaned block // don't add to cache, process this block right after loading the details - block = newBlock(c.indexer.dynSsz, executionPayloadEvent.BlockRoot, executionPayloadEvent.Slot) + block = newBlock(c.indexer.dynSsz, executionPayloadEvent.BlockRoot, executionPayloadEvent.Slot, 0) dbBlockHead := db.GetBlockHeadByRoot(executionPayloadEvent.BlockRoot[:]) if dbBlockHead != nil { From 121c9772d769b02e2c5b7a7f0d82731a0f043cc1 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 8 Jan 2026 21:48:53 +0100 Subject: [PATCH 23/99] rename epbs db migration --- ...208225212_epbs-payload.sql => 20260108202212_epbs-payload.sql} | 0 ...208225212_epbs-payload.sql => 20260108202212_epbs-payload.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename db/schema/pgsql/{20250208225212_epbs-payload.sql => 20260108202212_epbs-payload.sql} (100%) rename db/schema/sqlite/{20250208225212_epbs-payload.sql => 20260108202212_epbs-payload.sql} (100%) diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql similarity index 100% rename from db/schema/pgsql/20250208225212_epbs-payload.sql rename to db/schema/pgsql/20260108202212_epbs-payload.sql diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20260108202212_epbs-payload.sql similarity index 100% rename from db/schema/sqlite/20250208225212_epbs-payload.sql rename to db/schema/sqlite/20260108202212_epbs-payload.sql From 45d7f48218e0a41b63e6583ae8c585adddaf726d Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 9 Jan 2026 14:21:12 +0100 Subject: [PATCH 24/99] index & show block bids --- clients/consensus/client.go | 72 +++--- clients/consensus/clientlogic.go | 15 +- clients/consensus/rpc/beaconstream.go | 47 +++- db/block_bids.go | 113 ++++++++++ .../pgsql/20260108202212_epbs-payload.sql | 25 +++ .../sqlite/20260108202212_epbs-payload.sql | 19 ++ dbtypes/dbtypes.go | 14 ++ go.mod | 2 +- go.sum | 4 +- handlers/slot.go | 60 +++++ indexer/beacon/bidcache.go | 211 ++++++++++++++++++ indexer/beacon/client.go | 35 ++- indexer/beacon/indexer.go | 15 ++ indexer/beacon/indexer_getter.go | 14 ++ templates/slot/bids.html | 33 +++ templates/slot/slot.html | 17 ++ types/models/slot.go | 17 ++ 17 files changed, 652 insertions(+), 61 deletions(-) create mode 100644 db/block_bids.go create mode 100644 indexer/beacon/bidcache.go create mode 100644 templates/slot/bids.html diff --git a/clients/consensus/client.go b/clients/consensus/client.go index d395b1db9..e641f0222 100644 --- a/clients/consensus/client.go +++ b/clients/consensus/client.go @@ -6,6 +6,7 @@ import ( "time" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" @@ -23,38 +24,39 @@ type ClientConfig struct { } type Client struct { - pool *Pool - clientIdx uint16 - endpointConfig *ClientConfig - clientCtx context.Context - clientCtxCancel context.CancelFunc - rpcClient *rpc.BeaconClient - logger *logrus.Entry - isOnline bool - isSyncing bool - isOptimistic bool - versionStr string - nodeIdentity *rpc.NodeIdentity - clientType ClientType - lastEvent time.Time - retryCounter uint64 - lastError error - headMutex sync.RWMutex - headRoot phase0.Root - headSlot phase0.Slot - justifiedRoot phase0.Root - justifiedEpoch phase0.Epoch - finalizedRoot phase0.Root - finalizedEpoch phase0.Epoch - lastFinalityUpdateEpoch phase0.Epoch - lastMetadataUpdateEpoch phase0.Epoch - lastMetadataUpdateTime time.Time - lastSyncUpdateEpoch phase0.Epoch - peers []*v1.Peer - blockDispatcher utils.Dispatcher[*v1.BlockEvent] - headDispatcher utils.Dispatcher[*v1.HeadEvent] - checkpointDispatcher utils.Dispatcher[*v1.Finality] - executionPayloadDispatcher utils.Dispatcher[*v1.ExecutionPayloadEvent] + pool *Pool + clientIdx uint16 + endpointConfig *ClientConfig + clientCtx context.Context + clientCtxCancel context.CancelFunc + rpcClient *rpc.BeaconClient + logger *logrus.Entry + isOnline bool + isSyncing bool + isOptimistic bool + versionStr string + nodeIdentity *rpc.NodeIdentity + clientType ClientType + lastEvent time.Time + retryCounter uint64 + lastError error + headMutex sync.RWMutex + headRoot phase0.Root + headSlot phase0.Slot + justifiedRoot phase0.Root + justifiedEpoch phase0.Epoch + finalizedRoot phase0.Root + finalizedEpoch phase0.Epoch + lastFinalityUpdateEpoch phase0.Epoch + lastMetadataUpdateEpoch phase0.Epoch + lastMetadataUpdateTime time.Time + lastSyncUpdateEpoch phase0.Epoch + peers []*v1.Peer + blockDispatcher utils.Dispatcher[*v1.BlockEvent] + headDispatcher utils.Dispatcher[*v1.HeadEvent] + checkpointDispatcher utils.Dispatcher[*v1.Finality] + executionPayloadDispatcher utils.Dispatcher[*v1.ExecutionPayloadAvailableEvent] + executionPayloadBidDispatcher utils.Dispatcher[*gloas.SignedExecutionPayloadBid] specWarnings []string // warnings from incomplete spec checks specs map[string]interface{} @@ -103,10 +105,14 @@ func (client *Client) SubscribeFinalizedEvent(capacity int) *utils.Subscription[ return client.checkpointDispatcher.Subscribe(capacity, false) } -func (client *Client) SubscribeExecutionPayloadEvent(capacity int, blocking bool) *utils.Subscription[*v1.ExecutionPayloadEvent] { +func (client *Client) SubscribeExecutionPayloadAvailableEvent(capacity int, blocking bool) *utils.Subscription[*v1.ExecutionPayloadAvailableEvent] { return client.executionPayloadDispatcher.Subscribe(capacity, blocking) } +func (client *Client) SubscribeExecutionPayloadBidEvent(capacity int, blocking bool) *utils.Subscription[*gloas.SignedExecutionPayloadBid] { + return client.executionPayloadBidDispatcher.Subscribe(capacity, blocking) +} + func (client *Client) GetPool() *Pool { return client.pool } diff --git a/clients/consensus/clientlogic.go b/clients/consensus/clientlogic.go index 716ca8762..59b938a62 100644 --- a/clients/consensus/clientlogic.go +++ b/clients/consensus/clientlogic.go @@ -8,6 +8,7 @@ import ( "time" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" @@ -177,10 +178,10 @@ func (client *Client) runClientLogic() error { } case rpc.StreamExecutionPayloadEvent: - err := client.processExecutionPayloadEvent(evt.Data.(*v1.ExecutionPayloadEvent)) - if err != nil { - client.logger.Warnf("failed processing execution payload event: %v", err) - } + client.executionPayloadDispatcher.Fire(evt.Data.(*v1.ExecutionPayloadAvailableEvent)) + + case rpc.StreamExecutionPayloadBidEvent: + client.executionPayloadBidDispatcher.Fire(evt.Data.(*gloas.SignedExecutionPayloadBid)) } client.logger.Tracef("event (%v) processing time: %v ms", evt.Event, time.Since(now).Milliseconds()) @@ -419,9 +420,3 @@ func (client *Client) pollClientHead() error { return nil } - -func (client *Client) processExecutionPayloadEvent(evt *v1.ExecutionPayloadEvent) error { - client.executionPayloadDispatcher.Fire(evt) - - return nil -} diff --git a/clients/consensus/rpc/beaconstream.go b/clients/consensus/rpc/beaconstream.go index 5020b5ccc..8c91b7aef 100644 --- a/clients/consensus/rpc/beaconstream.go +++ b/clients/consensus/rpc/beaconstream.go @@ -10,6 +10,7 @@ import ( "time" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/donovanhide/eventsource" "github.com/sirupsen/logrus" @@ -17,10 +18,11 @@ import ( ) const ( - StreamBlockEvent uint16 = 0x01 - StreamHeadEvent uint16 = 0x02 - StreamFinalizedEvent uint16 = 0x04 - StreamExecutionPayloadEvent uint16 = 0x08 + StreamBlockEvent uint16 = 0x01 + StreamHeadEvent uint16 = 0x02 + StreamFinalizedEvent uint16 = 0x04 + StreamExecutionPayloadEvent uint16 = 0x08 + StreamExecutionPayloadBidEvent uint16 = 0x10 ) type BeaconStreamEvent struct { @@ -88,8 +90,10 @@ func (bs *BeaconStream) startStream() { bs.processHeadEvent(evt) case "finalized_checkpoint": bs.processFinalizedEvent(evt) - case "execution_payload": - bs.processExecutionPayloadEvent(evt) + case "execution_payload_available": + bs.processExecutionPayloadAvailableEvent(evt) + case "execution_payload_bid": + bs.processExecutionPayloadBidEvent(evt) } case <-stream.Ready: bs.ReadyChan <- &BeaconStreamStatus{ @@ -156,7 +160,17 @@ func (bs *BeaconStream) subscribeStream(endpoint string, events uint16) *eventst fmt.Fprintf(&topics, ",") } - fmt.Fprintf(&topics, "execution_payload") + fmt.Fprintf(&topics, "execution_payload_available") + + topicsCount++ + } + + if events&StreamExecutionPayloadBidEvent > 0 { + if topicsCount > 0 { + fmt.Fprintf(&topics, ",") + } + + fmt.Fprintf(&topics, "execution_payload_bid") topicsCount++ } @@ -238,8 +252,8 @@ func (bs *BeaconStream) processFinalizedEvent(evt eventsource.Event) { } } -func (bs *BeaconStream) processExecutionPayloadEvent(evt eventsource.Event) { - var parsed v1.ExecutionPayloadEvent +func (bs *BeaconStream) processExecutionPayloadAvailableEvent(evt eventsource.Event) { + var parsed v1.ExecutionPayloadAvailableEvent err := json.Unmarshal([]byte(evt.Data()), &parsed) if err != nil { @@ -253,6 +267,21 @@ func (bs *BeaconStream) processExecutionPayloadEvent(evt eventsource.Event) { } } +func (bs *BeaconStream) processExecutionPayloadBidEvent(evt eventsource.Event) { + var parsed gloas.SignedExecutionPayloadBid + + err := json.Unmarshal([]byte(evt.Data()), &parsed) + if err != nil { + bs.logger.Warnf("beacon block stream failed to decode execution_payload_bid event: %v", err) + return + } + + bs.EventChan <- &BeaconStreamEvent{ + Event: StreamExecutionPayloadBidEvent, + Data: &parsed, + } +} + func getRedactedURL(requrl string) string { var logurl string diff --git a/db/block_bids.go b/db/block_bids.go new file mode 100644 index 000000000..bfcfac474 --- /dev/null +++ b/db/block_bids.go @@ -0,0 +1,113 @@ +package db + +import ( + "fmt" + "strings" + + "github.com/ethpandaops/dora/dbtypes" + "github.com/jmoiron/sqlx" +) + +func InsertBids(bids []*dbtypes.BlockBid, tx *sqlx.Tx) error { + var sql strings.Builder + fmt.Fprint(&sql, + EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: "INSERT INTO block_bids ", + dbtypes.DBEngineSqlite: "INSERT OR REPLACE INTO block_bids ", + }), + "(parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment)", + " VALUES ", + ) + argIdx := 0 + fieldCount := 9 + + args := make([]any, len(bids)*fieldCount) + for i, bid := range bids { + if i > 0 { + fmt.Fprintf(&sql, ", ") + } + fmt.Fprintf(&sql, "(") + for f := 0; f < fieldCount; f++ { + if f > 0 { + fmt.Fprintf(&sql, ", ") + } + fmt.Fprintf(&sql, "$%v", argIdx+f+1) + } + fmt.Fprintf(&sql, ")") + + args[argIdx+0] = bid.ParentRoot + args[argIdx+1] = bid.ParentHash + args[argIdx+2] = bid.BlockHash + args[argIdx+3] = bid.FeeRecipient + args[argIdx+4] = bid.GasLimit + args[argIdx+5] = bid.BuilderIndex + args[argIdx+6] = bid.Slot + args[argIdx+7] = bid.Value + args[argIdx+8] = bid.ElPayment + argIdx += fieldCount + } + fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: " ON CONFLICT (parent_root, parent_hash, block_hash, builder_index) DO UPDATE SET " + + "fee_recipient = excluded.fee_recipient, " + + "gas_limit = excluded.gas_limit, " + + "slot = excluded.slot, " + + "value = excluded.value, " + + "el_payment = excluded.el_payment", + dbtypes.DBEngineSqlite: "", + })) + + _, err := tx.Exec(sql.String(), args...) + if err != nil { + return err + } + return nil +} + +func GetBidsForBlockRoot(blockRoot []byte) []*dbtypes.BlockBid { + var sql strings.Builder + args := []any{ + blockRoot, + } + fmt.Fprint(&sql, ` + SELECT + parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment + FROM block_bids + WHERE parent_root = $1 + ORDER BY value DESC + `) + + bids := []*dbtypes.BlockBid{} + err := ReaderDb.Select(&bids, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching bids for block root: %v", err) + return nil + } + return bids +} + +func GetBidsForSlotRange(minSlot uint64) []*dbtypes.BlockBid { + var sql strings.Builder + args := []any{ + minSlot, + } + fmt.Fprint(&sql, ` + SELECT + parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment + FROM block_bids + WHERE slot >= $1 + ORDER BY slot DESC, value DESC + `) + + bids := []*dbtypes.BlockBid{} + err := ReaderDb.Select(&bids, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching bids for slot range: %v", err) + return nil + } + return bids +} + +func DeleteBidsBeforeSlot(minSlot uint64, tx *sqlx.Tx) error { + _, err := tx.Exec(`DELETE FROM block_bids WHERE slot < $1`, minSlot) + return err +} diff --git a/db/schema/pgsql/20260108202212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql index bccb6ba96..7432fb9e1 100644 --- a/db/schema/pgsql/20260108202212_epbs-payload.sql +++ b/db/schema/pgsql/20260108202212_epbs-payload.sql @@ -22,6 +22,31 @@ ALTER TABLE public."epochs" ADD ALTER TABLE public."unfinalized_epochs" ADD "payload_count" int NOT NULL DEFAULT 0; +CREATE TABLE IF NOT EXISTS public."block_bids" ( + "parent_root" bytea NOT NULL, + "parent_hash" bytea NOT NULL, + "block_hash" bytea NOT NULL, + "fee_recipient" bytea NOT NULL, + "gas_limit" bigint NOT NULL, + "builder_index" bigint NOT NULL, + "slot" bigint NOT NULL, + "value" bigint NOT NULL, + "el_payment" bigint NOT NULL, + CONSTRAINT block_bids_pkey PRIMARY KEY (parent_root, parent_hash, block_hash, builder_index) +); + +CREATE INDEX IF NOT EXISTS "block_bids_parent_root_idx" + ON public."block_bids" + ("parent_root" ASC NULLS LAST); + +CREATE INDEX IF NOT EXISTS "block_bids_builder_index_idx" + ON public."block_bids" + ("builder_index" ASC NULLS LAST); + +CREATE INDEX IF NOT EXISTS "block_bids_slot_idx" + ON public."block_bids" + ("slot" ASC NULLS LAST); + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/schema/sqlite/20260108202212_epbs-payload.sql b/db/schema/sqlite/20260108202212_epbs-payload.sql index 8dc13a74f..41f1343c4 100644 --- a/db/schema/sqlite/20260108202212_epbs-payload.sql +++ b/db/schema/sqlite/20260108202212_epbs-payload.sql @@ -15,6 +15,25 @@ ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; ALTER TABLE "unfinalized_epochs" ADD "payload_count" int NOT NULL DEFAULT 0; +CREATE TABLE IF NOT EXISTS "block_bids" ( + "parent_root" BLOB NOT NULL, + "parent_hash" BLOB NOT NULL, + "block_hash" BLOB NOT NULL, + "fee_recipient" BLOB NOT NULL, + "gas_limit" BIGINT NOT NULL, + "builder_index" BIGINT NOT NULL, + "slot" BIGINT NOT NULL, + "value" BIGINT NOT NULL, + "el_payment" BIGINT NOT NULL, + CONSTRAINT block_bids_pkey PRIMARY KEY (parent_root, parent_hash, block_hash, builder_index) +); + +CREATE INDEX IF NOT EXISTS "block_bids_parent_root_idx" ON "block_bids" ("parent_root" ASC); + +CREATE INDEX IF NOT EXISTS "block_bids_builder_index_idx" ON "block_bids" ("builder_index" ASC); + +CREATE INDEX IF NOT EXISTS "block_bids_slot_idx" ON "block_bids" ("slot" ASC); + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index e3a8ee891..cadaa7f15 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -545,6 +545,20 @@ type ElTokenTransfer struct { AmountRaw []byte `db:"amount_raw"` } +// ePBS types + +type BlockBid struct { + ParentRoot []byte `db:"parent_root"` + ParentHash []byte `db:"parent_hash"` + BlockHash []byte `db:"block_hash"` + FeeRecipient []byte `db:"fee_recipient"` + GasLimit uint64 `db:"gas_limit"` + BuilderIndex uint64 `db:"builder_index"` + Slot uint64 `db:"slot"` + Value uint64 `db:"value"` + ElPayment uint64 `db:"el_payment"` +} + // Withdrawal types const ( WithdrawalTypeBeaconWithdrawal = 0 diff --git a/go.mod b/go.mod index 872d6b852..739c87205 100644 --- a/go.mod +++ b/go.mod @@ -260,4 +260,4 @@ require ( modernc.org/sqlite v1.38.2 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1 diff --git a/go.sum b/go.sum index 94585ab4f..615b00d21 100644 --- a/go.sum +++ b/go.sum @@ -566,8 +566,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.0 h1:25Kb7CQKKkh8r8mlj/exNJY5vSoe3wGm/8QpsfayRtM= github.com/pk910/dynamic-ssz v1.2.0/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c h1:1ZlGOxqx5QOTus5NGPZH/dVqQ7tnGrqituiQYnOh0eY= -github.com/pk910/go-eth2-client v0.0.0-20260108190445-f2788345b80c/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= +github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1 h1:Obn5KbqFo+T0Sr8fRGapWbKU5c61twez6ei5LQqA0gE= +github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/handlers/slot.go b/handlers/slot.go index 90205719f..6a9ca2efd 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -45,6 +45,7 @@ func Slot(w http.ResponseWriter, r *http.Request) { "slot/deposit_requests.html", "slot/withdrawal_requests.html", "slot/consolidation_requests.html", + "slot/bids.html", ) var notfoundTemplateFiles = append(layoutTemplateFiles, "slot/notfound.html", @@ -793,6 +794,11 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } } + // Load execution payload bids for ePBS (gloas+) blocks + if blockData.Block.Version >= spec.DataVersionGloas { + getSlotPageBids(pageData) + } + return pageData } @@ -994,6 +1000,60 @@ func getSlotPageConsolidationRequests(pageData *models.SlotPageBlockData, consol pageData.ConsolidationRequestsCount = uint64(len(pageData.ConsolidationRequests)) } +func getSlotPageBids(pageData *models.SlotPageBlockData) { + beaconIndexer := services.GlobalBeaconService.GetBeaconIndexer() + bids := beaconIndexer.GetBlockBids(phase0.Root(pageData.ParentRoot)) + + pageData.Bids = make([]*models.SlotPageBid, 0, len(bids)) + + // Get the winning block hash for comparison + var winningBlockHash []byte + if pageData.ExecutionData != nil { + winningBlockHash = pageData.ExecutionData.BlockHash + } + + for _, bid := range bids { + bidData := &models.SlotPageBid{ + ParentRoot: bid.ParentRoot, + ParentHash: bid.ParentHash, + BlockHash: bid.BlockHash, + FeeRecipient: bid.FeeRecipient, + GasLimit: bid.GasLimit, + BuilderIndex: bid.BuilderIndex, + BuilderName: services.GlobalBeaconService.GetValidatorName(bid.BuilderIndex), + Slot: bid.Slot, + Value: bid.Value, + ElPayment: bid.ElPayment, + TotalValue: bid.Value + bid.ElPayment, + } + + // Check if this is the winning bid + if winningBlockHash != nil && len(bid.BlockHash) == len(winningBlockHash) { + isWinning := true + for i := range bid.BlockHash { + if bid.BlockHash[i] != winningBlockHash[i] { + isWinning = false + break + } + } + bidData.IsWinning = isWinning + } + + pageData.Bids = append(pageData.Bids, bidData) + } + + // Sort by total value (value + el_payment) descending + for i := 0; i < len(pageData.Bids)-1; i++ { + for j := i + 1; j < len(pageData.Bids); j++ { + if pageData.Bids[j].TotalValue > pageData.Bids[i].TotalValue { + pageData.Bids[i], pageData.Bids[j] = pageData.Bids[j], pageData.Bids[i] + } + } + } + + pageData.BidsCount = uint64(len(pageData.Bids)) +} + func handleSlotDownload(ctx context.Context, w http.ResponseWriter, blockSlot int64, blockRoot []byte, downloadType string) error { chainState := services.GlobalBeaconService.GetChainState() currentSlot := chainState.CurrentSlot() diff --git a/indexer/beacon/bidcache.go b/indexer/beacon/bidcache.go new file mode 100644 index 000000000..ce4c5ac65 --- /dev/null +++ b/indexer/beacon/bidcache.go @@ -0,0 +1,211 @@ +package beacon + +import ( + "sync" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/dora/dbtypes" + "github.com/jmoiron/sqlx" +) + +const ( + // bidCacheMaxSlots is the maximum number of slots to keep in the cache + bidCacheMaxSlots = 15 + // bidCacheFlushThreshold is the slot span that triggers a flush + bidCacheFlushThreshold = 15 + // bidCacheRetainSlots is the number of slots to retain after a flush + bidCacheRetainSlots = 10 +) + +// bidCacheKey uniquely identifies a bid in the cache +type bidCacheKey struct { + ParentRoot phase0.Root + ParentHash phase0.Hash32 + BlockHash phase0.Hash32 + BuilderIndex uint64 +} + +// blockBidCache caches execution payload bids for recent blocks. +// Bids for older slots are ignored. The cache is flushed to DB on shutdown +// or when the slot span exceeds the threshold. +type blockBidCache struct { + indexer *Indexer + cacheMutex sync.RWMutex + bids map[bidCacheKey]*dbtypes.BlockBid + minSlot phase0.Slot + maxSlot phase0.Slot +} + +// newBlockBidCache creates a new instance of blockBidCache. +func newBlockBidCache(indexer *Indexer) *blockBidCache { + return &blockBidCache{ + indexer: indexer, + bids: make(map[bidCacheKey]*dbtypes.BlockBid, 64), + } +} + +// loadFromDB loads bids from the last N slots from the database. +func (cache *blockBidCache) loadFromDB(currentSlot phase0.Slot) { + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + + minSlot := phase0.Slot(0) + if currentSlot > bidCacheRetainSlots { + minSlot = currentSlot - bidCacheRetainSlots + } + + dbBids := db.GetBidsForSlotRange(uint64(minSlot)) + for _, bid := range dbBids { + key := bidCacheKey{ + ParentRoot: phase0.Root(bid.ParentRoot), + ParentHash: phase0.Hash32(bid.ParentHash), + BlockHash: phase0.Hash32(bid.BlockHash), + BuilderIndex: bid.BuilderIndex, + } + cache.bids[key] = bid + + slot := phase0.Slot(bid.Slot) + if cache.minSlot == 0 || slot < cache.minSlot { + cache.minSlot = slot + } + if slot > cache.maxSlot { + cache.maxSlot = slot + } + } + + if len(dbBids) > 0 { + cache.indexer.logger.Infof("loaded %d bids from DB (slots %d-%d)", len(dbBids), cache.minSlot, cache.maxSlot) + } +} + +// AddBid adds a bid to the cache. Returns true if the bid was added, +// false if it was ignored (too old) or already exists. +func (cache *blockBidCache) AddBid(bid *dbtypes.BlockBid) bool { + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + + slot := phase0.Slot(bid.Slot) + + // Ignore bids for slots that are too old + if cache.maxSlot > 0 && slot+bidCacheMaxSlots < cache.maxSlot { + return false + } + + key := bidCacheKey{ + ParentRoot: phase0.Root(bid.ParentRoot), + ParentHash: phase0.Hash32(bid.ParentHash), + BlockHash: phase0.Hash32(bid.BlockHash), + BuilderIndex: bid.BuilderIndex, + } + + // Check if bid already exists + if _, exists := cache.bids[key]; exists { + return false + } + + cache.bids[key] = bid + + // Update slot bounds + if cache.minSlot == 0 || slot < cache.minSlot { + cache.minSlot = slot + } + if slot > cache.maxSlot { + cache.maxSlot = slot + } + + return true +} + +// GetBidsForBlockRoot returns all bids for a given parent block root. +func (cache *blockBidCache) GetBidsForBlockRoot(blockRoot phase0.Root) []*dbtypes.BlockBid { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + result := make([]*dbtypes.BlockBid, 0) + for key, bid := range cache.bids { + if key.ParentRoot == blockRoot { + result = append(result, bid) + } + } + + return result +} + +// checkAndFlush checks if the cache needs to be flushed and performs the flush if necessary. +// This should be called periodically (e.g., on each new block). +func (cache *blockBidCache) checkAndFlush() error { + cache.cacheMutex.Lock() + + // Check if we need to flush + if cache.maxSlot == 0 || cache.maxSlot-cache.minSlot < bidCacheFlushThreshold { + cache.cacheMutex.Unlock() + return nil + } + + // Calculate the cutoff slot - we'll flush bids older than this + cutoffSlot := cache.maxSlot - bidCacheRetainSlots + + // Collect bids to flush (from minSlot to cutoffSlot) + bidsToFlush := make([]*dbtypes.BlockBid, 0) + for key, bid := range cache.bids { + if phase0.Slot(bid.Slot) < cutoffSlot { + bidsToFlush = append(bidsToFlush, bid) + delete(cache.bids, key) + } + } + + // Update minSlot + cache.minSlot = cutoffSlot + + cache.cacheMutex.Unlock() + + // Write to DB outside of lock + if len(bidsToFlush) > 0 { + err := db.RunDBTransaction(func(tx *sqlx.Tx) error { + return db.InsertBids(bidsToFlush, tx) + }) + if err != nil { + cache.indexer.logger.Errorf("error flushing bids to db: %v", err) + return err + } + cache.indexer.logger.Debugf("flushed %d bids to DB (slots < %d)", len(bidsToFlush), cutoffSlot) + } + + return nil +} + +// flushAll flushes all cached bids to the database. +// This should be called on shutdown. +func (cache *blockBidCache) flushAll() error { + cache.cacheMutex.Lock() + + if len(cache.bids) == 0 { + cache.cacheMutex.Unlock() + return nil + } + + bidsToFlush := make([]*dbtypes.BlockBid, 0, len(cache.bids)) + for _, bid := range cache.bids { + bidsToFlush = append(bidsToFlush, bid) + } + + // Clear the cache + cache.bids = make(map[bidCacheKey]*dbtypes.BlockBid, 64) + cache.minSlot = 0 + cache.maxSlot = 0 + + cache.cacheMutex.Unlock() + + // Write to DB outside of lock + err := db.RunDBTransaction(func(tx *sqlx.Tx) error { + return db.InsertBids(bidsToFlush, tx) + }) + if err != nil { + cache.indexer.logger.Errorf("error flushing all bids to db: %v", err) + return err + } + + cache.indexer.logger.Infof("flushed %d bids to DB on shutdown", len(bidsToFlush)) + return nil +} diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 68b099925..895e8c6d8 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -33,9 +33,10 @@ type Client struct { archive bool skipValidators bool - blockSubscription *utils.Subscription[*v1.BlockEvent] - headSubscription *utils.Subscription[*v1.HeadEvent] - executionPayloadSubscription *utils.Subscription[*v1.ExecutionPayloadEvent] + blockSubscription *utils.Subscription[*v1.BlockEvent] + headSubscription *utils.Subscription[*v1.HeadEvent] + executionPayloadSubscription *utils.Subscription[*v1.ExecutionPayloadAvailableEvent] + executionPayloadBidSubscription *utils.Subscription[*gloas.SignedExecutionPayloadBid] headRoot phase0.Root } @@ -83,7 +84,8 @@ func (c *Client) startIndexing() { // blocking block subscription with a buffer to ensure no blocks are missed c.blockSubscription = c.client.SubscribeBlockEvent(100, true) c.headSubscription = c.client.SubscribeHeadEvent(100, true) - c.executionPayloadSubscription = c.client.SubscribeExecutionPayloadEvent(100, true) + c.executionPayloadSubscription = c.client.SubscribeExecutionPayloadAvailableEvent(100, true) + c.executionPayloadBidSubscription = c.client.SubscribeExecutionPayloadBidEvent(100, true) go c.startClientLoop() } @@ -183,10 +185,15 @@ func (c *Client) runClientLoop() error { c.logger.Errorf("failed processing head %v (%v): %v", headEvent.Slot, headEvent.Block.String(), err) } case executionPayloadEvent := <-c.executionPayloadSubscription.Channel(): - err := c.processExecutionPayloadEvent(executionPayloadEvent) + err := c.processExecutionPayloadAvailableEvent(executionPayloadEvent) if err != nil { c.logger.Errorf("failed processing execution payload %v (%v): %v", executionPayloadEvent.Slot, executionPayloadEvent.BlockRoot.String(), err) } + case executionPayloadBidEvent := <-c.executionPayloadBidSubscription.Channel(): + err := c.processExecutionPayloadBidEvent(executionPayloadBidEvent) + if err != nil { + c.logger.Errorf("failed processing execution payload bid %v (%v): %v", executionPayloadBidEvent.Message.Slot, executionPayloadBidEvent.Message.ParentBlockRoot.String(), err) + } } } @@ -582,7 +589,7 @@ func (c *Client) backfillParentBlocks(headBlock *Block) error { } // processExecutionPayloadEvent processes an execution payload event from the event stream. -func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.ExecutionPayloadEvent) error { +func (c *Client) processExecutionPayloadAvailableEvent(executionPayloadEvent *v1.ExecutionPayloadAvailableEvent) error { if c.client.GetStatus() != consensus.ClientStatusOnline && c.client.GetStatus() != consensus.ClientStatusOptimistic { // client is not ready, skip return nil @@ -648,3 +655,19 @@ func (c *Client) persistExecutionPayload(block *Block) error { return nil }) } + +func (c *Client) processExecutionPayloadBidEvent(executionPayloadBidEvent *gloas.SignedExecutionPayloadBid) error { + bid := &dbtypes.BlockBid{ + ParentRoot: executionPayloadBidEvent.Message.ParentBlockRoot[:], + ParentHash: executionPayloadBidEvent.Message.ParentBlockHash[:], + BlockHash: executionPayloadBidEvent.Message.BlockHash[:], + FeeRecipient: executionPayloadBidEvent.Message.FeeRecipient[:], + GasLimit: uint64(executionPayloadBidEvent.Message.GasLimit), + BuilderIndex: uint64(executionPayloadBidEvent.Message.BuilderIndex), + Slot: uint64(executionPayloadBidEvent.Message.Slot), + Value: uint64(executionPayloadBidEvent.Message.Value), + ElPayment: uint64(executionPayloadBidEvent.Message.ExecutionPayment), + } + c.indexer.blockBidCache.AddBid(bid) + return nil +} diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index a5ff02a42..16f2aba02 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -46,6 +46,7 @@ type Indexer struct { pubkeyCache *pubkeyCache validatorCache *validatorCache validatorActivity *validatorActivityCache + blockBidCache *blockBidCache // indexer state clients []*Client @@ -116,6 +117,7 @@ func NewIndexer(logger logrus.FieldLogger, consensusPool *consensus.Pool) *Index indexer.pubkeyCache = newPubkeyCache(indexer, utils.Config.Indexer.PubkeyCachePath) indexer.validatorCache = newValidatorCache(indexer) indexer.validatorActivity = newValidatorActivityCache(indexer) + indexer.blockBidCache = newBlockBidCache(indexer) indexer.dbWriter = newDbWriter(indexer) badChainRoots := utils.Config.Indexer.BadChainRoots @@ -416,6 +418,9 @@ func (indexer *Indexer) StartIndexer() { indexer.logger.Infof("restored %v unfinalized blocks from DB (%v with bodies, %.3f sec)", restoredBlockCount, restoredBodyCount, time.Since(t1).Seconds()) } + // restore block bids from db + indexer.blockBidCache.loadFromDB(chainState.CurrentSlot()) + // start indexing for all clients for _, client := range indexer.clients { client.startIndexing() @@ -450,6 +455,11 @@ func (indexer *Indexer) StartIndexer() { } func (indexer *Indexer) StopIndexer() { + // flush block bids to db before shutdown + if err := indexer.blockBidCache.flushAll(); err != nil { + indexer.logger.WithError(err).Errorf("error flushing block bids on shutdown") + } + indexer.pubkeyCache.Close() } @@ -501,6 +511,11 @@ func (indexer *Indexer) runIndexerLoop() { slotIndex := chainState.SlotToSlotIndex(phase0.Slot(slotEvent.Number())) slotProgress := uint8(100 / chainState.GetSpecs().SlotsPerEpoch * uint64(slotIndex)) + // flush old block bids if needed + if err := indexer.blockBidCache.checkAndFlush(); err != nil { + indexer.logger.WithError(err).Errorf("failed flushing block bids") + } + // precalc next canonical duties on epoch start if epoch >= indexer.lastPrecalcRunEpoch { err := indexer.precalcNextEpochStats(epoch) diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index a403b3eb3..64ef00973 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -12,6 +12,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/dora/dbtypes" dynssz "github.com/pk910/dynamic-ssz" ) @@ -507,3 +508,16 @@ func (indexer *Indexer) GetFullValidatorByIndex(validatorIndex phase0.ValidatorI return validatorData } + +// GetBlockBids returns the execution payload bids for a given parent block root. +// It first checks the in-memory cache, then falls back to the database. +func (indexer *Indexer) GetBlockBids(parentBlockRoot phase0.Root) []*dbtypes.BlockBid { + // First check the in-memory cache + bids := indexer.blockBidCache.GetBidsForBlockRoot(parentBlockRoot) + if len(bids) > 0 { + return bids + } + + // Fall back to database + return db.GetBidsForBlockRoot(parentBlockRoot[:]) +} diff --git a/templates/slot/bids.html b/templates/slot/bids.html new file mode 100644 index 000000000..e0e1770c6 --- /dev/null +++ b/templates/slot/bids.html @@ -0,0 +1,33 @@ +{{ define "block_bids" }} +
+ + + + + + + + + + + + + + {{ range $i, $bid := .Block.Bids }} + + + + + + + + + + {{ end }} + +
BuilderBlock HashFee RecipientGas LimitValueEL PaymentTotal
+ {{ formatValidatorWithIndex $bid.BuilderIndex $bid.BuilderName }} + {{ if $bid.IsWinning }}Winner{{ end }} + 0x{{ printf "%.16x" $bid.BlockHash }}...{{ ethAddressLink $bid.FeeRecipient }}{{ formatAddCommas $bid.GasLimit }}{{ formatEthFromGwei $bid.Value }}{{ formatEthFromGwei $bid.ElPayment }}{{ formatEthFromGwei $bid.TotalValue }}
+
+{{ end }} diff --git a/templates/slot/slot.html b/templates/slot/slot.html index f14d0f202..9de932940 100644 --- a/templates/slot/slot.html +++ b/templates/slot/slot.html @@ -96,6 +96,11 @@

Consolidation Requests {{ .Block.ConsolidationRequestsCount }} {{ end }} + {{ if gt .Block.BidsCount 0 }} +
  • + Bids {{ .Block.BidsCount }} +
    • + {{ end }} {{ if .Block }}
  • @@ -258,6 +263,18 @@

    Showing {{ .Block.ConsolidationRequestsC {{ template "block_consolidation_requests" . }}

    • {{ end }} + {{ if gt .Block.BidsCount 0 }} +
    +
    +
    +
    +

    Showing {{ .Block.BidsCount }} Execution Payload Bids

    +
    +
    + {{ template "block_bids" . }} +
    +
    + {{ end }} {{ if .Block }}
    diff --git a/types/models/slot.go b/types/models/slot.go index b2ef489fa..6a262a40e 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -68,6 +68,7 @@ type SlotPageBlockData struct { DepositRequestsCount uint64 `json:"deposit_receipts_count"` WithdrawalRequestsCount uint64 `json:"withdrawal_requests_count"` ConsolidationRequestsCount uint64 `json:"consolidation_requests_count"` + BidsCount uint64 `json:"bids_count"` PayloadHeader *SlotPagePayloadHeader `json:"payload_header"` ExecutionData *SlotPageExecutionData `json:"execution_data"` @@ -84,6 +85,7 @@ type SlotPageBlockData struct { DepositRequests []*SlotPageDepositRequest `json:"deposit_receipts"` // DepositRequests included in this block WithdrawalRequests []*SlotPageWithdrawalRequest `json:"withdrawal_requests"` // WithdrawalRequests included in this block ConsolidationRequests []*SlotPageConsolidationRequest `json:"consolidation_requests"` // ConsolidationRequests included in this block + Bids []*SlotPageBid `json:"bids"` // Execution payload bids for this block (ePBS) } type SlotPageExecutionData struct { @@ -285,3 +287,18 @@ type SlotPageConsolidationRequest struct { TargetName string `db:"target_name"` Epoch uint64 `db:"epoch"` } + +type SlotPageBid struct { + ParentRoot []byte `json:"parent_root"` + ParentHash []byte `json:"parent_hash"` + BlockHash []byte `json:"block_hash"` + FeeRecipient []byte `json:"fee_recipient"` + GasLimit uint64 `json:"gas_limit"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` + Slot uint64 `json:"slot"` + Value uint64 `json:"value"` + ElPayment uint64 `json:"el_payment"` + TotalValue uint64 `json:"total_value"` + IsWinning bool `json:"is_winning"` +} From b1d5cc55c11a35904d84f122d5fdaa06ebe449bf Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 14 Jan 2026 21:54:16 +0100 Subject: [PATCH 25/99] rework blockdb to serve as long term tired bal/payload store --- blockdb/blockdb.go | 77 ++++- blockdb/pebble/cleanup.go | 439 +++++++++++++++++++++++++++ blockdb/pebble/pebble.go | 340 ++++++++++++--------- blockdb/s3/format.go | 202 +++++++++++++ blockdb/s3/s3store.go | 507 +++++++++++++++++++++++++------- blockdb/tiered/tiered.go | 278 +++++++++++++++++ blockdb/types/engine.go | 52 +++- blockdb/types/flags.go | 38 +++ cmd/dora-utils/blockdb_sync.go | 2 +- indexer/beacon/block.go | 2 +- services/chainservice.go | 7 + services/chainservice_blocks.go | 29 +- types/config.go | 33 ++- 13 files changed, 1717 insertions(+), 289 deletions(-) create mode 100644 blockdb/pebble/cleanup.go create mode 100644 blockdb/s3/format.go create mode 100644 blockdb/tiered/tiered.go create mode 100644 blockdb/types/flags.go diff --git a/blockdb/blockdb.go b/blockdb/blockdb.go index 2e13cb6ce..f31fa2273 100644 --- a/blockdb/blockdb.go +++ b/blockdb/blockdb.go @@ -3,18 +3,24 @@ package blockdb import ( "context" + "github.com/sirupsen/logrus" + "github.com/ethpandaops/dora/blockdb/pebble" "github.com/ethpandaops/dora/blockdb/s3" + "github.com/ethpandaops/dora/blockdb/tiered" "github.com/ethpandaops/dora/blockdb/types" dtypes "github.com/ethpandaops/dora/types" ) +// BlockDb is the main wrapper for block database operations. type BlockDb struct { engine types.BlockDbEngine } +// GlobalBlockDb is the global block database instance. var GlobalBlockDb *BlockDb +// InitWithPebble initializes the block database with Pebble (local) storage. func InitWithPebble(config dtypes.PebbleBlockDBConfig) error { engine, err := pebble.NewPebbleEngine(config) if err != nil { @@ -28,6 +34,7 @@ func InitWithPebble(config dtypes.PebbleBlockDBConfig) error { return nil } +// InitWithS3 initializes the block database with S3 (remote) storage. func InitWithS3(config dtypes.S3BlockDBConfig) error { engine, err := s3.NewS3Engine(config) if err != nil { @@ -41,27 +48,77 @@ func InitWithS3(config dtypes.S3BlockDBConfig) error { return nil } +// InitWithTiered initializes the block database with tiered storage (Pebble cache + S3 backend). +func InitWithTiered(config dtypes.TieredBlockDBConfig, logger logrus.FieldLogger) error { + engine, err := tiered.NewTieredEngine(config, logger) + if err != nil { + return err + } + + GlobalBlockDb = &BlockDb{ + engine: engine, + } + + return nil +} + +// Close closes the block database. func (db *BlockDb) Close() error { return db.engine.Close() } -func (db *BlockDb) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { - return db.engine.GetBlock(ctx, slot, root, parseBlock, parsePayload) +// GetBlock retrieves block data with selective loading based on flags. +func (db *BlockDb) GetBlock( + ctx context.Context, + slot uint64, + root []byte, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { + return db.engine.GetBlock(ctx, slot, root, flags, parseBlock, parsePayload) +} + +// GetStoredComponents returns which components exist for a block. +func (db *BlockDb) GetStoredComponents(ctx context.Context, slot uint64, root []byte) (types.BlockDataFlags, error) { + return db.engine.GetStoredComponents(ctx, slot, root) } -func (db *BlockDb) AddBlock(ctx context.Context, slot uint64, root []byte, header_ver uint64, header_data []byte, body_ver uint64, body_data []byte, payload_ver uint64, payload_data []byte) (bool, error) { +// AddBlock stores block data. Returns (added, updated, error). +func (db *BlockDb) AddBlock( + ctx context.Context, + slot uint64, + root []byte, + headerVer uint64, + headerData []byte, + bodyVer uint64, + bodyData []byte, + payloadVer uint64, + payloadData []byte, + balVer uint64, + balData []byte, +) (bool, bool, error) { return db.engine.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { return &types.BlockData{ - HeaderVersion: header_ver, - HeaderData: header_data, - BodyVersion: body_ver, - BodyData: body_data, - PayloadVersion: payload_ver, - PayloadData: payload_data, + HeaderVersion: headerVer, + HeaderData: headerData, + BodyVersion: bodyVer, + BodyData: bodyData, + PayloadVersion: payloadVer, + PayloadData: payloadData, + BalVersion: balVer, + BalData: balData, }, nil }) } -func (db *BlockDb) AddBlockWithCallback(ctx context.Context, slot uint64, root []byte, dataCb func() (*types.BlockData, error)) (bool, error) { +// AddBlockWithCallback stores block data using a callback for deferred data loading. +// Returns (added, updated, error). +func (db *BlockDb) AddBlockWithCallback( + ctx context.Context, + slot uint64, + root []byte, + dataCb func() (*types.BlockData, error), +) (bool, bool, error) { return db.engine.AddBlock(ctx, slot, root, dataCb) } diff --git a/blockdb/pebble/cleanup.go b/blockdb/pebble/cleanup.go new file mode 100644 index 000000000..5a3cf787e --- /dev/null +++ b/blockdb/pebble/cleanup.go @@ -0,0 +1,439 @@ +package pebble + +import ( + "context" + "encoding/binary" + "sort" + "sync" + "time" + + "github.com/cockroachdb/pebble" + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/dora/blockdb/types" + dtypes "github.com/ethpandaops/dora/types" +) + +const ( + // KeyNamespaceLRU is the namespace for LRU tracking data. + KeyNamespaceLRU uint16 = 2 + + // LRU value format: [headerAccess (8B)] [bodyAccess (8B)] [payloadAccess (8B)] [balAccess (8B)] + // Each access time is a Unix nanosecond timestamp, 0 means never accessed. + lruValueSize = 32 + + // Maximum number of LRU updates to buffer before forcing a flush. + maxLRUBufferSize = 1000 +) + +// CacheCleanup manages background cleanup of cached data. +type CacheCleanup struct { + engine *PebbleEngine + config dtypes.PebbleBlockDBConfig + logger logrus.FieldLogger + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + // LRU update buffer + lruMu sync.Mutex + lruBuffer map[string]*lruUpdate // root hex -> update +} + +// lruUpdate holds pending LRU timestamp updates for a block. +type lruUpdate struct { + root []byte + headerAccess int64 // Unix nano, 0 = no update + bodyAccess int64 + payloadAccess int64 + balAccess int64 +} + +// NewCacheCleanup creates a new cache cleanup manager. +func NewCacheCleanup(engine *PebbleEngine, logger logrus.FieldLogger) *CacheCleanup { + ctx, cancel := context.WithCancel(context.Background()) + + return &CacheCleanup{ + engine: engine, + config: engine.GetConfig(), + logger: logger.WithField("component", "pebble-cleanup"), + ctx: ctx, + cancel: cancel, + lruBuffer: make(map[string]*lruUpdate, 100), + } +} + +// Start begins the background cleanup loop. +func (c *CacheCleanup) Start() { + if c.config.CleanupInterval == 0 { + c.logger.Info("cleanup disabled (interval is 0)") + return + } + + c.wg.Add(1) + go c.runCleanupLoop() +} + +// Stop stops the background cleanup loop. +func (c *CacheCleanup) Stop() { + c.cancel() + c.wg.Wait() + + // Final flush of LRU buffer + c.FlushLRU() +} + +// runCleanupLoop runs the periodic cleanup. +func (c *CacheCleanup) runCleanupLoop() { + defer c.wg.Done() + + ticker := time.NewTicker(c.config.CleanupInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + c.FlushLRU() + c.runCleanup() + } + } +} + +// RecordAccess records an access for LRU tracking. Buffered until flush. +func (c *CacheCleanup) RecordAccess(root []byte, flags types.BlockDataFlags) { + c.lruMu.Lock() + defer c.lruMu.Unlock() + + key := string(root) + now := time.Now().UnixNano() + + update, exists := c.lruBuffer[key] + if !exists { + rootCopy := make([]byte, len(root)) + copy(rootCopy, root) + update = &lruUpdate{root: rootCopy} + c.lruBuffer[key] = update + } + + if flags.Has(types.BlockDataFlagHeader) { + update.headerAccess = now + } + if flags.Has(types.BlockDataFlagBody) { + update.bodyAccess = now + } + if flags.Has(types.BlockDataFlagPayload) { + update.payloadAccess = now + } + if flags.Has(types.BlockDataFlagBal) { + update.balAccess = now + } + + // Force flush if buffer is too large + if len(c.lruBuffer) >= maxLRUBufferSize { + c.flushLRULocked() + } +} + +// FlushLRU flushes buffered LRU updates to Pebble. +func (c *CacheCleanup) FlushLRU() { + c.lruMu.Lock() + defer c.lruMu.Unlock() + c.flushLRULocked() +} + +// flushLRULocked flushes LRU buffer (must hold lruMu). +func (c *CacheCleanup) flushLRULocked() { + if len(c.lruBuffer) == 0 { + return + } + + db := c.engine.GetDB() + batch := db.NewBatch() + + for _, update := range c.lruBuffer { + key := makeLRUKey(update.root) + + // Read existing LRU data + existing := make([]byte, lruValueSize) + if res, closer, err := db.Get(key); err == nil { + if len(res) >= lruValueSize { + copy(existing, res) + } + closer.Close() + } + + // Merge updates (only update non-zero values) + value := make([]byte, lruValueSize) + copy(value, existing) + + if update.headerAccess > 0 { + binary.BigEndian.PutUint64(value[0:8], uint64(update.headerAccess)) + } + if update.bodyAccess > 0 { + binary.BigEndian.PutUint64(value[8:16], uint64(update.bodyAccess)) + } + if update.payloadAccess > 0 { + binary.BigEndian.PutUint64(value[16:24], uint64(update.payloadAccess)) + } + if update.balAccess > 0 { + binary.BigEndian.PutUint64(value[24:32], uint64(update.balAccess)) + } + + batch.Set(key, value, nil) + } + + if err := batch.Commit(nil); err != nil { + c.logger.Errorf("failed to flush LRU updates: %v", err) + } + batch.Close() + + // Clear buffer + c.lruBuffer = make(map[string]*lruUpdate, 100) +} + +// makeLRUKey creates the key for LRU data. +func makeLRUKey(root []byte) []byte { + key := make([]byte, 2+len(root)) + binary.BigEndian.PutUint16(key[:2], KeyNamespaceLRU) + copy(key[2:], root) + return key +} + +// runCleanup performs cleanup for all configured component types. +func (c *CacheCleanup) runCleanup() { + c.logger.Debug("starting cache cleanup") + + componentConfigs := map[uint16]*dtypes.BlockDbRetentionConfig{ + BlockTypeHeader: &c.config.HeaderRetention, + BlockTypeBody: &c.config.BodyRetention, + BlockTypePayload: &c.config.PayloadRetention, + BlockTypeBal: &c.config.BalRetention, + } + + for blockType, config := range componentConfigs { + if config == nil || !config.Enabled { + continue + } + + switch config.CleanupMode { + case "age": + c.cleanupByAge(blockType, config.RetentionTime) + case "lru": + c.cleanupByLRU(blockType, config.MaxSize*1024*1024) // Convert MB to bytes + } + } +} + +// cleanupByAge removes entries older than the retention time based on storage timestamp. +func (c *CacheCleanup) cleanupByAge(blockType uint16, retention time.Duration) { + if retention == 0 { + return + } + + cutoff := time.Now().Add(-retention) + deleted := 0 + + db := c.engine.GetDB() + iter, err := db.NewIter(&pebble.IterOptions{}) + if err != nil { + c.logger.Errorf("failed to create iterator: %v", err) + return + } + defer iter.Close() + + batch := db.NewBatch() + defer batch.Close() + + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + + // Check if this key is in the block namespace + if len(key) < 36 { // 2 (namespace) + 32 (root) + 2 (type) + continue + } + + namespace := binary.BigEndian.Uint16(key[:2]) + if namespace != KeyNamespaceBlock { + continue + } + + keyType := binary.BigEndian.Uint16(key[len(key)-2:]) + if keyType != blockType { + continue + } + + // Check timestamp from value (stored at offset 8) + value := iter.Value() + if len(value) < valueHeaderSize { + continue + } + + timestamp := time.Unix(0, int64(binary.BigEndian.Uint64(value[8:16]))) + if timestamp.Before(cutoff) { + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + batch.Delete(keyCopy, nil) + deleted++ + } + } + + if deleted > 0 { + if err := batch.Commit(nil); err != nil { + c.logger.Errorf("failed to commit age cleanup batch: %v", err) + } else { + c.logger.Infof("cleaned up %d entries for block type %d (age-based)", deleted, blockType) + } + } +} + +// lruEntry represents an entry for LRU cleanup sorting. +type lruEntry struct { + root []byte + key []byte + size int64 + lastAccess int64 +} + +// cleanupByLRU removes least recently used entries when size exceeds limit. +func (c *CacheCleanup) cleanupByLRU(blockType uint16, maxSize int64) { + if maxSize == 0 { + return + } + + db := c.engine.GetDB() + + // First pass: collect all entries with their sizes and LRU timestamps + entries := make([]*lruEntry, 0, 1000) + var totalSize int64 + + iter, err := db.NewIter(&pebble.IterOptions{}) + if err != nil { + c.logger.Errorf("failed to create iterator: %v", err) + return + } + + // Scan block entries + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + + if len(key) < 36 { + continue + } + + namespace := binary.BigEndian.Uint16(key[:2]) + if namespace != KeyNamespaceBlock { + continue + } + + keyType := binary.BigEndian.Uint16(key[len(key)-2:]) + if keyType != blockType { + continue + } + + // Extract root from key + root := key[2 : len(key)-2] + value := iter.Value() + size := int64(len(value)) + totalSize += size + + // Get LRU timestamp for this entry + lastAccess := c.getLRUTimestamp(db, root, blockType) + + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + rootCopy := make([]byte, len(root)) + copy(rootCopy, root) + + entries = append(entries, &lruEntry{ + root: rootCopy, + key: keyCopy, + size: size, + lastAccess: lastAccess, + }) + } + iter.Close() + + // Check if we need to clean up + if totalSize <= maxSize { + return + } + + // Sort by last access time (oldest first, 0 = never accessed = oldest) + sort.Slice(entries, func(i, j int) bool { + return entries[i].lastAccess < entries[j].lastAccess + }) + + // Delete oldest entries until we're under the limit + batch := db.NewBatch() + defer batch.Close() + + deleted := 0 + freedSize := int64(0) + targetFree := totalSize - maxSize + + for _, entry := range entries { + if freedSize >= targetFree { + break + } + + batch.Delete(entry.key, nil) + freedSize += entry.size + deleted++ + } + + if deleted > 0 { + if err := batch.Commit(nil); err != nil { + c.logger.Errorf("failed to commit LRU cleanup batch: %v", err) + } else { + c.logger.Infof("cleaned up %d entries for block type %d (LRU-based, freed %d bytes)", + deleted, blockType, freedSize) + } + } +} + +// getLRUTimestamp retrieves the LRU timestamp for a specific component. +func (c *CacheCleanup) getLRUTimestamp(db *pebble.DB, root []byte, blockType uint16) int64 { + key := makeLRUKey(root) + + res, closer, err := db.Get(key) + if err != nil { + return 0 // Never accessed + } + defer closer.Close() + + if len(res) < lruValueSize { + return 0 + } + + // Extract timestamp based on block type + var offset int + switch blockType { + case BlockTypeHeader: + offset = 0 + case BlockTypeBody: + offset = 8 + case BlockTypePayload: + offset = 16 + case BlockTypeBal: + offset = 24 + default: + return 0 + } + + return int64(binary.BigEndian.Uint64(res[offset : offset+8])) +} + +// DeleteLRU removes LRU data for a block (call when deleting block data). +func (c *CacheCleanup) DeleteLRU(root []byte) { + db := c.engine.GetDB() + key := makeLRUKey(root) + db.Delete(key, nil) + + // Also remove from buffer + c.lruMu.Lock() + delete(c.lruBuffer, string(root)) + c.lruMu.Unlock() +} diff --git a/blockdb/pebble/pebble.go b/blockdb/pebble/pebble.go index 7bea79c25..1da3d6b8c 100644 --- a/blockdb/pebble/pebble.go +++ b/blockdb/pebble/pebble.go @@ -3,6 +3,8 @@ package pebble import ( "context" "encoding/binary" + "fmt" + "time" "github.com/cockroachdb/pebble" "github.com/ethpandaops/dora/blockdb/types" @@ -17,10 +19,15 @@ const ( BlockTypeHeader uint16 = 1 BlockTypeBody uint16 = 2 BlockTypePayload uint16 = 3 + BlockTypeBal uint16 = 4 ) +// Value format: [version (8 bytes)] [timestamp (8 bytes)] [data] +const valueHeaderSize = 16 + type PebbleEngine struct { - db *pebble.DB + db *pebble.DB + config dtypes.PebbleBlockDBConfig } func NewPebbleEngine(config dtypes.PebbleBlockDBConfig) (types.BlockDbEngine, error) { @@ -35,208 +42,259 @@ func NewPebbleEngine(config dtypes.PebbleBlockDBConfig) (types.BlockDbEngine, er } return &PebbleEngine{ - db: db, + db: db, + config: config, }, nil } func (e *PebbleEngine) Close() error { - err := e.db.Close() - if err != nil { - return err - } - - return nil + return e.db.Close() } -func (e *PebbleEngine) getBlockHeader(root []byte) ([]byte, uint64, error) { +// makeKey creates a key for the given root and block type. +func makeKey(root []byte, blockType uint16) []byte { key := make([]byte, 2+len(root)+2) binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypeHeader) + binary.BigEndian.PutUint16(key[2+len(root):], blockType) + return key +} + +// getComponent retrieves a single component from the database. +// Returns (data, version, timestamp, error). Returns nil data if not found. +func (e *PebbleEngine) getComponent(root []byte, blockType uint16) ([]byte, uint64, time.Time, error) { + key := makeKey(root, blockType) res, closer, err := e.db.Get(key) - if err != nil && err != pebble.ErrNotFound { - return nil, 0, err + if err == pebble.ErrNotFound { + return nil, 0, time.Time{}, nil + } + if err != nil { + return nil, 0, time.Time{}, err } defer closer.Close() - if err == pebble.ErrNotFound || len(res) == 0 { - return nil, 0, nil + if len(res) < valueHeaderSize { + return nil, 0, time.Time{}, nil } version := binary.BigEndian.Uint64(res[:8]) - header := make([]byte, len(res)-8) - copy(header, res[8:]) + timestamp := time.Unix(0, int64(binary.BigEndian.Uint64(res[8:16]))) + + data := make([]byte, len(res)-valueHeaderSize) + copy(data, res[valueHeaderSize:]) - return header, version, nil + return data, version, timestamp, nil } -func (e *PebbleEngine) getBlockBody(root []byte, parser func(uint64, []byte) (interface{}, error)) (interface{}, uint64, error) { - key := make([]byte, 2+len(root)+2) - binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) - copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypeBody) +// setComponent stores a single component in the database. +func (e *PebbleEngine) setComponent(root []byte, blockType uint16, version uint64, data []byte) error { + key := makeKey(root, blockType) - res, closer, err := e.db.Get(key) - if err != nil && err != pebble.ErrNotFound { - return nil, 0, err - } - defer closer.Close() + value := make([]byte, valueHeaderSize+len(data)) + binary.BigEndian.PutUint64(value[:8], version) + binary.BigEndian.PutUint64(value[8:16], uint64(time.Now().UnixNano())) + copy(value[valueHeaderSize:], data) - if err == pebble.ErrNotFound || len(res) == 0 { - return nil, 0, nil - } + return e.db.Set(key, value, nil) +} - version := binary.BigEndian.Uint64(res[:8]) - block := res[8:] +// componentExists checks if a component exists in the database. +func (e *PebbleEngine) componentExists(root []byte, blockType uint16) bool { + key := makeKey(root, blockType) - body, err := parser(version, block) - if err != nil { - return nil, 0, err + res, closer, err := e.db.Get(key) + if err == nil && len(res) >= valueHeaderSize { + closer.Close() + return true } - - return body, version, nil + return false } -func (e *PebbleEngine) getBlockPayload(root []byte, parser func(uint64, []byte) (interface{}, error)) (interface{}, uint64, error) { - key := make([]byte, 2+len(root)+2) - binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) - copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) +// GetStoredComponents returns which components exist for a block. +func (e *PebbleEngine) GetStoredComponents(_ context.Context, _ uint64, root []byte) (types.BlockDataFlags, error) { + var flags types.BlockDataFlags - res, closer, err := e.db.Get(key) - if err != nil && err != pebble.ErrNotFound { - return nil, 0, err + if e.componentExists(root, BlockTypeHeader) { + flags |= types.BlockDataFlagHeader } - defer closer.Close() - - if err == pebble.ErrNotFound || len(res) == 0 { - return nil, 0, nil + if e.componentExists(root, BlockTypeBody) { + flags |= types.BlockDataFlagBody } - - version := binary.BigEndian.Uint64(res[:8]) - block := res[8:] - - body, err := parser(version, block) - if err != nil { - return nil, 0, err + if e.componentExists(root, BlockTypePayload) { + flags |= types.BlockDataFlagPayload + } + if e.componentExists(root, BlockTypeBal) { + flags |= types.BlockDataFlagBal } - return body, version, nil + return flags, nil } -func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { - header, header_ver, err := e.getBlockHeader(root) - if err != nil { - return nil, err +// GetBlock retrieves block data with selective loading based on flags. +// Note: LRU access tracking should be done by the caller via CacheCleanup.RecordAccess() +// to avoid expensive read-modify-write operations on every access. +func (e *PebbleEngine) GetBlock( + _ context.Context, + _ uint64, + root []byte, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { + blockData := &types.BlockData{} + + // Load header if requested + if flags.Has(types.BlockDataFlagHeader) { + data, version, _, err := e.getComponent(root, BlockTypeHeader) + if err != nil { + return nil, fmt.Errorf("failed to get header: %w", err) + } + if data != nil { + blockData.HeaderVersion = version + blockData.HeaderData = data + } } - blockData := &types.BlockData{ - HeaderVersion: header_ver, - HeaderData: header, - } + // Load body if requested + if flags.Has(types.BlockDataFlagBody) { + data, version, _, err := e.getComponent(root, BlockTypeBody) + if err != nil { + return nil, fmt.Errorf("failed to get body: %w", err) + } - if parseBlock == nil { - parseBlock = func(version uint64, block []byte) (interface{}, error) { - blockData.BodyData = make([]byte, len(block)) - copy(blockData.BodyData, block) - return nil, nil + if data != nil { + blockData.BodyVersion = version + if parseBlock != nil { + body, err := parseBlock(version, data) + if err != nil { + return nil, fmt.Errorf("failed to parse body: %w", err) + } + blockData.Body = body + } else { + blockData.BodyData = data + } } } - body, body_ver, err := e.getBlockBody(root, parseBlock) - if err != nil { - return nil, err + // Load payload if requested + if flags.Has(types.BlockDataFlagPayload) { + data, version, _, err := e.getComponent(root, BlockTypePayload) + if err != nil { + return nil, fmt.Errorf("failed to get payload: %w", err) + } + + if data != nil { + blockData.PayloadVersion = version + if parsePayload != nil { + payload, err := parsePayload(version, data) + if err != nil { + return nil, fmt.Errorf("failed to parse payload: %w", err) + } + blockData.Payload = payload + } else { + blockData.PayloadData = data + } + } } - blockData.Body = body - blockData.BodyVersion = body_ver + // Load BAL if requested + if flags.Has(types.BlockDataFlagBal) { + data, version, _, err := e.getComponent(root, BlockTypeBal) + if err != nil { + return nil, fmt.Errorf("failed to get BAL: %w", err) + } - payload, payload_ver, err := e.getBlockPayload(root, parsePayload) - if err != nil { - return nil, err + if data != nil { + blockData.BalVersion = version + blockData.BalData = data + } } - blockData.Payload = payload - blockData.PayloadVersion = payload_ver - return blockData, nil } -func (e *PebbleEngine) checkBlock(key []byte) bool { - res, closer, err := e.db.Get(key) - if err == nil && len(res) > 0 { - closer.Close() - return true +// AddBlock stores block data. Returns (added, updated, error). +// - added: true if a new block was created +// - updated: true if an existing block was updated with new components +func (e *PebbleEngine) AddBlock( + _ context.Context, + _ uint64, + root []byte, + dataCb func() (*types.BlockData, error), +) (bool, bool, error) { + // Check what components already exist + existingFlags, err := e.GetStoredComponents(context.Background(), 0, root) + if err != nil { + return false, false, fmt.Errorf("failed to check existing components: %w", err) } - return false -} - -func (e *PebbleEngine) addBlockHeader(key []byte, version uint64, header []byte) error { - data := make([]byte, 8+len(header)) - binary.BigEndian.PutUint64(data[:8], version) - - return e.db.Set(key, data, nil) -} - -func (e *PebbleEngine) addBlockBody(root []byte, version uint64, block []byte) error { - key := make([]byte, 2+len(root)+2) - binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) - copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypeBody) - - data := make([]byte, 8+len(block)) - binary.BigEndian.PutUint64(data[:8], version) - copy(data[8:], block) - - return e.db.Set(key, data, nil) -} - -func (e *PebbleEngine) addBlockPayload(root []byte, version uint64, payload []byte) error { - key := make([]byte, 2+len(root)+2) - binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) - copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) - - data := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint64(data[:8], version) - copy(data[8:], payload) + // Get the new data + blockData, err := dataCb() + if err != nil { + return false, false, err + } - return e.db.Set(key, data, nil) -} + // Determine what new components we have + var newFlags types.BlockDataFlags + if len(blockData.HeaderData) > 0 { + newFlags |= types.BlockDataFlagHeader + } + if len(blockData.BodyData) > 0 { + newFlags |= types.BlockDataFlagBody + } + if blockData.PayloadVersion != 0 && len(blockData.PayloadData) > 0 { + newFlags |= types.BlockDataFlagPayload + } + if blockData.BalVersion != 0 && len(blockData.BalData) > 0 { + newFlags |= types.BlockDataFlagBal + } -func (e *PebbleEngine) AddBlock(_ context.Context, _ uint64, root []byte, dataCb func() (*types.BlockData, error)) (bool, error) { - key := make([]byte, 2+len(root)+2) - binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) - copy(key[2:], root) - binary.BigEndian.PutUint16(key[2+len(root):], BlockTypeHeader) + // Calculate components to add (new components not in existing) + toAdd := newFlags &^ existingFlags - if e.checkBlock(key) { - return false, nil + if toAdd == 0 { + // Nothing new to add + return false, false, nil } - blockData, err := dataCb() - if err != nil { - return false, err + isNew := existingFlags == 0 + isUpdated := !isNew + + // Store new components + if toAdd.Has(types.BlockDataFlagHeader) { + if err := e.setComponent(root, BlockTypeHeader, blockData.HeaderVersion, blockData.HeaderData); err != nil { + return false, false, fmt.Errorf("failed to store header: %w", err) + } } - err = e.addBlockHeader(key, blockData.HeaderVersion, blockData.HeaderData) - if err != nil { - return false, err + if toAdd.Has(types.BlockDataFlagBody) { + if err := e.setComponent(root, BlockTypeBody, blockData.BodyVersion, blockData.BodyData); err != nil { + return false, false, fmt.Errorf("failed to store body: %w", err) + } } - err = e.addBlockBody(root, blockData.BodyVersion, blockData.BodyData) - if err != nil { - return false, err + if toAdd.Has(types.BlockDataFlagPayload) { + if err := e.setComponent(root, BlockTypePayload, blockData.PayloadVersion, blockData.PayloadData); err != nil { + return false, false, fmt.Errorf("failed to store payload: %w", err) + } } - if blockData.PayloadVersion != 0 { - err = e.addBlockPayload(root, blockData.PayloadVersion, blockData.PayloadData) - if err != nil { - return false, err + if toAdd.Has(types.BlockDataFlagBal) { + if err := e.setComponent(root, BlockTypeBal, blockData.BalVersion, blockData.BalData); err != nil { + return false, false, fmt.Errorf("failed to store BAL: %w", err) } } - return true, nil + return isNew, isUpdated, nil +} + +// GetDB returns the underlying Pebble database for cleanup operations. +func (e *PebbleEngine) GetDB() *pebble.DB { + return e.db +} + +// GetConfig returns the engine configuration. +func (e *PebbleEngine) GetConfig() dtypes.PebbleBlockDBConfig { + return e.config } diff --git a/blockdb/s3/format.go b/blockdb/s3/format.go new file mode 100644 index 000000000..79fa2c04a --- /dev/null +++ b/blockdb/s3/format.go @@ -0,0 +1,202 @@ +package s3 + +import ( + "encoding/binary" + "fmt" + + "github.com/attestantio/go-eth2-client/spec" + + "github.com/ethpandaops/dora/blockdb/types" +) + +// Object format versions: +// v1: header + body (pre-gloas blocks) +// v2: header + body + payload + bal (gloas+ blocks, payload/BAL introduced in same fork) +// +// Note: Both payload and BAL may be empty (length 0), but body is always required. + +// Metadata sizes by version +const ( + metadataSizeV1 = 16 // 4 (version) + 4 (headerLen) + 4 (bodyVer) + 4 (bodyLen) + metadataSizeV2 = 32 // v1 + 4 (payloadVer) + 4 (payloadLen) + 4 (balVer) + 4 (balLen) + + // Maximum metadata size for initial read + maxMetadataSize = 64 +) + +// objectMetadata represents the metadata for all format versions. +type objectMetadata struct { + ObjVersion uint32 + + // Header (always present) + HeaderLength uint32 + + // Body (always required) + BodyVersion uint32 + BodyLength uint32 + + // Payload (v2+, may be empty) + PayloadVersion uint32 + PayloadLength uint32 + + // BAL (v2+, may be empty) + BalVersion uint32 + BalLength uint32 +} + +// metadataSize returns the metadata size for this object. +func (m *objectMetadata) metadataSize() int { + switch m.ObjVersion { + case 1: + return metadataSizeV1 + case 2: + return metadataSizeV2 + default: + return metadataSizeV2 + } +} + +// headerOffset returns the byte offset of the header data. +func (m *objectMetadata) headerOffset() int { + return m.metadataSize() +} + +// bodyOffset returns the byte offset of the body data. +func (m *objectMetadata) bodyOffset() int { + return m.metadataSize() + int(m.HeaderLength) +} + +// payloadOffset returns the byte offset of the payload data. +func (m *objectMetadata) payloadOffset() int { + return m.metadataSize() + int(m.HeaderLength) + int(m.BodyLength) +} + +// balOffset returns the byte offset of the BAL data. +func (m *objectMetadata) balOffset() int { + return m.metadataSize() + int(m.HeaderLength) + int(m.BodyLength) + int(m.PayloadLength) +} + +// storedFlags returns which components are stored in this object. +func (m *objectMetadata) storedFlags() types.BlockDataFlags { + var flags types.BlockDataFlags + + if m.HeaderLength > 0 { + flags |= types.BlockDataFlagHeader + } + if m.BodyLength > 0 { + flags |= types.BlockDataFlagBody + } + if m.PayloadLength > 0 && m.ObjVersion >= 2 { + flags |= types.BlockDataFlagPayload + } + if m.BalLength > 0 && m.ObjVersion >= 2 { + flags |= types.BlockDataFlagBal + } + + return flags +} + +// readObjectMetadata reads metadata from any format version. +func readObjectMetadata(data []byte) (*objectMetadata, error) { + if len(data) < 4 { + return nil, fmt.Errorf("data too short for metadata version") + } + + version := binary.BigEndian.Uint32(data[:4]) + meta := &objectMetadata{ObjVersion: version} + + switch version { + case 1: + if len(data) < metadataSizeV1 { + return nil, fmt.Errorf("data too short for v1 metadata: need %d, got %d", metadataSizeV1, len(data)) + } + meta.HeaderLength = binary.BigEndian.Uint32(data[4:8]) + meta.BodyVersion = binary.BigEndian.Uint32(data[8:12]) + meta.BodyLength = binary.BigEndian.Uint32(data[12:16]) + + case 2: + if len(data) < metadataSizeV2 { + return nil, fmt.Errorf("data too short for v2 metadata: need %d, got %d", metadataSizeV2, len(data)) + } + meta.HeaderLength = binary.BigEndian.Uint32(data[4:8]) + meta.BodyVersion = binary.BigEndian.Uint32(data[8:12]) + meta.BodyLength = binary.BigEndian.Uint32(data[12:16]) + meta.PayloadVersion = binary.BigEndian.Uint32(data[16:20]) + meta.PayloadLength = binary.BigEndian.Uint32(data[20:24]) + meta.BalVersion = binary.BigEndian.Uint32(data[24:28]) + meta.BalLength = binary.BigEndian.Uint32(data[28:32]) + + default: + return nil, fmt.Errorf("unsupported object version: %d", version) + } + + return meta, nil +} + +// writeObjectMetadata creates metadata bytes for the given BlockData. +// Uses v1 format for pre-gloas blocks, v2 for gloas+ blocks. +func writeObjectMetadata(data *types.BlockData) []byte { + // Use v2 format only for gloas+ blocks (which can have payload/BAL) + if data.BodyVersion >= uint64(spec.DataVersionGloas) { + meta := make([]byte, metadataSizeV2) + binary.BigEndian.PutUint32(meta[0:4], 2) + binary.BigEndian.PutUint32(meta[4:8], uint32(len(data.HeaderData))) + binary.BigEndian.PutUint32(meta[8:12], uint32(data.BodyVersion)) + binary.BigEndian.PutUint32(meta[12:16], uint32(len(data.BodyData))) + binary.BigEndian.PutUint32(meta[16:20], uint32(data.PayloadVersion)) + binary.BigEndian.PutUint32(meta[20:24], uint32(len(data.PayloadData))) + binary.BigEndian.PutUint32(meta[24:28], uint32(data.BalVersion)) + binary.BigEndian.PutUint32(meta[28:32], uint32(len(data.BalData))) + return meta + } + + // Use v1 format for pre-gloas blocks + meta := make([]byte, metadataSizeV1) + binary.BigEndian.PutUint32(meta[0:4], 1) + binary.BigEndian.PutUint32(meta[4:8], uint32(len(data.HeaderData))) + binary.BigEndian.PutUint32(meta[8:12], uint32(data.BodyVersion)) + binary.BigEndian.PutUint32(meta[12:16], uint32(len(data.BodyData))) + return meta +} + +// getDataRange calculates the single byte range spanning all requested components. +// Returns (start, end) where end is inclusive. Returns (-1, -1) if no data to fetch. +func (m *objectMetadata) getDataRange(flags types.BlockDataFlags) (int64, int64) { + var start int64 = -1 + var end int64 = -1 + + // Check each component in order (they're stored sequentially) + if flags.Has(types.BlockDataFlagHeader) && m.HeaderLength > 0 { + start = int64(m.headerOffset()) + end = start + int64(m.HeaderLength) - 1 + } + + if flags.Has(types.BlockDataFlagBody) && m.BodyLength > 0 { + bodyStart := int64(m.bodyOffset()) + bodyEnd := bodyStart + int64(m.BodyLength) - 1 + if start < 0 { + start = bodyStart + } + end = bodyEnd + } + + if flags.Has(types.BlockDataFlagPayload) && m.PayloadLength > 0 && m.ObjVersion >= 2 { + payloadStart := int64(m.payloadOffset()) + payloadEnd := payloadStart + int64(m.PayloadLength) - 1 + if start < 0 { + start = payloadStart + } + end = payloadEnd + } + + if flags.Has(types.BlockDataFlagBal) && m.BalLength > 0 && m.ObjVersion >= 2 { + balStart := int64(m.balOffset()) + balEnd := balStart + int64(m.BalLength) - 1 + if start < 0 { + start = balStart + } + end = balEnd + } + + return start, end +} diff --git a/blockdb/s3/s3store.go b/blockdb/s3/s3store.go index e3aec098d..6e4a32a27 100644 --- a/blockdb/s3/s3store.go +++ b/blockdb/s3/s3store.go @@ -3,13 +3,13 @@ package s3 import ( "bytes" "context" - "encoding/binary" "encoding/hex" "fmt" "io" "path" "strings" + "github.com/attestantio/go-eth2-client/spec" "github.com/ethpandaops/dora/blockdb/types" dtypes "github.com/ethpandaops/dora/types" "github.com/minio/minio-go/v7" @@ -20,6 +20,10 @@ type S3Engine struct { client *minio.Client bucket string pathPrefix string + config dtypes.S3BlockDBConfig + + // Range request support (configured via EnableRangeRequests) + rangeRequestsEnabled bool } func NewS3Engine(config dtypes.S3BlockDBConfig) (types.BlockDbEngine, error) { @@ -42,9 +46,11 @@ func NewS3Engine(config dtypes.S3BlockDBConfig) (types.BlockDbEngine, error) { } engine := &S3Engine{ - client: client, - bucket: config.Bucket, - pathPrefix: strings.TrimPrefix(config.Path, "/"), + client: client, + bucket: config.Bucket, + pathPrefix: strings.TrimPrefix(config.Path, "/"), + config: config, + rangeRequestsEnabled: config.EnableRangeRequests, } return engine, nil @@ -59,182 +65,457 @@ func (e *S3Engine) getObjectKey(root []byte, slot uint64) string { return path.Join(e.pathPrefix, fmt.Sprintf("%06d", slot/10000), fmt.Sprintf("%010d_%s", slot, rootHex)) } -type objectMetadata struct { - objVersion uint32 - headerLength uint32 - bodyVersion uint32 - bodyLength uint32 - payloadVersion uint32 - payloadLength uint32 +// GetStoredComponents returns which components exist for a block by reading metadata. +func (e *S3Engine) GetStoredComponents(ctx context.Context, slot uint64, root []byte) (types.BlockDataFlags, error) { + key := e.getObjectKey(root, slot) + + // Read just the metadata + meta, err := e.readMetadata(ctx, key) + if err != nil { + return 0, err + } + if meta == nil { + return 0, nil + } + + return meta.storedFlags(), nil } -func (e *S3Engine) readObjectMetadata(data []byte) (*objectMetadata, int, error) { - metadataLength := 4 - metadata := &objectMetadata{ - objVersion: binary.BigEndian.Uint32(data[:4]), +// readMetadata reads object metadata using range request if enabled, otherwise full read. +func (e *S3Engine) readMetadata(ctx context.Context, key string) (*objectMetadata, error) { + if e.config.EnableRangeRequests { + meta, err := e.readMetadataWithRange(ctx, key) + if err == nil { + return meta, nil + } + // Fall through to full read on error + } + + // Full read fallback + obj, err := e.client.GetObject(ctx, e.bucket, key, minio.GetObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.Code == "NoSuchKey" { + return nil, nil + } + return nil, fmt.Errorf("failed to get object: %w", err) } + defer obj.Close() - switch metadata.objVersion { - case 1: - metadata.headerLength = binary.BigEndian.Uint32(data[4:8]) - metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) - metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) - metadataLength += 12 - case 2: - metadata.headerLength = binary.BigEndian.Uint32(data[4:8]) - metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) - metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) - metadata.payloadVersion = binary.BigEndian.Uint32(data[16:20]) - metadata.payloadLength = binary.BigEndian.Uint32(data[20:24]) - metadataLength += 20 + buf := make([]byte, maxMetadataSize) + n, err := obj.Read(buf) + if (err != nil && err != io.EOF) || n == 0 { + return nil, fmt.Errorf("failed to read metadata: %w", err) } - return metadata, metadataLength, nil + return readObjectMetadata(buf[:n]) } -func (e *S3Engine) writeObjectMetadata(metadata *objectMetadata) []byte { - data := make([]byte, 4, 16) - binary.BigEndian.PutUint32(data, metadata.objVersion) +// readMetadataWithRange reads metadata using HTTP Range request. +func (e *S3Engine) readMetadataWithRange(ctx context.Context, key string) (*objectMetadata, error) { + opts := minio.GetObjectOptions{} + if err := opts.SetRange(0, int64(maxMetadataSize-1)); err != nil { + return nil, err + } - switch metadata.objVersion { - case 1: - data = binary.BigEndian.AppendUint32(data, metadata.headerLength) - data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) - data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) - case 2: - data = binary.BigEndian.AppendUint32(data, metadata.headerLength) - data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) - data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) - data = binary.BigEndian.AppendUint32(data, metadata.payloadVersion) - data = binary.BigEndian.AppendUint32(data, metadata.payloadLength) + obj, err := e.client.GetObject(ctx, e.bucket, key, opts) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.Code == "NoSuchKey" { + return nil, nil + } + return nil, fmt.Errorf("failed to get object with range: %w", err) + } + defer obj.Close() + + buf := make([]byte, maxMetadataSize) + n, err := obj.Read(buf) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("failed to read range: %w", err) } - return data + return readObjectMetadata(buf[:n]) } -func (e *S3Engine) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { +// GetBlock retrieves block data with selective loading based on flags. +func (e *S3Engine) GetBlock( + ctx context.Context, + slot uint64, + root []byte, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { key := e.getObjectKey(root, slot) + // Try range-based loading if enabled + if e.config.EnableRangeRequests && e.rangeRequestsEnabled { + data, err := e.getBlockWithRanges(ctx, key, flags, parseBlock, parsePayload) + if err == nil { + return data, nil + } + // Fall through to full read on error + } + + // Full read fallback + return e.getBlockFull(ctx, key, flags, parseBlock, parsePayload) +} + +// getBlockWithRanges uses a single range request for selective loading. +// Makes exactly 2 GET requests: one for metadata, one for all requested data. +func (e *S3Engine) getBlockWithRanges( + ctx context.Context, + key string, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { + // First, get metadata (1 GET request) + meta, err := e.readMetadataWithRange(ctx, key) + if err != nil { + return nil, err + } + if meta == nil { + return nil, nil + } + + // Calculate the single byte range spanning all requested components + rangeStart, rangeEnd := meta.getDataRange(flags) + if rangeStart < 0 { + // No data to fetch + return &types.BlockData{ + HeaderVersion: uint64(meta.ObjVersion), + BodyVersion: uint64(meta.BodyVersion), + PayloadVersion: uint64(meta.PayloadVersion), + BalVersion: uint64(meta.BalVersion), + }, nil + } + + // Fetch all requested data in a single GET request + opts := minio.GetObjectOptions{} + if err := opts.SetRange(rangeStart, rangeEnd); err != nil { + return nil, err + } + + obj, err := e.client.GetObject(ctx, e.bucket, key, opts) + if err != nil { + return nil, fmt.Errorf("failed to get object range: %w", err) + } + defer obj.Close() + + data, err := io.ReadAll(obj) + if err != nil { + return nil, fmt.Errorf("failed to read object range: %w", err) + } + + // Extract requested components from the fetched data + return e.extractComponents(meta, flags, data, rangeStart, parseBlock, parsePayload) +} + +// extractComponents extracts requested components from fetched data. +func (e *S3Engine) extractComponents( + meta *objectMetadata, + flags types.BlockDataFlags, + data []byte, + dataStartOffset int64, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { + blockData := &types.BlockData{ + HeaderVersion: uint64(meta.ObjVersion), + BodyVersion: uint64(meta.BodyVersion), + PayloadVersion: uint64(meta.PayloadVersion), + BalVersion: uint64(meta.BalVersion), + } + + // Extract header if requested + if flags.Has(types.BlockDataFlagHeader) && meta.HeaderLength > 0 { + start := int64(meta.headerOffset()) - dataStartOffset + end := start + int64(meta.HeaderLength) + if start >= 0 && end <= int64(len(data)) { + blockData.HeaderData = data[start:end] + } + } + + // Extract body if requested + if flags.Has(types.BlockDataFlagBody) && meta.BodyLength > 0 { + start := int64(meta.bodyOffset()) - dataStartOffset + end := start + int64(meta.BodyLength) + if start >= 0 && end <= int64(len(data)) { + bodyData := data[start:end] + if parseBlock != nil { + body, err := parseBlock(uint64(meta.BodyVersion), bodyData) + if err != nil { + return nil, fmt.Errorf("failed to parse body: %w", err) + } + blockData.Body = body + } else { + blockData.BodyData = bodyData + } + } + } + + // Extract payload if requested (v2+) + if flags.Has(types.BlockDataFlagPayload) && meta.PayloadLength > 0 && meta.ObjVersion >= 2 { + start := int64(meta.payloadOffset()) - dataStartOffset + end := start + int64(meta.PayloadLength) + if start >= 0 && end <= int64(len(data)) { + payloadData := data[start:end] + if parsePayload != nil { + payload, err := parsePayload(uint64(meta.PayloadVersion), payloadData) + if err != nil { + return nil, fmt.Errorf("failed to parse payload: %w", err) + } + blockData.Payload = payload + } else { + blockData.PayloadData = payloadData + } + } + } + + // Extract BAL if requested (v2+) + if flags.Has(types.BlockDataFlagBal) && meta.BalLength > 0 && meta.ObjVersion >= 2 { + start := int64(meta.balOffset()) - dataStartOffset + end := start + int64(meta.BalLength) + if start >= 0 && end <= int64(len(data)) { + blockData.BalData = data[start:end] + } + } + + return blockData, nil +} + +// getBlockFull performs a full object read (fallback when range requests fail). +func (e *S3Engine) getBlockFull( + ctx context.Context, + key string, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { obj, err := e.client.GetObject(ctx, e.bucket, key, minio.GetObjectOptions{}) if err != nil { - if minio.ToErrorResponse(err).Code == "NoSuchKey" { + errResp := minio.ToErrorResponse(err) + if errResp.Code == "NoSuchKey" { return nil, nil } return nil, fmt.Errorf("failed to get object: %w", err) } defer obj.Close() - // read metadata - buf := make([]byte, 1024) - buflen, err := obj.Read(buf) - if (err != nil && err != io.EOF) || buflen == 0 { - return nil, fmt.Errorf("failed to read metadata: %w", err) + // Read entire object + data, err := io.ReadAll(obj) + if err != nil { + return nil, fmt.Errorf("failed to read object: %w", err) } - metadata, metadataLength, err := e.readObjectMetadata(buf) + // Parse metadata + meta, err := readObjectMetadata(data) if err != nil { return nil, fmt.Errorf("failed to read metadata: %w", err) } - headerData := make([]byte, metadata.headerLength) - headerOffset := 0 - if buflen > metadataLength { - copy(headerData, buf[metadataLength:buflen]) - headerOffset = buflen - metadataLength + blockData := &types.BlockData{ + HeaderVersion: uint64(meta.ObjVersion), + BodyVersion: uint64(meta.BodyVersion), + PayloadVersion: uint64(meta.PayloadVersion), + BalVersion: uint64(meta.BalVersion), } - if buflen < int(metadataLength)+int(metadata.headerLength) { - _, err = obj.Read(headerData[headerOffset:]) - if err != nil { - return nil, fmt.Errorf("failed to read header data: %w", err) - } - } + metaSize := meta.metadataSize() - bodyData := make([]byte, metadata.bodyLength) - bodyOffset := 0 - if buflen > int(metadataLength)+int(metadata.headerLength) { - copy(bodyData, buf[int(metadataLength)+int(metadata.headerLength):buflen]) - bodyOffset = buflen - int(metadataLength) - int(metadata.headerLength) + // Extract header if requested + if flags.Has(types.BlockDataFlagHeader) && meta.HeaderLength > 0 { + headerEnd := metaSize + int(meta.HeaderLength) + if headerEnd <= len(data) { + blockData.HeaderData = data[metaSize:headerEnd] + } } - if buflen < int(metadataLength)+int(metadata.headerLength)+int(metadata.bodyLength) { - _, err = obj.Read(bodyData[bodyOffset:]) - if err != nil { - return nil, fmt.Errorf("failed to read body data: %w", err) + // Extract body if requested + if flags.Has(types.BlockDataFlagBody) && meta.BodyLength > 0 { + bodyStart := metaSize + int(meta.HeaderLength) + bodyEnd := bodyStart + int(meta.BodyLength) + if bodyEnd <= len(data) { + bodyData := data[bodyStart:bodyEnd] + if parseBlock != nil { + body, err := parseBlock(uint64(meta.BodyVersion), bodyData) + if err != nil { + return nil, fmt.Errorf("failed to parse body: %w", err) + } + blockData.Body = body + } else { + blockData.BodyData = bodyData + } } } - blockData := &types.BlockData{ - HeaderVersion: uint64(metadata.objVersion), - HeaderData: headerData, - BodyVersion: uint64(metadata.bodyVersion), + // Extract payload if requested (v2+) + if flags.Has(types.BlockDataFlagPayload) && meta.PayloadLength > 0 && meta.ObjVersion >= 2 { + payloadStart := metaSize + int(meta.HeaderLength) + int(meta.BodyLength) + payloadEnd := payloadStart + int(meta.PayloadLength) + if payloadEnd <= len(data) { + payloadData := data[payloadStart:payloadEnd] + if parsePayload != nil { + payload, err := parsePayload(uint64(meta.PayloadVersion), payloadData) + if err != nil { + return nil, fmt.Errorf("failed to parse payload: %w", err) + } + blockData.Payload = payload + } else { + blockData.PayloadData = payloadData + } + } } - if parseBlock != nil { - body, err := parseBlock(uint64(metadata.bodyVersion), bodyData) - if err != nil { - return nil, fmt.Errorf("failed to parse body: %w", err) + // Extract BAL if requested (v3+) + if flags.Has(types.BlockDataFlagBal) && meta.BalLength > 0 && meta.ObjVersion >= 2 { + balStart := metaSize + int(meta.HeaderLength) + int(meta.BodyLength) + int(meta.PayloadLength) + balEnd := balStart + int(meta.BalLength) + if balEnd <= len(data) { + blockData.BalData = data[balStart:balEnd] } - - blockData.Body = body - } else { - blockData.BodyData = bodyData } return blockData, nil } -func (e *S3Engine) AddBlock(ctx context.Context, slot uint64, root []byte, dataCb func() (*types.BlockData, error)) (bool, error) { +// AddBlock stores block data. Returns (added, updated, error). +func (e *S3Engine) AddBlock( + ctx context.Context, + slot uint64, + root []byte, + dataCb func() (*types.BlockData, error), +) (bool, bool, error) { key := e.getObjectKey(root, slot) - // Check if object already exists - stat, err := e.client.StatObject(ctx, e.bucket, key, minio.StatObjectOptions{}) - if err == nil && stat.Size > 0 { - return false, nil + // Check what components already exist + existingMeta, err := e.readMetadata(ctx, key) + if err != nil && err.Error() != "failed to get object: NoSuchKey" { + // Ignore "not found" errors + existingFlags, _ := e.GetStoredComponents(ctx, slot, root) + if existingFlags == 0 { + existingMeta = nil + } } + // Get the new data blockData, err := dataCb() if err != nil { - return false, fmt.Errorf("failed to get block data: %w", err) + return false, false, fmt.Errorf("failed to get block data: %w", err) + } + + // Calculate what we already have + var existingFlags types.BlockDataFlags + if existingMeta != nil { + existingFlags = existingMeta.storedFlags() } - metadata := &objectMetadata{ - objVersion: 1, - headerLength: uint32(len(blockData.HeaderData)), - bodyVersion: uint32(blockData.BodyVersion), - bodyLength: uint32(len(blockData.BodyData)), + // Calculate what the new data provides + var newFlags types.BlockDataFlags + if len(blockData.HeaderData) > 0 { + newFlags |= types.BlockDataFlagHeader + } + if len(blockData.BodyData) > 0 { + newFlags |= types.BlockDataFlagBody + } + if blockData.PayloadVersion != 0 && len(blockData.PayloadData) > 0 { + newFlags |= types.BlockDataFlagPayload + } + if blockData.BalVersion != 0 && len(blockData.BalData) > 0 { + newFlags |= types.BlockDataFlagBal } - if blockData.PayloadVersion != 0 { - metadata.objVersion = 2 - metadata.payloadVersion = uint32(blockData.PayloadVersion) - metadata.payloadLength = uint32(len(blockData.PayloadData)) + // Check if we need to update (new data has more components) + needsUpdate := (newFlags &^ existingFlags) != 0 + isNew := existingFlags == 0 + + if !isNew && !needsUpdate { + // Already have all the data + return false, false, nil } - metadataBytes := e.writeObjectMetadata(metadata) - metadataLength := len(metadataBytes) + // If updating, merge with existing data + finalData := blockData + if !isNew && needsUpdate { + // Fetch existing data and merge + existingData, err := e.GetBlock(ctx, slot, root, types.BlockDataFlagAll, nil, nil) + if err == nil && existingData != nil { + finalData = mergeBlockData(existingData, blockData) + } + } - // Prepare data with header and body versions and lengths - data := make([]byte, metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)) - copy(data[:metadataLength], metadataBytes) - copy(data[metadataLength:metadataLength+int(metadata.headerLength)], blockData.HeaderData) - copy(data[metadataLength+int(metadata.headerLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)], blockData.BodyData) - if metadata.objVersion == 2 { - copy(data[metadataLength+int(metadata.headerLength)+int(metadata.bodyLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)], blockData.PayloadData) + // Write object (v1 for pre-gloas, v2 for gloas+) + metaBytes := writeObjectMetadata(finalData) + + // Calculate total size and build reader chain (avoids copying to concatenated buffer) + totalSize := int64(len(metaBytes) + len(finalData.HeaderData) + len(finalData.BodyData)) + readers := []io.Reader{ + bytes.NewReader(metaBytes), + bytes.NewReader(finalData.HeaderData), + bytes.NewReader(finalData.BodyData), + } + + if finalData.BodyVersion >= uint64(spec.DataVersionGloas) { + totalSize += int64(len(finalData.PayloadData) + len(finalData.BalData)) + readers = append(readers, + bytes.NewReader(finalData.PayloadData), + bytes.NewReader(finalData.BalData), + ) } - // Upload object + // Upload object using MultiReader to stream without extra buffer allocation _, err = e.client.PutObject( ctx, e.bucket, key, - bytes.NewReader(data), - int64(len(data)), + io.MultiReader(readers...), + totalSize, minio.PutObjectOptions{ContentType: "application/octet-stream"}, ) if err != nil { - return false, fmt.Errorf("failed to upload block: %w", err) + return false, false, fmt.Errorf("failed to upload block: %w", err) + } + + return isNew, !isNew && needsUpdate, nil +} + +// mergeBlockData merges existing data with new data (new takes precedence for non-empty fields). +func mergeBlockData(existing, new *types.BlockData) *types.BlockData { + result := &types.BlockData{} + + // Use new data if available, otherwise keep existing + if len(new.HeaderData) > 0 { + result.HeaderVersion = new.HeaderVersion + result.HeaderData = new.HeaderData + } else { + result.HeaderVersion = existing.HeaderVersion + result.HeaderData = existing.HeaderData + } + + if len(new.BodyData) > 0 { + result.BodyVersion = new.BodyVersion + result.BodyData = new.BodyData + } else { + result.BodyVersion = existing.BodyVersion + result.BodyData = existing.BodyData + } + + if new.PayloadVersion != 0 && len(new.PayloadData) > 0 { + result.PayloadVersion = new.PayloadVersion + result.PayloadData = new.PayloadData + } else { + result.PayloadVersion = existing.PayloadVersion + result.PayloadData = existing.PayloadData + } + + if new.BalVersion != 0 && len(new.BalData) > 0 { + result.BalVersion = new.BalVersion + result.BalData = new.BalData + } else { + result.BalVersion = existing.BalVersion + result.BalData = existing.BalData } - return true, nil + return result } diff --git a/blockdb/tiered/tiered.go b/blockdb/tiered/tiered.go new file mode 100644 index 000000000..04f05a16f --- /dev/null +++ b/blockdb/tiered/tiered.go @@ -0,0 +1,278 @@ +package tiered + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/dora/blockdb/pebble" + "github.com/ethpandaops/dora/blockdb/s3" + "github.com/ethpandaops/dora/blockdb/types" + dtypes "github.com/ethpandaops/dora/types" +) + +// TieredEngine combines Pebble (cache) and S3 (primary storage) in a tiered architecture. +// Reads check cache first, then fall back to S3. +// Writes go to both (write-through). +type TieredEngine struct { + cache *pebble.PebbleEngine + primary *s3.S3Engine + cleanup *pebble.CacheCleanup + logger logrus.FieldLogger +} + +// NewTieredEngine creates a new tiered storage engine. +func NewTieredEngine(config dtypes.TieredBlockDBConfig, logger logrus.FieldLogger) (types.BlockDbEngine, error) { + // Initialize Pebble cache + cacheEngine, err := pebble.NewPebbleEngine(config.Pebble) + if err != nil { + return nil, fmt.Errorf("failed to initialize pebble cache: %w", err) + } + + pebbleEngine, ok := cacheEngine.(*pebble.PebbleEngine) + if !ok { + return nil, fmt.Errorf("unexpected pebble engine type") + } + + // Initialize S3 primary storage + primaryEngine, err := s3.NewS3Engine(config.S3) + if err != nil { + cacheEngine.Close() + return nil, fmt.Errorf("failed to initialize s3 primary storage: %w", err) + } + + s3Engine, ok := primaryEngine.(*s3.S3Engine) + if !ok { + cacheEngine.Close() + return nil, fmt.Errorf("unexpected s3 engine type") + } + + // Initialize cache cleanup + cleanup := pebble.NewCacheCleanup(pebbleEngine, logger) + cleanup.Start() + + return &TieredEngine{ + cache: pebbleEngine, + primary: s3Engine, + cleanup: cleanup, + logger: logger.WithField("component", "tiered-blockdb"), + }, nil +} + +// Close closes both storage engines. +func (e *TieredEngine) Close() error { + if e.cleanup != nil { + e.cleanup.Stop() + } + + var errs []error + if err := e.cache.Close(); err != nil { + errs = append(errs, fmt.Errorf("cache close: %w", err)) + } + if err := e.primary.Close(); err != nil { + errs = append(errs, fmt.Errorf("primary close: %w", err)) + } + + if len(errs) > 0 { + return errs[0] + } + return nil +} + +// GetStoredComponents returns which components exist for a block. +// Checks cache first, then S3. +func (e *TieredEngine) GetStoredComponents(ctx context.Context, slot uint64, root []byte) (types.BlockDataFlags, error) { + // Check cache first + cacheFlags, err := e.cache.GetStoredComponents(ctx, slot, root) + if err != nil { + e.logger.Debugf("cache GetStoredComponents error: %v", err) + } + + // If cache has all components, return early + if cacheFlags == types.BlockDataFlagAll { + return cacheFlags, nil + } + + // Check S3 for additional components + s3Flags, err := e.primary.GetStoredComponents(ctx, slot, root) + if err != nil { + return cacheFlags, nil // Return cache result on S3 error + } + + return cacheFlags | s3Flags, nil +} + +// GetBlock retrieves block data with selective loading. +// Checks cache first, fetches missing components from S3. +func (e *TieredEngine) GetBlock( + ctx context.Context, + slot uint64, + root []byte, + flags types.BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), +) (*types.BlockData, error) { + // Check what's in cache + cacheFlags, _ := e.cache.GetStoredComponents(ctx, slot, root) + + // Determine what we can get from cache vs S3 + cacheRequestFlags := flags & cacheFlags + s3RequestFlags := flags &^ cacheFlags + + result := &types.BlockData{} + + // Get from cache + if cacheRequestFlags != 0 { + cacheData, err := e.cache.GetBlock(ctx, slot, root, cacheRequestFlags, parseBlock, parsePayload) + if err != nil { + e.logger.Debugf("cache GetBlock error: %v", err) + } else if cacheData != nil { + mergeBlockDataInto(result, cacheData) + + // Record LRU access + if e.cleanup != nil { + e.cleanup.RecordAccess(root, cacheRequestFlags) + } + } + } + + // Get missing components from S3 + if s3RequestFlags != 0 { + s3Data, err := e.primary.GetBlock(ctx, slot, root, s3RequestFlags, parseBlock, parsePayload) + if err != nil { + e.logger.Debugf("s3 GetBlock error: %v", err) + } else if s3Data != nil { + mergeBlockDataInto(result, s3Data) + + // Cache the S3 data for future reads + e.cacheS3Data(ctx, slot, root, s3Data, s3RequestFlags) + } + } + + return result, nil +} + +// cacheS3Data stores S3 data in the cache for future reads. +func (e *TieredEngine) cacheS3Data(ctx context.Context, slot uint64, root []byte, data *types.BlockData, flags types.BlockDataFlags) { + // Build cache data with only the components we fetched from S3 + cacheData := &types.BlockData{} + + if flags.Has(types.BlockDataFlagHeader) && len(data.HeaderData) > 0 { + cacheData.HeaderVersion = data.HeaderVersion + cacheData.HeaderData = data.HeaderData + } + if flags.Has(types.BlockDataFlagBody) && len(data.BodyData) > 0 { + cacheData.BodyVersion = data.BodyVersion + cacheData.BodyData = data.BodyData + } + if flags.Has(types.BlockDataFlagPayload) && len(data.PayloadData) > 0 { + cacheData.PayloadVersion = data.PayloadVersion + cacheData.PayloadData = data.PayloadData + } + if flags.Has(types.BlockDataFlagBal) && len(data.BalData) > 0 { + cacheData.BalVersion = data.BalVersion + cacheData.BalData = data.BalData + } + + // Add to cache (ignore errors - caching is best effort) + _, _, err := e.cache.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { + return cacheData, nil + }) + if err != nil { + e.logger.Debugf("failed to cache S3 data: %v", err) + } + + // Flush LRU updates since we did a write + if e.cleanup != nil { + e.cleanup.FlushLRU() + } +} + +// AddBlock stores block data using write-through to both cache and S3. +// Returns (added, updated, error). +func (e *TieredEngine) AddBlock( + ctx context.Context, + slot uint64, + root []byte, + dataCb func() (*types.BlockData, error), +) (bool, bool, error) { + // Get the data once + data, err := dataCb() + if err != nil { + return false, false, err + } + + // Check what components already exist (in cache or S3) + existingFlags, _ := e.GetStoredComponents(ctx, slot, root) + + // Determine what new data provides + var newFlags types.BlockDataFlags + if len(data.HeaderData) > 0 { + newFlags |= types.BlockDataFlagHeader + } + if len(data.BodyData) > 0 { + newFlags |= types.BlockDataFlagBody + } + if data.PayloadVersion != 0 && len(data.PayloadData) > 0 { + newFlags |= types.BlockDataFlagPayload + } + if data.BalVersion != 0 && len(data.BalData) > 0 { + newFlags |= types.BlockDataFlagBal + } + + // Check if we need to update + needsUpdate := (newFlags &^ existingFlags) != 0 + isNew := existingFlags == 0 + + if !isNew && !needsUpdate { + return false, false, nil + } + + // Write-through: write to S3 first (primary), then cache + // S3 handles merging with existing data + s3Added, s3Updated, err := e.primary.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { + return data, nil + }) + if err != nil { + return false, false, fmt.Errorf("failed to write to S3: %w", err) + } + + // Write to cache + _, _, err = e.cache.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { + return data, nil + }) + if err != nil { + e.logger.Warnf("failed to write to cache: %v", err) + // Don't fail - S3 write succeeded + } + + // Flush LRU updates after write + if e.cleanup != nil { + e.cleanup.FlushLRU() + } + + return s3Added, s3Updated, nil +} + +// mergeBlockDataInto merges source data into target (source values take precedence for non-empty fields). +func mergeBlockDataInto(target, source *types.BlockData) { + if source.HeaderVersion != 0 || len(source.HeaderData) > 0 { + target.HeaderVersion = source.HeaderVersion + target.HeaderData = source.HeaderData + } + if source.BodyVersion != 0 || len(source.BodyData) > 0 { + target.BodyVersion = source.BodyVersion + target.BodyData = source.BodyData + target.Body = source.Body + } + if source.PayloadVersion != 0 || len(source.PayloadData) > 0 { + target.PayloadVersion = source.PayloadVersion + target.PayloadData = source.PayloadData + target.Payload = source.Payload + } + if source.BalVersion != 0 || len(source.BalData) > 0 { + target.BalVersion = source.BalVersion + target.BalData = source.BalData + } +} diff --git a/blockdb/types/engine.go b/blockdb/types/engine.go index 33d3c7fc0..a10bdc66f 100644 --- a/blockdb/types/engine.go +++ b/blockdb/types/engine.go @@ -2,18 +2,54 @@ package types import "context" +// BlockData contains all data components for a block. type BlockData struct { - HeaderVersion uint64 - HeaderData []byte - BodyVersion uint64 - BodyData []byte - Body interface{} + // Header data + HeaderVersion uint64 + HeaderData []byte + + // Body data + BodyVersion uint64 + BodyData []byte + Body any // Parsed body (optional) + + // Execution payload data (ePBS) PayloadVersion uint64 PayloadData []byte - Payload interface{} + Payload any // Parsed payload (optional) + + // Block access list data + BalVersion uint64 + BalData []byte } + +// BlockDbEngine defines the interface for block database engines. type BlockDbEngine interface { + // Close closes the database engine. Close() error - GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*BlockData, error) - AddBlock(ctx context.Context, slot uint64, root []byte, dataCb func() (*BlockData, error)) (bool, error) + + // GetBlock retrieves block data with selective loading based on flags. + // If parseBlock is nil, raw body data is stored in BlockData.BodyData. + // If parsePayload is nil, raw payload data is stored in BlockData.PayloadData. + GetBlock( + ctx context.Context, + slot uint64, + root []byte, + flags BlockDataFlags, + parseBlock func(uint64, []byte) (any, error), + parsePayload func(uint64, []byte) (any, error), + ) (*BlockData, error) + + // AddBlock stores block data. Returns: + // - added: true if a new block was created + // - updated: true if an existing block was updated with new components + AddBlock( + ctx context.Context, + slot uint64, + root []byte, + dataCb func() (*BlockData, error), + ) (added bool, updated bool, err error) + + // GetStoredComponents returns which components exist for a block. + GetStoredComponents(ctx context.Context, slot uint64, root []byte) (BlockDataFlags, error) } diff --git a/blockdb/types/flags.go b/blockdb/types/flags.go new file mode 100644 index 000000000..34aff4dbb --- /dev/null +++ b/blockdb/types/flags.go @@ -0,0 +1,38 @@ +package types + +// BlockDataFlags specifies which components to load from storage. +type BlockDataFlags uint8 + +const ( + // BlockDataFlagHeader requests the block header data. + BlockDataFlagHeader BlockDataFlags = 1 << iota // 0x01 + // BlockDataFlagBody requests the block body data. + BlockDataFlagBody // 0x02 + // BlockDataFlagPayload requests the execution payload data. + BlockDataFlagPayload // 0x04 + // BlockDataFlagBal requests the block access list data. + BlockDataFlagBal // 0x08 + + // BlockDataFlagAll requests all block components. + BlockDataFlagAll = BlockDataFlagHeader | BlockDataFlagBody | BlockDataFlagPayload | BlockDataFlagBal +) + +// Has returns true if the flag set contains the specified flag. +func (f BlockDataFlags) Has(flag BlockDataFlags) bool { + return f&flag == flag +} + +// HasAny returns true if the flag set contains any of the specified flags. +func (f BlockDataFlags) HasAny(flags BlockDataFlags) bool { + return f&flags != 0 +} + +// Add returns a new flag set with the specified flag added. +func (f BlockDataFlags) Add(flag BlockDataFlags) BlockDataFlags { + return f | flag +} + +// Remove returns a new flag set with the specified flag removed. +func (f BlockDataFlags) Remove(flag BlockDataFlags) BlockDataFlags { + return f &^ flag +} diff --git a/cmd/dora-utils/blockdb_sync.go b/cmd/dora-utils/blockdb_sync.go index 0385017bc..7905d7e9f 100644 --- a/cmd/dora-utils/blockdb_sync.go +++ b/cmd/dora-utils/blockdb_sync.go @@ -271,7 +271,7 @@ func processSlot(ctx context.Context, pool *consensus.Pool, dynSsz *dynssz.DynSs return slotResult{slot: slot, err: fmt.Errorf("failed to marshal block header for slot %d: %v", slot, err), time: time.Since(t1)} } - added, err := blockdb.GlobalBlockDb.AddBlockWithCallback(ctx, slot, blockHeader.Root[:], func() (*btypes.BlockData, error) { + added, _, err := blockdb.GlobalBlockDb.AddBlockWithCallback(ctx, slot, blockHeader.Root[:], func() (*btypes.BlockData, error) { blockBody, err := client.GetRPCClient().GetBlockBodyByBlockroot(ctx, blockHeader.Root) if err != nil { return nil, fmt.Errorf("failed to get block body for slot %d: %v", slot, err) diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index aea81b98f..4a7a5d21f 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -534,7 +534,7 @@ func (block *Block) writeToBlockDb() error { return nil } - _, err := blockdb.GlobalBlockDb.AddBlockWithCallback(context.Background(), uint64(block.Slot), block.Root[:], func() (*btypes.BlockData, error) { + _, _, err := blockdb.GlobalBlockDb.AddBlockWithCallback(context.Background(), uint64(block.Slot), block.Root[:], func() (*btypes.BlockData, error) { headerSSZ, err := block.header.MarshalSSZ() if err != nil { return nil, fmt.Errorf("marshal header ssz failed: %v", err) diff --git a/services/chainservice.go b/services/chainservice.go index 634d5626c..8677cf7fa 100644 --- a/services/chainservice.go +++ b/services/chainservice.go @@ -252,6 +252,13 @@ func (cs *ChainService) StartService() error { return fmt.Errorf("failed initializing s3 blockdb: %v", err) } cs.logger.Infof("S3 blockdb initialized at %v", utils.Config.BlockDb.S3.Bucket) + case "tiered": + err := blockdb.InitWithTiered(utils.Config.BlockDb.Tiered, cs.logger) + if err != nil { + return fmt.Errorf("failed initializing tiered blockdb: %v", err) + } + cs.logger.Infof("Tiered blockdb initialized (Pebble cache: %v, S3: %v)", + utils.Config.BlockDb.Tiered.Pebble.Path, utils.Config.BlockDb.Tiered.S3.Bucket) default: cs.logger.Infof("Blockdb disabled") } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index ffb525dd5..1cce33123 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -13,6 +13,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" + btypes "github.com/ethpandaops/dora/blockdb/types" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" @@ -174,12 +175,14 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot // try loading from block db if result == nil && header != nil && blockdb.GlobalBlockDb != nil { - blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(header.Message.Slot), blockroot[:], func(version uint64, block []byte) (interface{}, error) { - return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) - }, func(version uint64, payload []byte) (interface{}, error) { - return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) - }) - if err == nil && blockData != nil { + blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(header.Message.Slot), blockroot[:], + btypes.BlockDataFlagBody|btypes.BlockDataFlagPayload, + func(version uint64, block []byte) (any, error) { + return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (any, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) + }) + if err == nil && blockData != nil && blockData.Body != nil { result = &CombinedBlockResponse{ Root: blockroot, Header: header, @@ -313,12 +316,14 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl // try loading from block db if result == nil && header != nil && blockdb.GlobalBlockDb != nil { - blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(slot), blockRoot[:], func(version uint64, block []byte) (interface{}, error) { - return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) - }, func(version uint64, payload []byte) (interface{}, error) { - return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) - }) - if err == nil && blockData != nil { + blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(slot), blockRoot[:], + btypes.BlockDataFlagHeader|btypes.BlockDataFlagBody|btypes.BlockDataFlagPayload, + func(version uint64, block []byte) (any, error) { + return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (any, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) + }) + if err == nil && blockData != nil && blockData.Body != nil { header := &phase0.SignedBeaconBlockHeader{} err = header.UnmarshalSSZ(blockData.HeaderData) if err != nil { diff --git a/types/config.go b/types/config.go index b67df1284..ba9680ad6 100644 --- a/types/config.go +++ b/types/config.go @@ -144,9 +144,10 @@ type Config struct { Database DatabaseConfig `yaml:"database"` BlockDb struct { - Engine string `yaml:"engine" envconfig:"BLOCKDB_ENGINE"` + Engine string `yaml:"engine" envconfig:"BLOCKDB_ENGINE"` // "pebble", "s3", or "tiered" Pebble PebbleBlockDBConfig `yaml:"pebble"` S3 S3BlockDBConfig `yaml:"s3"` + Tiered TieredBlockDBConfig `yaml:"tiered"` // For tiered storage (Pebble cache + S3 backend) } `yaml:"blockDb"` KillSwitch struct { @@ -238,11 +239,30 @@ type PgsqlWriterDatabaseConfig struct { MaxIdleConns int `yaml:"maxIdleConns" envconfig:"DATABASE_PGSQL_WRITER_MAX_IDLE_CONNS"` } +// BlockDbRetentionConfig configures per-object-type retention behavior. +type BlockDbRetentionConfig struct { + Enabled bool `yaml:"enabled"` + RetentionTime time.Duration `yaml:"retentionTime"` // For age-based cleanup + MaxSize int64 `yaml:"maxSize"` // Size limit in MB (0 = unlimited) + CleanupMode string `yaml:"cleanupMode"` // "age" or "lru" +} + +// PebbleBlockDBConfig configures the Pebble (local) storage engine. type PebbleBlockDBConfig struct { - Path string `yaml:"path" envconfig:"BLOCKDB_ROCKSDB_PATH"` - CacheSize int `yaml:"cacheSize" envconfig:"BLOCKDB_ROCKSDB_CACHE_SIZE"` + Path string `yaml:"path" envconfig:"BLOCKDB_PEBBLE_PATH"` + CacheSize int `yaml:"cacheSize" envconfig:"BLOCKDB_PEBBLE_CACHE_SIZE"` // Pebble internal cache in MB + + // Per-object-type retention configuration (used in tiered mode) + HeaderRetention BlockDbRetentionConfig `yaml:"headerRetention"` + BodyRetention BlockDbRetentionConfig `yaml:"bodyRetention"` + PayloadRetention BlockDbRetentionConfig `yaml:"payloadRetention"` + BalRetention BlockDbRetentionConfig `yaml:"balRetention"` + + // Cleanup configuration + CleanupInterval time.Duration `yaml:"cleanupInterval" envconfig:"BLOCKDB_PEBBLE_CLEANUP_INTERVAL"` } +// S3BlockDBConfig configures the S3 (remote) storage engine. type S3BlockDBConfig struct { Endpoint string `yaml:"endpoint" envconfig:"BLOCKDB_S3_ENDPOINT"` Secure bool `yaml:"secure" envconfig:"BLOCKDB_S3_SECURE"` @@ -253,4 +273,11 @@ type S3BlockDBConfig struct { Path string `yaml:"path" envconfig:"BLOCKDB_S3_PATH"` MaxConcurrentUploads uint `yaml:"maxConcurrentUploads" envconfig:"BLOCKDB_S3_MAX_CONCURRENT_UPLOADS"` UploadQueueSize uint `yaml:"uploadQueueSize" envconfig:"BLOCKDB_S3_UPLOAD_QUEUE_SIZE"` + EnableRangeRequests bool `yaml:"enableRangeRequests" envconfig:"BLOCKDB_S3_ENABLE_RANGE_REQUESTS"` // Use HTTP Range requests for selective loading +} + +// TieredBlockDBConfig configures tiered storage (Pebble cache + S3 backend). +type TieredBlockDBConfig struct { + Pebble PebbleBlockDBConfig `yaml:"pebble"` + S3 S3BlockDBConfig `yaml:"s3"` } From 1ba48fd0835381365f66016369a58a46734df044 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 15 Jan 2026 22:10:52 +0100 Subject: [PATCH 26/99] index & display builders --- cmd/dora-explorer/main.go | 1 + db/builders.go | 449 +++++++++++ .../pgsql/20260108202212_epbs-payload.sql | 19 + .../sqlite/20260108202212_epbs-payload.sql | 15 + dbtypes/dbtypes.go | 10 + dbtypes/other.go | 37 + handlers/builders.go | 294 +++++++ indexer/beacon/buildercache.go | 751 ++++++++++++++++++ indexer/beacon/epochstate.go | 15 + indexer/beacon/finalization.go | 1 + indexer/beacon/indexer.go | 10 + indexer/beacon/indexer_getter.go | 50 ++ services/chainservice_builder.go | 253 ++++++ templates/builders/builders.html | 303 +++++++ types/models/builders.go | 50 ++ 15 files changed, 2258 insertions(+) create mode 100644 db/builders.go create mode 100644 handlers/builders.go create mode 100644 indexer/beacon/buildercache.go create mode 100644 services/chainservice_builder.go create mode 100644 templates/builders/builders.html create mode 100644 types/models/builders.go diff --git a/cmd/dora-explorer/main.go b/cmd/dora-explorer/main.go index 3bb577bbd..eee977e44 100644 --- a/cmd/dora-explorer/main.go +++ b/cmd/dora-explorer/main.go @@ -231,6 +231,7 @@ func startFrontend(router *mux.Router) { router.HandleFunc("/validators/submit_withdrawals", handlers.SubmitWithdrawal).Methods("GET") router.HandleFunc("/validator/{idxOrPubKey}", handlers.Validator).Methods("GET") router.HandleFunc("/validator/{index}/slots", handlers.ValidatorSlots).Methods("GET") + router.HandleFunc("/builders", handlers.Builders).Methods("GET") if utils.Config.Frontend.Pprof { // add pprof handler diff --git a/db/builders.go b/db/builders.go new file mode 100644 index 000000000..cb7d314d8 --- /dev/null +++ b/db/builders.go @@ -0,0 +1,449 @@ +package db + +import ( + "fmt" + "strings" + + "github.com/ethpandaops/dora/dbtypes" + "github.com/jmoiron/sqlx" +) + +// InsertBuilder inserts a single builder into the database +func InsertBuilder(builder *dbtypes.Builder, tx *sqlx.Tx) error { + _, err := tx.Exec(EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: ` + INSERT INTO builders ( + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (pubkey) DO UPDATE SET + builder_index = excluded.builder_index, + version = excluded.version, + execution_address = excluded.execution_address, + deposit_epoch = excluded.deposit_epoch, + withdrawable_epoch = excluded.withdrawable_epoch, + superseded = excluded.superseded`, + dbtypes.DBEngineSqlite: ` + INSERT OR REPLACE INTO builders ( + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + }), + builder.Pubkey, + builder.BuilderIndex, + builder.Version, + builder.ExecutionAddress, + builder.DepositEpoch, + builder.WithdrawableEpoch, + builder.Superseded) + + if err != nil { + return fmt.Errorf("error inserting builder: %w", err) + } + return nil +} + +// InsertBuilderBatch inserts multiple builders in a batch +func InsertBuilderBatch(builders []*dbtypes.Builder, tx *sqlx.Tx) error { + if len(builders) == 0 { + return nil + } + + valueStrings := make([]string, len(builders)) + valueArgs := make([]any, 0, len(builders)*7) + for i, b := range builders { + valueStrings[i] = fmt.Sprintf("($%v, $%v, $%v, $%v, $%v, $%v, $%v)", + i*7+1, i*7+2, i*7+3, i*7+4, i*7+5, i*7+6, i*7+7) + valueArgs = append(valueArgs, + b.Pubkey, + b.BuilderIndex, + b.Version, + b.ExecutionAddress, + b.DepositEpoch, + b.WithdrawableEpoch, + b.Superseded) + } + + stmt := fmt.Sprintf(EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: ` + INSERT INTO builders ( + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + ) VALUES %s + ON CONFLICT (pubkey) DO UPDATE SET + builder_index = excluded.builder_index, + version = excluded.version, + execution_address = excluded.execution_address, + deposit_epoch = excluded.deposit_epoch, + withdrawable_epoch = excluded.withdrawable_epoch, + superseded = excluded.superseded`, + dbtypes.DBEngineSqlite: ` + INSERT OR REPLACE INTO builders ( + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + ) VALUES %s`, + }), strings.Join(valueStrings, ",")) + + _, err := tx.Exec(stmt, valueArgs...) + if err != nil { + return fmt.Errorf("error inserting builder batch: %w", err) + } + + return nil +} + +// GetBuilderByPubkey returns a builder by pubkey (primary key) +func GetBuilderByPubkey(pubkey []byte) *dbtypes.Builder { + builder := dbtypes.Builder{} + err := ReaderDb.Get(&builder, ` + SELECT * FROM builders WHERE pubkey = $1 + `, pubkey) + if err != nil { + return nil + } + return &builder +} + +// GetActiveBuilderByIndex returns the active (non-superseded) builder for a given index +func GetActiveBuilderByIndex(index uint64) *dbtypes.Builder { + builder := dbtypes.Builder{} + err := ReaderDb.Get(&builder, ` + SELECT * FROM builders WHERE builder_index = $1 AND superseded = false + `, index) + if err != nil { + return nil + } + return &builder +} + +// GetBuildersByIndex returns all builders (including superseded) for a given index +func GetBuildersByIndex(index uint64) []*dbtypes.Builder { + builders := []*dbtypes.Builder{} + err := ReaderDb.Select(&builders, ` + SELECT * FROM builders WHERE builder_index = $1 ORDER BY superseded ASC + `, index) + if err != nil { + logger.Errorf("Error while fetching builders by index: %v", err) + return nil + } + return builders +} + +// GetBuilderRange returns builders in a given index range (only active builders) +func GetBuilderRange(startIndex uint64, endIndex uint64) []*dbtypes.Builder { + builders := []*dbtypes.Builder{} + err := ReaderDb.Select(&builders, ` + SELECT * FROM builders + WHERE builder_index >= $1 AND builder_index <= $2 AND superseded = false + ORDER BY builder_index ASC + `, startIndex, endIndex) + if err != nil { + logger.Errorf("Error while fetching builder range: %v", err) + return nil + } + return builders +} + +// GetMaxBuilderIndex returns the highest builder index in the database +func GetMaxBuilderIndex() (uint64, error) { + var maxIndex uint64 + err := ReaderDb.Get(&maxIndex, "SELECT COALESCE(MAX(builder_index), 0) FROM builders") + if err != nil { + return 0, fmt.Errorf("error getting max builder index: %w", err) + } + return maxIndex, nil +} + +// GetBuilderCount returns the count of builders (optionally only active) +func GetBuilderCount(activeOnly bool) (uint64, error) { + var count uint64 + var err error + if activeOnly { + err = ReaderDb.Get(&count, "SELECT COUNT(*) FROM builders WHERE superseded = false") + } else { + err = ReaderDb.Get(&count, "SELECT COUNT(*) FROM builders") + } + if err != nil { + return 0, fmt.Errorf("error getting builder count: %w", err) + } + return count, nil +} + +// SetBuilderSuperseded marks a builder as superseded +func SetBuilderSuperseded(pubkey []byte, tx *sqlx.Tx) error { + _, err := tx.Exec(` + UPDATE builders SET superseded = true WHERE pubkey = $1 + `, pubkey) + if err != nil { + return fmt.Errorf("error setting builder superseded: %w", err) + } + return nil +} + +// SetBuildersSuperseded marks multiple builders as superseded in a batch +func SetBuildersSuperseded(pubkeys [][]byte, tx *sqlx.Tx) error { + if len(pubkeys) == 0 { + return nil + } + + var sql strings.Builder + sql.WriteString("UPDATE builders SET superseded = true WHERE pubkey IN (") + + args := make([]any, len(pubkeys)) + for i, pk := range pubkeys { + if i > 0 { + sql.WriteString(", ") + } + fmt.Fprintf(&sql, "$%d", i+1) + args[i] = pk + } + sql.WriteString(")") + + _, err := tx.Exec(sql.String(), args...) + if err != nil { + return fmt.Errorf("error setting builders superseded: %w", err) + } + return nil +} + +// StreamBuildersByPubkeys streams builders by pubkeys in batches +func StreamBuildersByPubkeys(pubkeys [][]byte, cb func(builder *dbtypes.Builder) bool) error { + const batchSize = 1000 + + for i := 0; i < len(pubkeys); i += batchSize { + end := min(i+batchSize, len(pubkeys)) + batch := pubkeys[i:end] + + var sql strings.Builder + fmt.Fprintf(&sql, ` + SELECT + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + FROM builders + WHERE pubkey in (`) + + args := make([]any, len(batch)) + for j, pk := range batch { + if j > 0 { + fmt.Fprintf(&sql, ", ") + } + fmt.Fprintf(&sql, "$%v", j+1) + args[j] = pk + } + fmt.Fprintf(&sql, ")") + + // Create pubkey map for ordering + pubkeyMap := make(map[string]int, len(batch)) + for pos, pk := range batch { + pubkeyMap[string(pk)] = pos + } + + // Fetch all builders for this batch + builders := make([]*dbtypes.Builder, len(batch)) + rows, err := ReaderDb.Query(sql.String(), args...) + if err != nil { + return fmt.Errorf("error querying builders: %w", err) + } + defer rows.Close() + + for rows.Next() { + builder := &dbtypes.Builder{} + err := rows.Scan( + &builder.Pubkey, + &builder.BuilderIndex, + &builder.Version, + &builder.ExecutionAddress, + &builder.DepositEpoch, + &builder.WithdrawableEpoch, + &builder.Superseded, + ) + if err != nil { + return fmt.Errorf("error scanning builder: %w", err) + } + pos := pubkeyMap[string(builder.Pubkey)] + builders[pos] = builder + } + + if err = rows.Err(); err != nil { + return fmt.Errorf("error iterating rows: %w", err) + } + + // Stream in original order + for _, b := range builders { + if b != nil && !cb(b) { + return nil + } + } + } + + return nil +} + +// GetBuildersByExecutionAddress returns builders with a specific execution address +func GetBuildersByExecutionAddress(address []byte) []*dbtypes.Builder { + builders := []*dbtypes.Builder{} + err := ReaderDb.Select(&builders, ` + SELECT * FROM builders WHERE execution_address = $1 ORDER BY builder_index ASC + `, address) + if err != nil { + logger.Errorf("Error while fetching builders by execution address: %v", err) + return nil + } + return builders +} + +// GetBuilderIndexesByFilter returns builder indexes matching a filter +func GetBuilderIndexesByFilter(filter dbtypes.BuilderFilter, currentEpoch uint64) ([]uint64, error) { + var sql strings.Builder + args := []interface{}{} + fmt.Fprint(&sql, ` + SELECT + builder_index + FROM builders + `) + + args = buildBuilderFilterSql(filter, currentEpoch, &sql, args) + + switch filter.OrderBy { + case dbtypes.BuilderOrderIndexAsc: + fmt.Fprint(&sql, " ORDER BY builder_index ASC") + case dbtypes.BuilderOrderIndexDesc: + fmt.Fprint(&sql, " ORDER BY builder_index DESC") + case dbtypes.BuilderOrderPubKeyAsc: + fmt.Fprint(&sql, " ORDER BY pubkey ASC") + case dbtypes.BuilderOrderPubKeyDesc: + fmt.Fprint(&sql, " ORDER BY pubkey DESC") + case dbtypes.BuilderOrderDepositEpochAsc: + fmt.Fprint(&sql, " ORDER BY deposit_epoch ASC") + case dbtypes.BuilderOrderDepositEpochDesc: + fmt.Fprint(&sql, " ORDER BY deposit_epoch DESC") + case dbtypes.BuilderOrderWithdrawableEpochAsc: + fmt.Fprint(&sql, " ORDER BY withdrawable_epoch ASC") + case dbtypes.BuilderOrderWithdrawableEpochDesc: + fmt.Fprint(&sql, " ORDER BY withdrawable_epoch DESC") + } + + builderIds := []uint64{} + err := ReaderDb.Select(&builderIds, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching builders by filter: %v", err) + return nil, err + } + + return builderIds, nil +} + +func buildBuilderFilterSql(filter dbtypes.BuilderFilter, currentEpoch uint64, sql *strings.Builder, args []interface{}) []interface{} { + filterOp := "WHERE" + + if filter.MinIndex != nil { + fmt.Fprintf(sql, " %v builder_index >= $%v", filterOp, len(args)+1) + args = append(args, *filter.MinIndex) + filterOp = "AND" + } + if filter.MaxIndex != nil { + fmt.Fprintf(sql, " %v builder_index <= $%v", filterOp, len(args)+1) + args = append(args, *filter.MaxIndex) + filterOp = "AND" + } + if len(filter.PubKey) > 0 { + fmt.Fprintf(sql, " %v pubkey LIKE $%v", filterOp, len(args)+1) + args = append(args, append(filter.PubKey, '%')) + filterOp = "AND" + } + if len(filter.ExecutionAddress) > 0 { + fmt.Fprintf(sql, " %v execution_address = $%v", filterOp, len(args)+1) + args = append(args, filter.ExecutionAddress) + filterOp = "AND" + } + if len(filter.Status) > 0 { + statusConditions := make([]string, 0, len(filter.Status)) + for _, status := range filter.Status { + switch status { + case dbtypes.BuilderStatusActiveFilter: + statusConditions = append(statusConditions, fmt.Sprintf("(superseded = false AND withdrawable_epoch > $%v)", len(args)+1)) + args = append(args, ConvertUint64ToInt64(currentEpoch)) + case dbtypes.BuilderStatusExitedFilter: + statusConditions = append(statusConditions, fmt.Sprintf("(superseded = false AND withdrawable_epoch <= $%v)", len(args)+1)) + args = append(args, ConvertUint64ToInt64(currentEpoch)) + case dbtypes.BuilderStatusSupersededFilter: + statusConditions = append(statusConditions, "superseded = true") + } + } + if len(statusConditions) > 0 { + fmt.Fprintf(sql, " %v (%v)", filterOp, strings.Join(statusConditions, " OR ")) + } + } + + return args +} + +// StreamBuildersByIndexes streams builders by indexes +func StreamBuildersByIndexes(indexes []uint64, cb func(builder *dbtypes.Builder) bool) { + const batchSize = 1000 + + for i := 0; i < len(indexes); i += batchSize { + end := min(i+batchSize, len(indexes)) + batch := indexes[i:end] + + var sql strings.Builder + fmt.Fprint(&sql, ` + SELECT + pubkey, builder_index, version, execution_address, + deposit_epoch, withdrawable_epoch, superseded + FROM builders + WHERE builder_index IN (`) + + args := make([]any, len(batch)) + for j, idx := range batch { + if j > 0 { + fmt.Fprint(&sql, ", ") + } + fmt.Fprintf(&sql, "$%v", j+1) + args[j] = idx + } + fmt.Fprint(&sql, ")") + + // Create index map for ordering + indexMap := make(map[uint64]int, len(batch)) + for pos, idx := range batch { + indexMap[idx] = pos + } + + // Fetch all builders for this batch + builders := make([]*dbtypes.Builder, len(batch)) + rows, err := ReaderDb.Query(sql.String(), args...) + if err != nil { + logger.Errorf("Error querying builders: %v", err) + return + } + + for rows.Next() { + builder := &dbtypes.Builder{} + err := rows.Scan( + &builder.Pubkey, + &builder.BuilderIndex, + &builder.Version, + &builder.ExecutionAddress, + &builder.DepositEpoch, + &builder.WithdrawableEpoch, + &builder.Superseded, + ) + if err != nil { + logger.Errorf("Error scanning builder: %v", err) + rows.Close() + return + } + pos := indexMap[builder.BuilderIndex] + builders[pos] = builder + } + rows.Close() + + // Stream in original order + for _, b := range builders { + if b != nil && !cb(b) { + return + } + } + } +} diff --git a/db/schema/pgsql/20260108202212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql index 7432fb9e1..10de8df5f 100644 --- a/db/schema/pgsql/20260108202212_epbs-payload.sql +++ b/db/schema/pgsql/20260108202212_epbs-payload.sql @@ -47,6 +47,25 @@ CREATE INDEX IF NOT EXISTS "block_bids_slot_idx" ON public."block_bids" ("slot" ASC NULLS LAST); +CREATE TABLE IF NOT EXISTS public."builders" ( + "pubkey" bytea NOT NULL, + "builder_index" bigint NOT NULL, + "version" smallint NOT NULL, + "execution_address" bytea NOT NULL, + "deposit_epoch" bigint NOT NULL, + "withdrawable_epoch" bigint NOT NULL, + "superseded" boolean NOT NULL DEFAULT false, + CONSTRAINT builders_pkey PRIMARY KEY (pubkey) +); + +CREATE INDEX IF NOT EXISTS "builders_builder_index_idx" + ON public."builders" + ("builder_index" ASC NULLS LAST); + +CREATE INDEX IF NOT EXISTS "builders_execution_address_idx" + ON public."builders" + ("execution_address" ASC NULLS LAST); + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/schema/sqlite/20260108202212_epbs-payload.sql b/db/schema/sqlite/20260108202212_epbs-payload.sql index 41f1343c4..4535d7cb6 100644 --- a/db/schema/sqlite/20260108202212_epbs-payload.sql +++ b/db/schema/sqlite/20260108202212_epbs-payload.sql @@ -34,6 +34,21 @@ CREATE INDEX IF NOT EXISTS "block_bids_builder_index_idx" ON "block_bids" ("buil CREATE INDEX IF NOT EXISTS "block_bids_slot_idx" ON "block_bids" ("slot" ASC); +CREATE TABLE IF NOT EXISTS "builders" ( + "pubkey" BLOB NOT NULL, + "builder_index" BIGINT NOT NULL, + "version" SMALLINT NOT NULL, + "execution_address" BLOB NOT NULL, + "deposit_epoch" BIGINT NOT NULL, + "withdrawable_epoch" BIGINT NOT NULL, + "superseded" BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (pubkey) +); + +CREATE INDEX IF NOT EXISTS "builders_builder_index_idx" ON "builders" ("builder_index" ASC); + +CREATE INDEX IF NOT EXISTS "builders_execution_address_idx" ON "builders" ("execution_address" ASC); + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index cadaa7f15..74b7daea8 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -559,6 +559,16 @@ type BlockBid struct { ElPayment uint64 `db:"el_payment"` } +type Builder struct { + Pubkey []byte `db:"pubkey"` + BuilderIndex uint64 `db:"builder_index"` + Version uint8 `db:"version"` + ExecutionAddress []byte `db:"execution_address"` + DepositEpoch int64 `db:"deposit_epoch"` + WithdrawableEpoch int64 `db:"withdrawable_epoch"` + Superseded bool `db:"superseded"` +} + // Withdrawal types const ( WithdrawalTypeBeaconWithdrawal = 0 diff --git a/dbtypes/other.go b/dbtypes/other.go index d3755ec59..8871329fa 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -211,6 +211,43 @@ type ValidatorFilter struct { Offset uint64 } +// Builder filter types + +type BuilderOrder uint8 + +const ( + BuilderOrderIndexAsc BuilderOrder = iota + BuilderOrderIndexDesc + BuilderOrderPubKeyAsc + BuilderOrderPubKeyDesc + BuilderOrderBalanceAsc + BuilderOrderBalanceDesc + BuilderOrderDepositEpochAsc + BuilderOrderDepositEpochDesc + BuilderOrderWithdrawableEpochAsc + BuilderOrderWithdrawableEpochDesc +) + +type BuilderStatus uint8 + +const ( + BuilderStatusActiveFilter BuilderStatus = iota + BuilderStatusExitedFilter + BuilderStatusSupersededFilter +) + +type BuilderFilter struct { + MinIndex *uint64 + MaxIndex *uint64 + PubKey []byte + ExecutionAddress []byte + Status []BuilderStatus + + OrderBy BuilderOrder + Limit uint64 + Offset uint64 +} + // EL Explorer filters type ElTransactionFilter struct { diff --git a/handlers/builders.go b/handlers/builders.go new file mode 100644 index 000000000..ad1d90075 --- /dev/null +++ b/handlers/builders.go @@ -0,0 +1,294 @@ +package handlers + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/dora/templates" + "github.com/ethpandaops/dora/types/models" + "github.com/sirupsen/logrus" +) + +// Builders will return the main "builders" page using a go template +func Builders(w http.ResponseWriter, r *http.Request) { + var buildersTemplateFiles = append(layoutTemplateFiles, + "builders/builders.html", + "_svg/professor.html", + ) + + var pageTemplate = templates.GetTemplate(buildersTemplateFiles...) + data := InitPageData(w, r, "builders", "/builders", "Builders", buildersTemplateFiles) + + urlArgs := r.URL.Query() + var pageNumber uint64 = 1 + if urlArgs.Has("p") { + pageNumber, _ = strconv.ParseUint(urlArgs.Get("p"), 10, 64) + } + var pageSize uint64 = 50 + if urlArgs.Has("c") { + pageSize, _ = strconv.ParseUint(urlArgs.Get("c"), 10, 64) + } + if urlArgs.Has("json") && pageSize > 10000 { + pageSize = 10000 + } else if !urlArgs.Has("json") && pageSize > 1000 { + pageSize = 1000 + } + + var filterPubKey string + var filterIndex string + var filterExecutionAddr string + var filterStatus string + if urlArgs.Has("f") { + if urlArgs.Has("f.pubkey") { + filterPubKey = urlArgs.Get("f.pubkey") + } + if urlArgs.Has("f.index") { + filterIndex = urlArgs.Get("f.index") + } + if urlArgs.Has("f.execution_addr") { + filterExecutionAddr = urlArgs.Get("f.execution_addr") + } + if urlArgs.Has("f.status") { + filterStatus = strings.Join(urlArgs["f.status"], ",") + } + } + var sortOrder string + if urlArgs.Has("o") { + sortOrder = urlArgs.Get("o") + } + + var pageError error + pageError = services.GlobalCallRateLimiter.CheckCallLimit(r, 1) + if pageError == nil { + data.Data, pageError = getBuildersPageData(pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) + } + if pageError != nil { + handlePageError(w, r, pageError) + return + } + + if urlArgs.Has("json") { + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(data.Data) + if err != nil { + logrus.WithError(err).Error("error encoding builders data") + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + } + return + } + + w.Header().Set("Content-Type", "text/html") + if handleTemplateError(w, r, "builders.go", "Builders", "", pageTemplate.ExecuteTemplate(w, "layout", data)) != nil { + return // an error has occurred and was processed + } +} + +func getBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, filterPubKey string, filterIndex string, filterExecutionAddr string, filterStatus string) (*models.BuildersPageData, error) { + pageData := &models.BuildersPageData{} + pageCacheKey := fmt.Sprintf("builders:%v:%v:%v:%v:%v:%v:%v", pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) + pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { + pageData, cacheTimeout := buildBuildersPageData(pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) + pageCall.CacheTimeout = cacheTimeout + return pageData + }) + if pageErr == nil && pageRes != nil { + resData, resOk := pageRes.(*models.BuildersPageData) + if !resOk { + return nil, ErrInvalidPageModel + } + pageData = resData + } + return pageData, pageErr +} + +func buildBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, filterPubKey string, filterIndex string, filterExecutionAddr string, filterStatus string) (*models.BuildersPageData, time.Duration) { + logrus.Debugf("builders page called: %v:%v:%v:%v:%v:%v:%v", pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) + pageData := &models.BuildersPageData{} + cacheTime := 10 * time.Minute + + chainState := services.GlobalBeaconService.GetChainState() + + builderFilter := dbtypes.BuilderFilter{ + Limit: pageSize, + Offset: (pageNumber - 1) * pageSize, + } + + filterArgs := url.Values{} + if filterPubKey != "" || filterIndex != "" || filterExecutionAddr != "" || filterStatus != "" { + if filterPubKey != "" { + pageData.FilterPubKey = filterPubKey + filterArgs.Add("f.pubkey", filterPubKey) + filterPubKeyVal, _ := hex.DecodeString(strings.Replace(filterPubKey, "0x", "", -1)) + builderFilter.PubKey = filterPubKeyVal + } + if filterIndex != "" { + pageData.FilterIndex = filterIndex + filterArgs.Add("f.index", filterIndex) + filterIndexVal, _ := strconv.ParseUint(filterIndex, 10, 64) + builderFilter.MinIndex = &filterIndexVal + builderFilter.MaxIndex = &filterIndexVal + } + if filterExecutionAddr != "" { + pageData.FilterExecutionAddr = filterExecutionAddr + filterArgs.Add("f.execution_addr", filterExecutionAddr) + filterExecutionAddrVal, _ := hex.DecodeString(strings.Replace(filterExecutionAddr, "0x", "", -1)) + builderFilter.ExecutionAddress = filterExecutionAddrVal + } + if filterStatus != "" { + pageData.FilterStatus = filterStatus + filterArgs.Add("f.status", filterStatus) + filterStatusVal := strings.Split(filterStatus, ",") + builderFilter.Status = make([]dbtypes.BuilderStatus, 0, len(filterStatusVal)) + for _, status := range filterStatusVal { + switch status { + case "active": + builderFilter.Status = append(builderFilter.Status, dbtypes.BuilderStatusActiveFilter) + case "exited": + builderFilter.Status = append(builderFilter.Status, dbtypes.BuilderStatusExitedFilter) + case "superseded": + builderFilter.Status = append(builderFilter.Status, dbtypes.BuilderStatusSupersededFilter) + } + } + } + } + + // apply sort order + switch sortOrder { + case "index-d": + builderFilter.OrderBy = dbtypes.BuilderOrderIndexDesc + case "pubkey": + builderFilter.OrderBy = dbtypes.BuilderOrderPubKeyAsc + case "pubkey-d": + builderFilter.OrderBy = dbtypes.BuilderOrderPubKeyDesc + case "balance": + builderFilter.OrderBy = dbtypes.BuilderOrderBalanceAsc + case "balance-d": + builderFilter.OrderBy = dbtypes.BuilderOrderBalanceDesc + case "deposit": + builderFilter.OrderBy = dbtypes.BuilderOrderDepositEpochAsc + case "deposit-d": + builderFilter.OrderBy = dbtypes.BuilderOrderDepositEpochDesc + case "withdrawable": + builderFilter.OrderBy = dbtypes.BuilderOrderWithdrawableEpochAsc + case "withdrawable-d": + builderFilter.OrderBy = dbtypes.BuilderOrderWithdrawableEpochDesc + default: + builderFilter.OrderBy = dbtypes.BuilderOrderIndexAsc + pageData.IsDefaultSorting = true + sortOrder = "index" + } + pageData.Sorting = sortOrder + + // get latest builder set + builderSetRsp, builderSetLen := services.GlobalBeaconService.GetFilteredBuilderSet(&builderFilter, true) + if len(builderSetRsp) == 0 { + cacheTime = 5 * time.Minute + } + + currentEpoch := chainState.CurrentEpoch() + + // get status options + pageData.FilterStatusOpts = []models.BuildersPageDataStatusOption{ + {Status: "active", Count: 0}, + {Status: "exited", Count: 0}, + {Status: "superseded", Count: 0}, + } + + totalPages := builderSetLen / pageSize + if (builderSetLen % pageSize) > 0 { + totalPages++ + } + if pageNumber == 0 { + pageData.IsDefaultPage = true + } else if pageNumber >= totalPages { + if totalPages == 0 { + pageNumber = 0 + } else { + pageNumber = totalPages + } + } + + pageData.PageSize = pageSize + pageData.TotalPages = totalPages + pageData.CurrentPageIndex = pageNumber + if pageNumber > 1 { + pageData.PrevPageIndex = pageNumber - 1 + } + if pageNumber < totalPages { + pageData.NextPageIndex = pageNumber + 1 + } + if totalPages > 1 { + pageData.LastPageIndex = totalPages + } + + // get builders + pageData.Builders = make([]*models.BuildersPageDataBuilder, 0, len(builderSetRsp)) + + for _, builder := range builderSetRsp { + if builder.Builder == nil { + continue + } + + builderData := &models.BuildersPageDataBuilder{ + Index: uint64(builder.Index), + PublicKey: builder.Builder.PublicKey[:], + ExecutionAddress: builder.Builder.ExecutionAddress[:], + Balance: uint64(builder.Builder.Balance), + } + + // Determine state + if builder.Superseded { + builderData.State = "Superseded" + } else if builder.Builder.WithdrawableEpoch <= currentEpoch { + builderData.State = "Exited" + } else { + builderData.State = "Active" + } + + // Deposit epoch + if builder.Builder.DepositEpoch < 18446744073709551615 { + builderData.ShowDeposit = true + builderData.DepositEpoch = uint64(builder.Builder.DepositEpoch) + builderData.DepositTs = chainState.EpochToTime(builder.Builder.DepositEpoch) + } + + // Withdrawable epoch + if builder.Builder.WithdrawableEpoch < 18446744073709551615 { + builderData.ShowWithdrawable = true + builderData.WithdrawableEpoch = uint64(builder.Builder.WithdrawableEpoch) + builderData.WithdrawableTs = chainState.EpochToTime(builder.Builder.WithdrawableEpoch) + } + + pageData.Builders = append(pageData.Builders, builderData) + } + pageData.BuilderCount = builderSetLen + pageData.FirstBuilder = pageNumber * pageSize + pageData.LastBuilder = pageData.FirstBuilder + uint64(len(pageData.Builders)) + + // Populate UrlParams for page jump functionality + pageData.UrlParams = make(map[string]string) + for key, values := range filterArgs { + if len(values) > 0 { + pageData.UrlParams[key] = values[0] + } + } + pageData.UrlParams["c"] = fmt.Sprintf("%v", pageData.PageSize) + + pageData.FilteredPageLink = fmt.Sprintf("/builders?f&%v&c=%v", filterArgs.Encode(), pageData.PageSize) + + // Sort status options alphabetically + sort.Slice(pageData.FilterStatusOpts, func(a, b int) bool { + return strings.Compare(pageData.FilterStatusOpts[a].Status, pageData.FilterStatusOpts[b].Status) < 0 + }) + + return pageData, cacheTime +} diff --git a/indexer/beacon/buildercache.go b/indexer/beacon/buildercache.go new file mode 100644 index 000000000..41f9c12a1 --- /dev/null +++ b/indexer/beacon/buildercache.go @@ -0,0 +1,751 @@ +package beacon + +import ( + "bytes" + "fmt" + "hash/crc64" + "math" + "runtime/debug" + "sync" + "time" + + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/jmoiron/sqlx" + + "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/dora/dbtypes" +) + +// BuilderIndexFlag separates builder indices from validator indices in the pubkey cache +const BuilderIndexFlag = uint64(1 << 40) + +// Builder status flag constants representing different builder states +const ( + BuilderStatusExited uint16 = 1 << iota // Builder has exited (withdrawable_epoch reached) + BuilderStatusSuperseded // Builder index was reused, this pubkey is no longer active +) + +// builderCache manages the in-memory cache of builder states and handles updates +type builderCache struct { + indexer *Indexer + builderSetCache []*builderEntry + cacheMutex sync.RWMutex + triggerDbUpdate chan bool +} + +// builderEntry represents a single builder's state in the cache +type builderEntry struct { + builderDiffs []*builderDiff + finalChecksum uint64 + finalBuilder *gloas.Builder + activeData *BuilderData + statusFlags uint16 +} + +// BuilderData contains the essential builder state information for active builders. +// Only WithdrawableEpoch can change during a builder's lifetime; all other fields are static. +type BuilderData struct { + WithdrawableEpoch phase0.Epoch +} + +// builderDiff represents an updated builder entry in the builder set cache. +type builderDiff struct { + epoch phase0.Epoch + dependentRoot phase0.Root + builder *gloas.Builder +} + +// newBuilderCache initializes a new builder cache instance and starts the persist loop +func newBuilderCache(indexer *Indexer) *builderCache { + cache := &builderCache{ + indexer: indexer, + triggerDbUpdate: make(chan bool, 1), + } + + go cache.runPersistLoop() + + return cache +} + +// updateBuilderSet processes builder set updates and maintains the cache state +func (cache *builderCache) updateBuilderSet(slot phase0.Slot, dependentRoot phase0.Root, builders []*gloas.Builder) { + chainState := cache.indexer.consensusPool.GetChainState() + epoch := chainState.EpochOfSlot(slot) + currentEpoch := chainState.CurrentEpoch() + finalizedEpoch, finalizedRoot := chainState.GetFinalizedCheckpoint() + cutOffEpoch := phase0.Epoch(0) + if currentEpoch > phase0.Epoch(cache.indexer.inMemoryEpochs) { + cutOffEpoch = currentEpoch - phase0.Epoch(cache.indexer.inMemoryEpochs) + } + if cutOffEpoch > finalizedEpoch { + cutOffEpoch = finalizedEpoch + } + + if epoch < cutOffEpoch { + cache.indexer.logger.Infof("ignoring old builder set update for epoch %d", epoch) + return + } + + isFinalizedBuilderSet := false + if slot == 0 { + isFinalizedBuilderSet = true // genesis + } else if epoch <= finalizedEpoch { + finalizedBlock := cache.indexer.blockCache.getBlockByRoot(finalizedRoot) + if finalizedBlock != nil { + finalizedDependentBlock := cache.indexer.blockCache.getDependentBlock(chainState, finalizedBlock, nil) + if finalizedDependentBlock != nil && bytes.Equal(finalizedDependentBlock.Root[:], dependentRoot[:]) { + isFinalizedBuilderSet = true + } + } + } + + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + + t1 := time.Now() + + if len(cache.builderSetCache) < len(builders) { + if len(builders) > cap(cache.builderSetCache) { + newCache := make([]*builderEntry, len(builders), len(builders)+1000) + copy(newCache, cache.builderSetCache) + cache.builderSetCache = newCache + } else { + cache.builderSetCache = cache.builderSetCache[:len(builders)] + } + } + + isParentMap := map[phase0.Root]bool{} + isAheadMap := map[phase0.Root]bool{} + updatedCount := uint64(0) + + for i := range builders { + var parentChecksum uint64 + var parentBuilder *gloas.Builder + parentEpoch := phase0.Epoch(0) + + aheadDiffIdx := 0 + foundAhead := false + aheadEpoch := phase0.Epoch(math.MaxInt64) + + cachedBuilder := cache.builderSetCache[i] + if cachedBuilder == nil { + cachedBuilder = &builderEntry{} + cache.builderSetCache[i] = cachedBuilder + + cache.indexer.pubkeyCache.Add(builders[i].PublicKey, phase0.ValidatorIndex(uint64(i)|BuilderIndexFlag)) + } else { + parentBuilder = cachedBuilder.finalBuilder + parentChecksum = cachedBuilder.finalChecksum + } + + deleteKeys := []int{} + + if !isFinalizedBuilderSet { + for diffkey, diff := range cachedBuilder.builderDiffs { + if diff.epoch < cutOffEpoch { + deleteKeys = append(deleteKeys, diffkey) + continue + } + + if diff.epoch < epoch { + isParent, checkedParent := isParentMap[diff.dependentRoot] + if !checkedParent { + isParent = cache.indexer.blockCache.isCanonicalBlock(diff.dependentRoot, dependentRoot) + isParentMap[diff.dependentRoot] = isParent + } + + if isParent && diff.epoch > parentEpoch { + parentBuilder = diff.builder + parentEpoch = diff.epoch + } + } + + if diff.epoch > epoch { + isAhead, checkedAhead := isAheadMap[diff.dependentRoot] + if !checkedAhead { + isAhead = cache.indexer.blockCache.isCanonicalBlock(dependentRoot, diff.dependentRoot) + isAheadMap[diff.dependentRoot] = isAhead + } + + if isAhead && diff.epoch < aheadEpoch { + aheadDiffIdx = diffkey + aheadEpoch = diff.epoch + foundAhead = true + } + } + } + + if parentBuilder != nil { + parentChecksum = calculateBuilderChecksum(parentBuilder) + } + } + + checksum := calculateBuilderChecksum(builders[i]) + if checksum == parentChecksum { + continue + } + + if isFinalizedBuilderSet { + cachedBuilder.finalBuilder = builders[i] + cachedBuilder.finalChecksum = checksum + cachedBuilder.statusFlags = GetBuilderStatusFlags(builders[i]) + updatedCount++ + + activeData := &BuilderData{ + WithdrawableEpoch: builders[i].WithdrawableEpoch, + } + if cache.isActiveBuilder(activeData) { + cachedBuilder.activeData = activeData + } + } + + if foundAhead && cache.checkBuilderEqual(cachedBuilder.builderDiffs[aheadDiffIdx].builder, builders[i]) { + if isFinalizedBuilderSet { + deleteKeys = append(deleteKeys, aheadDiffIdx) + } else { + diff := cachedBuilder.builderDiffs[aheadDiffIdx] + diff.epoch = epoch + diff.dependentRoot = dependentRoot + cachedBuilder.builderDiffs[aheadDiffIdx] = diff + } + } else if isFinalizedBuilderSet { + } else if len(deleteKeys) == 0 { + cachedBuilder.builderDiffs = append(cachedBuilder.builderDiffs, &builderDiff{ + epoch: epoch, + dependentRoot: dependentRoot, + builder: builders[i], + }) + } else { + cachedBuilder.builderDiffs[deleteKeys[0]] = &builderDiff{ + epoch: epoch, + dependentRoot: dependentRoot, + builder: builders[i], + } + deleteKeys = deleteKeys[1:] + } + + if len(deleteKeys) > 0 { + lastIdx := len(cachedBuilder.builderDiffs) - 1 + delLen := len(deleteKeys) + for delIdx := 0; delIdx < delLen; delIdx++ { + for delLen > 0 && deleteKeys[delLen-1] == lastIdx { + lastIdx-- + delLen-- + } + if delLen == 0 { + break + } + cachedBuilder.builderDiffs[deleteKeys[delIdx]] = cachedBuilder.builderDiffs[lastIdx] + lastIdx-- + } + + cachedBuilder.builderDiffs = cachedBuilder.builderDiffs[:lastIdx+1] + } + } + + if updatedCount > 0 { + select { + case cache.triggerDbUpdate <- true: + default: + } + } + + isFinalizedStr := "" + if isFinalizedBuilderSet { + isFinalizedStr = "finalized " + } + cache.indexer.logger.Infof("processed %vbuilder set update for epoch %d in %v", isFinalizedStr, epoch, time.Since(t1)) +} + +// checkBuilderEqual compares two builder states for equality +func (cache *builderCache) checkBuilderEqual(builder1 *gloas.Builder, builder2 *gloas.Builder) bool { + if builder1 == nil && builder2 == nil { + return true + } + if builder1 == nil || builder2 == nil { + return false + } + return bytes.Equal(builder1.PublicKey[:], builder2.PublicKey[:]) && + builder1.Version == builder2.Version && + bytes.Equal(builder1.ExecutionAddress[:], builder2.ExecutionAddress[:]) && + builder1.DepositEpoch == builder2.DepositEpoch && + builder1.WithdrawableEpoch == builder2.WithdrawableEpoch +} + +// GetBuilderStatusFlags calculates the status flags for a builder +func GetBuilderStatusFlags(builder *gloas.Builder) uint16 { + flags := uint16(0) + if builder.WithdrawableEpoch != FarFutureEpoch { + flags |= BuilderStatusExited + } + return flags +} + +// getBuilderSetSize returns the current number of builders in the builder set +func (cache *builderCache) getBuilderSetSize() uint64 { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + return uint64(len(cache.builderSetCache)) +} + +// getBuilderFlags returns the status flags for a specific builder +func (cache *builderCache) getBuilderFlags(builderIndex gloas.BuilderIndex) uint16 { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + if uint64(builderIndex) >= uint64(len(cache.builderSetCache)) || cache.builderSetCache[builderIndex] == nil { + return 0 + } + + return cache.builderSetCache[builderIndex].statusFlags +} + +// setFinalizedEpoch updates the builder cache when a new epoch is finalized +func (cache *builderCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDependentRoot phase0.Root) { + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + + updatedCount := uint64(0) + + for _, cachedBuilder := range cache.builderSetCache { + if cachedBuilder == nil { + continue + } + + // Find the finalized builder state + for _, diff := range cachedBuilder.builderDiffs { + if diff.dependentRoot == nextEpochDependentRoot { + cachedBuilder.finalBuilder = diff.builder + cachedBuilder.finalChecksum = calculateBuilderChecksum(diff.builder) + cachedBuilder.statusFlags = GetBuilderStatusFlags(diff.builder) + updatedCount++ + + cachedBuilder.activeData = &BuilderData{ + WithdrawableEpoch: diff.builder.WithdrawableEpoch, + } + break + } + } + + // Clean up old diffs + newDiffs := make([]*builderDiff, 0) + for _, diff := range cachedBuilder.builderDiffs { + if diff.epoch > epoch { + newDiffs = append(newDiffs, diff) + } + } + cachedBuilder.builderDiffs = newDiffs + + // Clear old active data + if cachedBuilder.activeData != nil { + if !cache.isActiveBuilder(cachedBuilder.activeData) { + cachedBuilder.activeData = nil + } + } + } + + if updatedCount > 0 { + select { + case cache.triggerDbUpdate <- true: + default: + } + } +} + +// BuilderSetStreamer is a callback for streaming builder data +type BuilderSetStreamer func(index gloas.BuilderIndex, flags uint16, activeData *BuilderData, builder *gloas.Builder) error + +// streamBuilderSetForRoot streams the builder set for a given blockRoot +func (cache *builderCache) streamBuilderSetForRoot(blockRoot phase0.Root, onlyActive bool, epoch *phase0.Epoch, cb BuilderSetStreamer) error { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + isParentMap := map[phase0.Root]bool{} + isAheadMap := map[phase0.Root]bool{} + + for index, cachedBuilder := range cache.builderSetCache { + if cachedBuilder == nil { + continue + } + + latestBuilder := cachedBuilder.finalBuilder + builderData := cachedBuilder.activeData + builderEpoch := phase0.Epoch(0) + + var aheadBuilder *gloas.Builder + aheadEpoch := phase0.Epoch(math.MaxInt64) + + for _, diff := range cachedBuilder.builderDiffs { + isParent, checkedParent := isParentMap[diff.dependentRoot] + if !checkedParent { + isParent = cache.indexer.blockCache.isCanonicalBlock(diff.dependentRoot, blockRoot) + isParentMap[diff.dependentRoot] = isParent + } + + if isParent && diff.epoch >= builderEpoch { + builderData = &BuilderData{ + WithdrawableEpoch: diff.builder.WithdrawableEpoch, + } + builderEpoch = diff.epoch + latestBuilder = diff.builder + } + + if !isParent && builderData == nil { + isAhead, checkedAhead := isAheadMap[diff.dependentRoot] + if !checkedAhead { + isAhead = cache.indexer.blockCache.isCanonicalBlock(blockRoot, diff.dependentRoot) + isAheadMap[diff.dependentRoot] = isAhead + } + + if isAhead && diff.epoch < aheadEpoch { + aheadBuilder = diff.builder + aheadEpoch = diff.epoch + } + } + } + + if builderData == nil && aheadBuilder != nil { + builderData = &BuilderData{ + WithdrawableEpoch: aheadBuilder.WithdrawableEpoch, + } + latestBuilder = aheadBuilder + } + + if onlyActive && (builderData == nil || (epoch != nil && builderData.WithdrawableEpoch <= *epoch)) { + continue + } + + builderFlags := cachedBuilder.statusFlags + if latestBuilder != nil { + builderFlags = GetBuilderStatusFlags(latestBuilder) + } + + err := cb(gloas.BuilderIndex(index), builderFlags, builderData, latestBuilder) + if err != nil { + return err + } + } + + return nil +} + +// UnwrapDbBuilder converts a dbtypes.Builder to a gloas.Builder +func UnwrapDbBuilder(dbBuilder *dbtypes.Builder) *gloas.Builder { + builder := &gloas.Builder{ + Version: dbBuilder.Version, + Balance: 0, // Balance not persisted + DepositEpoch: phase0.Epoch(db.ConvertInt64ToUint64(dbBuilder.DepositEpoch)), + WithdrawableEpoch: phase0.Epoch(db.ConvertInt64ToUint64(dbBuilder.WithdrawableEpoch)), + } + copy(builder.PublicKey[:], dbBuilder.Pubkey) + copy(builder.ExecutionAddress[:], dbBuilder.ExecutionAddress) + return builder +} + +// isActiveBuilder determines if a builder is currently active +func (cache *builderCache) isActiveBuilder(builder *BuilderData) bool { + currentEpoch := cache.indexer.consensusPool.GetChainState().CurrentEpoch() + cutOffEpoch := phase0.Epoch(0) + if currentEpoch > 10 { + cutOffEpoch = currentEpoch - 10 + } + + return builder.WithdrawableEpoch > cutOffEpoch +} + +// getBuilderByIndex returns the builder by index for a given forkId +func (cache *builderCache) getBuilderByIndex(index gloas.BuilderIndex, overrideForkId *ForkKey) *gloas.Builder { + canonicalHead := cache.indexer.GetCanonicalHead(overrideForkId) + if canonicalHead == nil { + return nil + } + + return cache.getBuilderByIndexAndRoot(index, canonicalHead.Root) +} + +// getBuilderByIndexAndRoot returns the builder by index for a given blockRoot +func (cache *builderCache) getBuilderByIndexAndRoot(index gloas.BuilderIndex, blockRoot phase0.Root) *gloas.Builder { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + if uint64(index) >= uint64(len(cache.builderSetCache)) { + return nil + } + + cachedBuilder := cache.builderSetCache[index] + if cachedBuilder == nil { + return nil + } + + builder := cachedBuilder.finalBuilder + builderEpoch := phase0.Epoch(0) + + // Find the latest valid diff + for _, diff := range cachedBuilder.builderDiffs { + if cache.indexer.blockCache.isCanonicalBlock(diff.dependentRoot, blockRoot) && diff.epoch >= builderEpoch { + builder = diff.builder + builderEpoch = diff.epoch + } + } + + // Fallback to db if builder is not found in cache + if builder == nil { + if dbBuilder := db.GetActiveBuilderByIndex(uint64(index)); dbBuilder != nil { + builder = UnwrapDbBuilder(dbBuilder) + } + } else { + // Return a copy + builder = &gloas.Builder{ + PublicKey: builder.PublicKey, + Version: builder.Version, + ExecutionAddress: builder.ExecutionAddress, + Balance: builder.Balance, + DepositEpoch: builder.DepositEpoch, + WithdrawableEpoch: builder.WithdrawableEpoch, + } + } + + return builder +} + +// calculateBuilderChecksum generates a CRC64 checksum of all builder fields (except balance) +func calculateBuilderChecksum(b *gloas.Builder) uint64 { + if b == nil { + return 0 + } + + data := make([]byte, 0, 80) + data = append(data, b.PublicKey[:]...) + data = append(data, b.Version) + data = append(data, b.ExecutionAddress[:]...) + data = append(data, uint64ToBytes(uint64(b.DepositEpoch))...) + data = append(data, uint64ToBytes(uint64(b.WithdrawableEpoch))...) + + return crc64.Checksum(data, crc64Table) +} + +// prepopulateFromDB pre-populates the builder set cache from the database +func (cache *builderCache) prepopulateFromDB() (uint64, error) { + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + + maxIndex, err := db.GetMaxBuilderIndex() + if err != nil { + return 0, fmt.Errorf("error getting max builder index: %w", err) + } + + if maxIndex == 0 { + return 0, nil + } + + cache.builderSetCache = make([]*builderEntry, maxIndex+1, maxIndex+1+1000) + + restoreCount := uint64(0) + + batchSize := uint64(10000) + for start := uint64(0); start <= maxIndex; start += batchSize { + end := min(start+batchSize, maxIndex) + + builders := db.GetBuilderRange(start, end) + for _, dbBuilder := range builders { + if dbBuilder.Superseded { + continue + } + + builder := UnwrapDbBuilder(dbBuilder) + builderEntry := &builderEntry{ + finalChecksum: calculateBuilderChecksum(builder), + } + builderData := &BuilderData{ + WithdrawableEpoch: phase0.Epoch(db.ConvertInt64ToUint64(dbBuilder.WithdrawableEpoch)), + } + if cache.isActiveBuilder(builderData) { + builderEntry.activeData = builderData + } + builderEntry.statusFlags = GetBuilderStatusFlags(builder) + + cache.builderSetCache[dbBuilder.BuilderIndex] = builderEntry + + cache.indexer.pubkeyCache.Add(builder.PublicKey, phase0.ValidatorIndex(dbBuilder.BuilderIndex|BuilderIndexFlag)) + + restoreCount++ + } + } + + return restoreCount, nil +} + +// runPersistLoop handles the background persistence of builder states to the database +func (cache *builderCache) runPersistLoop() { + defer func() { + if err := recover(); err != nil { + cache.indexer.logger.WithError(err.(error)).Errorf( + "uncaught panic in indexer.beacon.builderCache.runPersistLoop subroutine: %v, stack: %v", + err, string(debug.Stack())) + time.Sleep(10 * time.Second) + + go cache.runPersistLoop() + } + }() + + for range cache.triggerDbUpdate { + time.Sleep(2 * time.Second) + err := db.RunDBTransaction(func(tx *sqlx.Tx) error { + hasMore, err := cache.persistBuilders(tx) + if hasMore { + select { + case cache.triggerDbUpdate <- true: + default: + } + } + return err + }) + if err != nil { + cache.indexer.logger.WithError(err).Errorf("error persisting builders") + } + } +} + +// persistBuilders writes a batch of builder states to the database +func (cache *builderCache) persistBuilders(tx *sqlx.Tx) (bool, error) { + cache.cacheMutex.RLock() + defer cache.cacheMutex.RUnlock() + + const batchSize = 1000 + const maxPerRun = 10000 + + batch := make([]*dbtypes.Builder, 0, batchSize) + batchIndices := make([]uint64, 0, batchSize) + supersededPubkeys := make([][]byte, 0) + persisted := 0 + firstIndex := uint64(0) + lastIndex := uint64(0) + hasMore := false + + for index, entry := range cache.builderSetCache { + if entry == nil || entry.finalBuilder == nil { + continue + } + + if persisted == 0 && len(batch) == 0 { + firstIndex = uint64(index) + } + lastIndex = uint64(index) + + dbBuilder := &dbtypes.Builder{ + Pubkey: entry.finalBuilder.PublicKey[:], + BuilderIndex: uint64(index), + Version: entry.finalBuilder.Version, + ExecutionAddress: entry.finalBuilder.ExecutionAddress[:], + DepositEpoch: db.ConvertUint64ToInt64(uint64(entry.finalBuilder.DepositEpoch)), + WithdrawableEpoch: db.ConvertUint64ToInt64(uint64(entry.finalBuilder.WithdrawableEpoch)), + Superseded: false, + } + + batch = append(batch, dbBuilder) + batchIndices = append(batchIndices, uint64(index)) + + if len(batch) >= batchSize { + superseded, err := cache.persistBuilderBatch(tx, batch, batchIndices) + if err != nil { + return false, err + } + supersededPubkeys = append(supersededPubkeys, superseded...) + + // Clear finalBuilder for persisted entries + for _, idx := range batchIndices { + if cache.builderSetCache[idx] != nil { + cache.builderSetCache[idx].finalBuilder = nil + } + } + + batch = batch[:0] + batchIndices = batchIndices[:0] + persisted += batchSize + + if persisted >= maxPerRun { + hasMore = true + break + } + } + } + + // Persist remaining batch + if len(batch) > 0 { + superseded, err := cache.persistBuilderBatch(tx, batch, batchIndices) + if err != nil { + return false, err + } + supersededPubkeys = append(supersededPubkeys, superseded...) + + // Clear finalBuilder for persisted entries + for _, idx := range batchIndices { + if cache.builderSetCache[idx] != nil { + cache.builderSetCache[idx].finalBuilder = nil + } + } + + persisted += len(batch) + } + + // Batch mark superseded builders + if len(supersededPubkeys) > 0 { + err := db.SetBuildersSuperseded(supersededPubkeys, tx) + if err != nil { + return false, fmt.Errorf("error marking builders as superseded: %w", err) + } + } + + if persisted > 0 || len(supersededPubkeys) > 0 { + cache.indexer.logger.Infof("persisted %d builders to db [%d-%d], marked %d as superseded", + persisted, firstIndex, lastIndex, len(supersededPubkeys)) + } + + return hasMore, nil +} + +// persistBuilderBatch persists a batch of builders and returns pubkeys that were superseded +func (cache *builderCache) persistBuilderBatch(tx *sqlx.Tx, batch []*dbtypes.Builder, indices []uint64) ([][]byte, error) { + if len(batch) == 0 { + return nil, nil + } + + // Get range for this batch + minIndex := indices[0] + maxIndex := indices[0] + for _, idx := range indices[1:] { + if idx < minIndex { + minIndex = idx + } + if idx > maxIndex { + maxIndex = idx + } + } + + // Fetch existing builders in this batch's range + existingBuilders := db.GetBuilderRange(minIndex, maxIndex) + existingByIndex := make(map[uint64]*dbtypes.Builder, len(existingBuilders)) + for _, b := range existingBuilders { + existingByIndex[b.BuilderIndex] = b + } + + // Find superseded pubkeys + supersededPubkeys := make([][]byte, 0) + for i, dbBuilder := range batch { + if existing, ok := existingByIndex[indices[i]]; ok { + if !bytes.Equal(existing.Pubkey, dbBuilder.Pubkey) { + supersededPubkeys = append(supersededPubkeys, existing.Pubkey) + } + } + } + + // Insert batch + err := db.InsertBuilderBatch(batch, tx) + if err != nil { + return nil, fmt.Errorf("error persisting builder batch: %w", err) + } + + return supersededPubkeys, nil +} diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 4d696f480..db65ec134 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -25,6 +25,7 @@ type epochState struct { stateSlot phase0.Slot validatorBalances []phase0.Gwei + builderBalances []phase0.Gwei randaoMixes []phase0.Root depositIndex uint64 syncCommittee []phase0.ValidatorIndex @@ -161,6 +162,20 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epoch cache.indexer.validatorCache.updateValidatorSet(slot, s.slotRoot, validatorList) } + // Process builder set for Gloas + if state.Version >= spec.DataVersionGloas && state.Gloas != nil { + if cache != nil { + cache.indexer.builderCache.updateBuilderSet(slot, s.slotRoot, state.Gloas.Builders) + } + + // Extract builder balances + builderBalances := make([]phase0.Gwei, len(state.Gloas.Builders)) + for i, builder := range state.Gloas.Builders { + builderBalances[i] = builder.Balance + } + s.builderBalances = builderBalances + } + validatorPubkeyMap := make(map[phase0.BLSPubKey]phase0.ValidatorIndex) for i, v := range validatorList { validatorPubkeyMap[v.PublicKey] = phase0.ValidatorIndex(i) diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 22568c62b..1a3604531 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -542,6 +542,7 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R // update validator cache if len(canonicalBlocks) > 0 { indexer.validatorCache.setFinalizedEpoch(epoch, canonicalBlocks[len(canonicalBlocks)-1].Root) + indexer.builderCache.setFinalizedEpoch(epoch, canonicalBlocks[len(canonicalBlocks)-1].Root) } // clean fork cache diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 16f2aba02..89141c959 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -47,6 +47,7 @@ type Indexer struct { validatorCache *validatorCache validatorActivity *validatorActivityCache blockBidCache *blockBidCache + builderCache *builderCache // indexer state clients []*Client @@ -118,6 +119,7 @@ func NewIndexer(logger logrus.FieldLogger, consensusPool *consensus.Pool) *Index indexer.validatorCache = newValidatorCache(indexer) indexer.validatorActivity = newValidatorActivityCache(indexer) indexer.blockBidCache = newBlockBidCache(indexer) + indexer.builderCache = newBuilderCache(indexer) indexer.dbWriter = newDbWriter(indexer) badChainRoots := utils.Config.Indexer.BadChainRoots @@ -275,6 +277,14 @@ func (indexer *Indexer) StartIndexer() { indexer.logger.Infof("restored %v validators from DB (%.3f sec)", validatorCount, time.Since(t1).Seconds()) } + // restore finalized builder set from db + t1 = time.Now() + if builderCount, err := indexer.builderCache.prepopulateFromDB(); err != nil { + indexer.logger.WithError(err).Errorf("failed loading builder set") + } else if builderCount > 0 { + indexer.logger.Infof("restored %v builders from DB (%.3f sec)", builderCount, time.Since(t1).Seconds()) + } + // restore unfinalized epoch stats from db restoredEpochStats := 0 t1 = time.Now() diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 64ef00973..678664f1f 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -521,3 +521,53 @@ func (indexer *Indexer) GetBlockBids(parentBlockRoot phase0.Root) []*dbtypes.Blo // Fall back to database return db.GetBidsForBlockRoot(parentBlockRoot[:]) } + +// StreamActiveBuilderDataForRoot streams the available builder set data for a given blockRoot. +func (indexer *Indexer) StreamActiveBuilderDataForRoot(blockRoot phase0.Root, activeOnly bool, epoch *phase0.Epoch, cb BuilderSetStreamer) error { + return indexer.builderCache.streamBuilderSetForRoot(blockRoot, activeOnly, epoch, cb) +} + +// GetBuilderSetSize returns the size of the builder set cache. +func (indexer *Indexer) GetBuilderSetSize() uint64 { + return indexer.builderCache.getBuilderSetSize() +} + +// GetRecentBuilderBalances returns the most recent builder balances for the given fork. +func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phase0.Gwei { + chainState := indexer.consensusPool.GetChainState() + + canonicalHead := indexer.GetCanonicalHead(overrideForkId) + if canonicalHead == nil { + return nil + } + + headEpoch := chainState.EpochOfSlot(canonicalHead.Slot) + + var epochStats *EpochStats + for { + cEpoch := chainState.EpochOfSlot(canonicalHead.Slot) + if headEpoch-cEpoch > 2 { + return nil + } + + dependentBlock := indexer.blockCache.getDependentBlock(chainState, canonicalHead, nil) + if dependentBlock == nil { + return nil + } + canonicalHead = dependentBlock + + stats := indexer.epochCache.getEpochStats(cEpoch, dependentBlock.Root) + if cEpoch > 0 && (stats == nil || stats.dependentState == nil || stats.dependentState.loadingStatus != 2) { + continue // retry previous state + } + + epochStats = stats + break + } + + if epochStats == nil || epochStats.dependentState == nil { + return nil + } + + return epochStats.dependentState.builderBalances +} diff --git a/services/chainservice_builder.go b/services/chainservice_builder.go new file mode 100644 index 000000000..33819ca9f --- /dev/null +++ b/services/chainservice_builder.go @@ -0,0 +1,253 @@ +package services + +import ( + "bytes" + "slices" + "sort" + + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + + "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/dora/indexer/beacon" +) + +type BuilderWithIndex struct { + Index gloas.BuilderIndex + Builder *gloas.Builder + Superseded bool +} + +// GetFilteredBuilderSet returns builders matching the filter criteria +func (bs *ChainService) GetFilteredBuilderSet(filter *dbtypes.BuilderFilter, withBalance bool) ([]BuilderWithIndex, uint64) { + var overrideForkId *beacon.ForkKey + + canonicalHead := bs.beaconIndexer.GetCanonicalHead(overrideForkId) + if canonicalHead == nil { + return nil, 0 + } + + var balances []phase0.Gwei + if withBalance { + balances = bs.beaconIndexer.GetRecentBuilderBalances(overrideForkId) + } + currentEpoch := bs.consensusPool.GetChainState().CurrentEpoch() + + cachedResults := make([]BuilderWithIndex, 0, 1000) + cachedIndexes := map[uint64]bool{} + + // Get matching entries from cached builders + bs.beaconIndexer.StreamActiveBuilderDataForRoot(canonicalHead.Root, false, ¤tEpoch, func(index gloas.BuilderIndex, flags uint16, activeData *beacon.BuilderData, builder *gloas.Builder) error { + if builder == nil { + return nil + } + if filter.MinIndex != nil && uint64(index) < *filter.MinIndex { + return nil + } + if filter.MaxIndex != nil && uint64(index) > *filter.MaxIndex { + return nil + } + if len(filter.PubKey) > 0 { + pubkeylen := min(len(filter.PubKey), 48) + if !bytes.Equal(builder.PublicKey[:pubkeylen], filter.PubKey) { + return nil + } + } + if len(filter.ExecutionAddress) > 0 { + if !bytes.Equal(builder.ExecutionAddress[:], filter.ExecutionAddress) { + return nil + } + } + + if len(filter.Status) > 0 { + builderStatus := getBuilderStatus(builder, currentEpoch, false) + if !slices.Contains(filter.Status, builderStatus) { + return nil + } + } + + cachedResults = append(cachedResults, BuilderWithIndex{ + Index: index, + Builder: builder, + }) + cachedIndexes[uint64(index)] = true + + return nil + }) + + // Get matching entries from DB + dbIndexes, err := db.GetBuilderIndexesByFilter(*filter, uint64(currentEpoch)) + if err != nil { + bs.logger.Warnf("error getting builder indexes by filter: %v", err) + return nil, 0 + } + + // Sort results + var sortFn func(builderA, builderB BuilderWithIndex) bool + switch filter.OrderBy { + case dbtypes.BuilderOrderIndexAsc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Index < builderB.Index + } + case dbtypes.BuilderOrderIndexDesc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Index > builderB.Index + } + case dbtypes.BuilderOrderPubKeyAsc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return bytes.Compare(builderA.Builder.PublicKey[:], builderB.Builder.PublicKey[:]) < 0 + } + case dbtypes.BuilderOrderPubKeyDesc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return bytes.Compare(builderA.Builder.PublicKey[:], builderB.Builder.PublicKey[:]) > 0 + } + case dbtypes.BuilderOrderBalanceAsc: + if balances == nil { + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.Balance < builderB.Builder.Balance + } + } else { + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return balances[builderA.Index] < balances[builderB.Index] + } + sort.Slice(dbIndexes, func(i, j int) bool { + if dbIndexes[i] >= uint64(len(balances)) || dbIndexes[j] >= uint64(len(balances)) { + return dbIndexes[i] < dbIndexes[j] + } + return balances[dbIndexes[i]] < balances[dbIndexes[j]] + }) + } + case dbtypes.BuilderOrderBalanceDesc: + if balances == nil { + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.Balance > builderB.Builder.Balance + } + } else { + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return balances[builderA.Index] > balances[builderB.Index] + } + sort.Slice(dbIndexes, func(i, j int) bool { + if dbIndexes[i] >= uint64(len(balances)) || dbIndexes[j] >= uint64(len(balances)) { + return dbIndexes[i] > dbIndexes[j] + } + return balances[dbIndexes[i]] > balances[dbIndexes[j]] + }) + } + case dbtypes.BuilderOrderDepositEpochAsc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.DepositEpoch < builderB.Builder.DepositEpoch + } + case dbtypes.BuilderOrderDepositEpochDesc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.DepositEpoch > builderB.Builder.DepositEpoch + } + case dbtypes.BuilderOrderWithdrawableEpochAsc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.WithdrawableEpoch < builderB.Builder.WithdrawableEpoch + } + case dbtypes.BuilderOrderWithdrawableEpochDesc: + sortFn = func(builderA, builderB BuilderWithIndex) bool { + return builderA.Builder.WithdrawableEpoch > builderB.Builder.WithdrawableEpoch + } + } + + sort.Slice(cachedResults, func(i, j int) bool { + return sortFn(cachedResults[i], cachedResults[j]) + }) + + // Stream builder set from db and merge cached results + resCap := filter.Limit + if resCap == 0 { + resCap = uint64(len(cachedResults) + len(dbIndexes)) + } + result := make([]BuilderWithIndex, 0, resCap) + cachedIndex := 0 + matchingCount := uint64(0) + resultCount := uint64(0) + dbEntryCount := uint64(0) + + db.StreamBuildersByIndexes(dbIndexes, func(dbBuilder *dbtypes.Builder) bool { + dbEntryCount++ + builderWithIndex := BuilderWithIndex{ + Index: gloas.BuilderIndex(dbBuilder.BuilderIndex), + Builder: beacon.UnwrapDbBuilder(dbBuilder), + Superseded: dbBuilder.Superseded, + } + + for cachedIndex < len(cachedResults) && (cachedResults[cachedIndex].Index == builderWithIndex.Index || sortFn(cachedResults[cachedIndex], builderWithIndex)) { + if matchingCount >= filter.Offset { + resultBuilder := cachedResults[cachedIndex] + if balances != nil && uint64(resultBuilder.Index) < uint64(len(balances)) { + resultBuilder.Builder.Balance = balances[resultBuilder.Index] + } + result = append(result, resultBuilder) + resultCount++ + } + matchingCount++ + cachedIndex++ + + if filter.Limit > 0 && resultCount >= filter.Limit { + return false // stop streaming + } + } + + if cachedIndexes[dbBuilder.BuilderIndex] { + return true // skip this index, cache entry is newer + } + + if matchingCount >= filter.Offset { + if !builderWithIndex.Superseded && balances != nil && dbBuilder.BuilderIndex < uint64(len(balances)) { + builderWithIndex.Builder.Balance = balances[dbBuilder.BuilderIndex] + } + result = append(result, builderWithIndex) + resultCount++ + } + matchingCount++ + + if filter.Limit > 0 && resultCount >= filter.Limit { + return false // stop streaming + } + + return true // get more from db + }) + + for cachedIndex < len(cachedResults) && (filter.Limit == 0 || resultCount < filter.Limit) { + if matchingCount >= filter.Offset { + resultBuilder := cachedResults[cachedIndex] + if balances != nil && uint64(resultBuilder.Index) < uint64(len(balances)) { + resultBuilder.Builder.Balance = balances[resultBuilder.Index] + } + result = append(result, resultBuilder) + resultCount++ + } + matchingCount++ + cachedIndex++ + } + + // Add remaining cached results + matchingCount += uint64(len(cachedResults) - cachedIndex) + + // Add remaining db results + remainingDbCount := uint64(0) + for i := dbEntryCount; i < uint64(len(dbIndexes)); i++ { + if cachedIndexes[dbIndexes[i]] { + continue + } + remainingDbCount++ + } + matchingCount += remainingDbCount + + return result, matchingCount +} + +// getBuilderStatus determines the status of a builder +func getBuilderStatus(builder *gloas.Builder, currentEpoch phase0.Epoch, superseded bool) dbtypes.BuilderStatus { + if superseded { + return dbtypes.BuilderStatusSupersededFilter + } + if builder.WithdrawableEpoch <= currentEpoch { + return dbtypes.BuilderStatusExitedFilter + } + return dbtypes.BuilderStatusActiveFilter +} diff --git a/templates/builders/builders.html b/templates/builders/builders.html new file mode 100644 index 000000000..25869e1a2 --- /dev/null +++ b/templates/builders/builders.html @@ -0,0 +1,303 @@ +{{ define "page" }} +
    +
    +

    Builders Overview

    +     +
    + +
    +
    + + {{ if not .IsDefaultSorting }}{{ end }} +
    +
    + Builder Filters +
    +
    +
    +
    +
    +
    +
    + PubKey +
    +
    + +
    +
    +
    +
    + Index +
    +
    + +
    +
    +
    +
    + Execution Address +
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    + Status +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + +
    +
    +
    + + + + + + + + + + + + + {{ if gt .BuilderCount 0 }} + + {{ range $i, $builder := .Builders }} + + + + + + + + + + {{ end }} + + {{ else }} + + + + + + + + {{ end }} +
    + Index +
    + + +
    +     		+ Public Key +
    + + +
    +     		Execution Address + Balance +
    + + +
    +     		State + Deposit +
    + + +
    +     		+ Withdrawable +
    + + +
    +
    {{ formatAddCommas $builder.Index }}0x{{ printf "%x" $builder.PublicKey }}{{ ethAddressLink .ExecutionAddress }}{{ formatEthFromGwei $builder.Balance }}{{ $builder.State }} + {{- if $builder.ShowDeposit -}} + {{ formatRecentTimeShort $builder.DepositTs }} + (Epoch {{ formatAddCommas $builder.DepositEpoch }}) + {{- else -}} + - + {{- end -}} + + {{- if $builder.ShowWithdrawable -}} + {{ formatRecentTimeShort $builder.WithdrawableTs }} + (Epoch {{ formatAddCommas $builder.WithdrawableEpoch }}) + {{- else -}} + - + {{- end -}} +
    +
    + {{ template "professor_svg" }} +
    +
    +
    + {{ if gt .TotalPages 1 }} +
    +
    +
    +
    Showing builder {{ .FirstBuilder }} to {{ .LastBuilder }}
    +
    +
    +
    +
    +
    + + {{ range $key, $value := .UrlParams }} + {{ if ne $key "p" }} + + {{ end }} + {{ end }} + {{ if not .IsDefaultSorting }} + + {{ end }} +
    + + +
    +
    +
    +
    + +
    +
    +
    + {{ end }} +
    + +
    +
    +{{ end }} +{{ define "js" }} + + +{{ end }} +{{ define "css" }} + + +{{ end }} diff --git a/types/models/builders.go b/types/models/builders.go new file mode 100644 index 000000000..8f705a48a --- /dev/null +++ b/types/models/builders.go @@ -0,0 +1,50 @@ +package models + +import ( + "time" +) + +// BuildersPageData is a struct to hold info for the builders page +type BuildersPageData struct { + FilterPubKey string `json:"filter_pubkey"` + FilterIndex string `json:"filter_index"` + FilterExecutionAddr string `json:"filter_execution_addr"` + FilterStatus string `json:"filter_status"` + FilterStatusOpts []BuildersPageDataStatusOption `json:"filter_status_opts"` + + Builders []*BuildersPageDataBuilder `json:"builders"` + BuilderCount uint64 `json:"builder_count"` + FirstBuilder uint64 `json:"first_builder"` + LastBuilder uint64 `json:"last_builder"` + Sorting string `json:"sorting"` + IsDefaultSorting bool `json:"default_sorting"` + IsDefaultPage bool `json:"default_page"` + TotalPages uint64 `json:"total_pages"` + PageSize uint64 `json:"page_size"` + CurrentPageIndex uint64 `json:"page_index"` + PrevPageIndex uint64 `json:"prev_page_index"` + NextPageIndex uint64 `json:"next_page_index"` + LastPageIndex uint64 `json:"last_page_index"` + FilteredPageLink string `json:"filtered_page_link"` + + UrlParams map[string]string `json:"url_params"` +} + +type BuildersPageDataStatusOption struct { + Status string `json:"status"` + Count uint64 `json:"count"` +} + +type BuildersPageDataBuilder struct { + Index uint64 `json:"index"` + PublicKey []byte `json:"pubkey"` + ExecutionAddress []byte `json:"execution_address"` + Balance uint64 `json:"balance"` + State string `json:"state"` + ShowDeposit bool `json:"show_deposit"` + DepositTs time.Time `json:"deposit_ts"` + DepositEpoch uint64 `json:"deposit_epoch"` + ShowWithdrawable bool `json:"show_withdrawable"` + WithdrawableTs time.Time `json:"withdrawable_ts"` + WithdrawableEpoch uint64 `json:"withdrawable_epoch"` +} From 6b3b42ae1f94cabcbf68108e752fb9bb0bdb86fd Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Mon, 19 Jan 2026 11:50:10 +0100 Subject: [PATCH 27/99] fix: update based on latest gloas spec --- clients/consensus/chainspec.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 834de0ae7..27226da10 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -122,9 +122,7 @@ type ChainSpecConfig struct { BlobSchedule []BlobScheduleEntry `yaml:"BLOB_SCHEDULE" check-if-fork:"FuluForkEpoch"` // Gloas - PtcSize uint64 `yaml:"PTC_SIZE" check-if-fork:"GloasForkEpoch"` - MaxPayloadAttestations uint64 `yaml:"MAX_PAYLOAD_ATTESTATIONS" check-if-fork:"GloasForkEpoch"` - DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"GloasForkEpoch"` + MinBuilderWithdrawabilityDelay uint64 `yaml:"MIN_BUILDER_WITHDRAWABILITY_DELAY" check-if-fork:"GloasForkEpoch"` } type ChainSpecPreset struct { @@ -212,6 +210,13 @@ type ChainSpecPreset struct { FieldElementsPerExtBlob uint64 `yaml:"FIELD_ELEMENTS_PER_EXT_BLOB" check-if-fork:"FuluForkEpoch"` CellsPerExtBlob uint64 `yaml:"CELLS_PER_EXT_BLOB" check-if-fork:"FuluForkEpoch"` NumberOfColumns *uint64 `yaml:"NUMBER_OF_COLUMNS" check-if-fork:"FuluForkEpoch"` + + // Gloas + PtcSize uint64 `yaml:"PTC_SIZE" check-if-fork:"GloasForkEpoch"` + MaxPayloadAttestations uint64 `yaml:"MAX_PAYLOAD_ATTESTATIONS" check-if-fork:"GloasForkEpoch"` + BuilderRegistryLimit uint64 `yaml:"BUILDER_REGISTRY_LIMIT" check-if-fork:"GloasForkEpoch"` + BuilderPendingWithdrawalsLimit uint64 `yaml:"BUILDER_PENDING_WITHDRAWALS_LIMIT" check-if-fork:"GloasForkEpoch"` + MaxBuildersPerWithdrawalsSweep uint64 `yaml:"MAX_BUILDERS_PER_WITHDRAWALS_SWEEP" check-if-fork:"GloasForkEpoch"` } type ChainSpecDomainTypes struct { @@ -226,6 +231,11 @@ type ChainSpecDomainTypes struct { DomainSyncCommitteeSelectionProof phase0.DomainType `yaml:"DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF"` DomainContributionAndProof phase0.DomainType `yaml:"DOMAIN_CONTRIBUTION_AND_PROOF"` DomainBlsToExecutionChange phase0.DomainType `yaml:"DOMAIN_BLS_TO_EXECUTION_CHANGE"` + + // Gloas + DomainBeaconBuilder phase0.DomainType `yaml:"DOMAIN_BEACON_BUILDER" check-if-fork:"GloasForkEpoch"` + DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"GloasForkEpoch"` + DomainProposerPreferences phase0.DomainType `yaml:"DOMAIN_PROPOSER_PREFERENCES" check-if-fork:"GloasForkEpoch"` } type ChainSpec struct { From 9835bf76889135a726fd4d53f7a27b3ea3ff212f Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 20 Jan 2026 18:21:26 +0100 Subject: [PATCH 28/99] implement ePBS builder deposits & exits, fix gloas slot details --- db/deposits.go | 8 +- go.mod | 2 +- go.sum | 4 +- handlers/deposits.go | 18 +++- handlers/exits.go | 92 ++++++++++++------- handlers/included_deposits.go | 6 +- handlers/initiated_deposits.go | 3 + handlers/queued_deposits.go | 6 +- handlers/slot.go | 42 ++++++++- handlers/voluntary_exits.go | 84 +++++++++++------ indexer/beacon/indexer_getter.go | 6 ++ services/chainservice_builder.go | 9 ++ services/chainservice_deposits.go | 25 +++-- templates/deposits/deposits.html | 30 ++++-- templates/exits/exits.html | 6 +- .../included_deposits/included_deposits.html | 6 +- .../initiated_deposits.html | 6 +- .../queued_deposits/queued_deposits.html | 6 +- templates/slot/overview.html | 4 +- templates/slot/voluntary_exits.html | 8 +- .../voluntary_exits/voluntary_exits.html | 8 +- types/models/deposits.go | 3 + types/models/exits.go | 1 + types/models/included_deposits.go | 1 + types/models/initiated_deposits.go | 1 + types/models/queued_deposits.go | 1 + types/models/slot.go | 1 + types/models/voluntary_exits.go | 1 + utils/format.go | 23 +++++ utils/templateFucs.go | 2 + 30 files changed, 317 insertions(+), 96 deletions(-) diff --git a/db/deposits.go b/db/deposits.go index d70485b13..1c0dac3ff 100644 --- a/db/deposits.go +++ b/db/deposits.go @@ -138,14 +138,18 @@ func GetDepositsFiltered(offset uint64, limit uint32, canonicalForkIds []uint64, } if len(txFilter.WithdrawalAddress) > 0 { + // 0x01 = ETH1, 0x02 = compounding, 0x03 = builder deposit wdcreds1 := make([]byte, 32) wdcreds1[0] = 0x01 copy(wdcreds1[12:], txFilter.WithdrawalAddress) wdcreds2 := make([]byte, 32) wdcreds2[0] = 0x02 copy(wdcreds2[12:], txFilter.WithdrawalAddress) - args = append(args, wdcreds1, wdcreds2) - fmt.Fprintf(&sql, " %v (deposits.withdrawalcredentials = $%v OR deposits.withdrawalcredentials = $%v)", filterOp, len(args)-1, len(args)) + wdcreds3 := make([]byte, 32) + wdcreds3[0] = 0x03 + copy(wdcreds3[12:], txFilter.WithdrawalAddress) + args = append(args, wdcreds1, wdcreds2, wdcreds3) + fmt.Fprintf(&sql, " %v (deposits.withdrawalcredentials = $%v OR deposits.withdrawalcredentials = $%v OR deposits.withdrawalcredentials = $%v)", filterOp, len(args)-2, len(args)-1, len(args)) filterOp = "AND" } diff --git a/go.mod b/go.mod index 70dc2edc4..3d0ab5103 100644 --- a/go.mod +++ b/go.mod @@ -260,4 +260,4 @@ require ( modernc.org/sqlite v1.38.2 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d diff --git a/go.sum b/go.sum index be1ae350d..ec2ccc890 100644 --- a/go.sum +++ b/go.sum @@ -566,8 +566,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.0 h1:25Kb7CQKKkh8r8mlj/exNJY5vSoe3wGm/8QpsfayRtM= github.com/pk910/dynamic-ssz v1.2.0/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1 h1:Obn5KbqFo+T0Sr8fRGapWbKU5c61twez6ei5LQqA0gE= -github.com/pk910/go-eth2-client v0.0.0-20260109010443-3742e71092e1/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= +github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d h1:BRJcO8QD00qTwDX4doVLvDgpzkECmVjb3HQ3P2GJF5g= +github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/handlers/deposits.go b/handlers/deposits.go index 6816db24b..a1a247906 100644 --- a/handlers/deposits.go +++ b/handlers/deposits.go @@ -154,6 +154,9 @@ func buildDepositsPageData(firstEpoch uint64, pageSize uint64, tabView string) ( // load initiated deposits dbDepositTxs := db.GetDepositTxs(0, 20) for _, depositTx := range dbDepositTxs { + // Check if this is a builder deposit (0x03 withdrawal credentials) + isBuilder := len(depositTx.WithdrawalCredentials) > 0 && depositTx.WithdrawalCredentials[0] == 0x03 + depositTxData := &models.DepositsPageDataInitiatedDeposit{ Index: depositTx.Index, Address: depositTx.TxSender, @@ -165,6 +168,7 @@ func buildDepositsPageData(firstEpoch uint64, pageSize uint64, tabView string) ( Block: depositTx.BlockNumber, Orphaned: depositTx.Orphaned, Valid: depositTx.ValidSignature == 1 || depositTx.ValidSignature == 2, + IsBuilder: isBuilder, } validatorIndex, found := services.GlobalBeaconService.GetValidatorIndexByPubkey(phase0.BLSPubKey(depositTx.PublicKey)) @@ -215,15 +219,20 @@ func buildDepositsPageData(firstEpoch uint64, pageSize uint64, tabView string) ( dbDeposits, _ := services.GlobalBeaconService.GetDepositRequestsByFilter(depositFilter, 0, uint32(20)) for _, deposit := range dbDeposits { + // Check if this is a builder deposit (0x03 withdrawal credentials) + wdCreds := deposit.WithdrawalCredentials() + isBuilder := len(wdCreds) > 0 && wdCreds[0] == 0x03 + depositData := &models.DepositsPageDataIncludedDeposit{ PublicKey: deposit.PublicKey(), - Withdrawalcredentials: deposit.WithdrawalCredentials(), + Withdrawalcredentials: wdCreds, Amount: deposit.Amount(), Time: chainState.SlotToTime(phase0.Slot(deposit.Request.SlotNumber)), SlotNumber: deposit.Request.SlotNumber, SlotRoot: deposit.Request.SlotRoot, Orphaned: deposit.RequestOrphaned, DepositorAddress: deposit.SourceAddress(), + IsBuilder: isBuilder, } if deposit.IsQueued { @@ -313,12 +322,17 @@ func buildDepositsPageData(firstEpoch uint64, pageSize uint64, tabView string) ( } for _, queueEntry := range queuedDeposits.Queue[:limit] { + // Check if this is a builder deposit (0x03 withdrawal credentials) + wdCreds := queueEntry.PendingDeposit.WithdrawalCredentials[:] + isBuilder := len(wdCreds) > 0 && wdCreds[0] == 0x03 + depositData := &models.DepositsPageDataQueuedDeposit{ QueuePosition: queueEntry.QueuePos, EstimatedTime: chainState.EpochToTime(queueEntry.EpochEstimate), PublicKey: queueEntry.PendingDeposit.Pubkey[:], - Withdrawalcredentials: queueEntry.PendingDeposit.WithdrawalCredentials[:], + Withdrawalcredentials: wdCreds, Amount: uint64(queueEntry.PendingDeposit.Amount), + IsBuilder: isBuilder, } if validatorIdx, found := services.GlobalBeaconService.GetValidatorIndexByPubkey(phase0.BLSPubKey(depositData.PublicKey)); !found { diff --git a/handlers/exits.go b/handlers/exits.go index fbcbef603..5a0b5ab7a 100644 --- a/handlers/exits.go +++ b/handlers/exits.go @@ -9,6 +9,7 @@ import ( "time" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" @@ -158,43 +159,70 @@ func buildExitsPageData(firstEpoch uint64, pageSize uint64, tabView string) (*mo dbVoluntaryExits, _ := services.GlobalBeaconService.GetVoluntaryExitsByFilter(voluntaryExitFilter, 0, uint32(20)) for _, voluntaryExit := range dbVoluntaryExits { exitData := &models.ExitsPageDataRecentExit{ - SlotNumber: voluntaryExit.SlotNumber, - SlotRoot: voluntaryExit.SlotRoot, - Time: chainState.SlotToTime(phase0.Slot(voluntaryExit.SlotNumber)), - Orphaned: voluntaryExit.Orphaned, - ValidatorIndex: voluntaryExit.ValidatorIndex, - ValidatorName: services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex), + SlotNumber: voluntaryExit.SlotNumber, + SlotRoot: voluntaryExit.SlotRoot, + Time: chainState.SlotToTime(phase0.Slot(voluntaryExit.SlotNumber)), + Orphaned: voluntaryExit.Orphaned, } - validator := services.GlobalBeaconService.GetValidatorByIndex(phase0.ValidatorIndex(voluntaryExit.ValidatorIndex), false) - if validator == nil { - exitData.ValidatorStatus = "Unknown" - } else { - exitData.PublicKey = validator.Validator.PublicKey[:] - exitData.WithdrawalCreds = validator.Validator.WithdrawalCredentials - - if strings.HasPrefix(validator.Status.String(), "pending") { - exitData.ValidatorStatus = "Pending" - } else if validator.Status == v1.ValidatorStateActiveOngoing { - exitData.ValidatorStatus = "Active" - exitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateActiveExiting { - exitData.ValidatorStatus = "Exiting" - exitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateActiveSlashed { - exitData.ValidatorStatus = "Slashed" - exitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateExitedUnslashed { - exitData.ValidatorStatus = "Exited" - } else if validator.Status == v1.ValidatorStateExitedSlashed { - exitData.ValidatorStatus = "Slashed" + // Check if this is a builder exit (validator index has BuilderIndexFlag set) + if voluntaryExit.ValidatorIndex&services.BuilderIndexFlag != 0 { + builderIndex := voluntaryExit.ValidatorIndex &^ services.BuilderIndexFlag + exitData.IsBuilder = true + exitData.ValidatorIndex = builderIndex + + // Resolve builder name via validatornames service (with BuilderIndexFlag) + exitData.ValidatorName = services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex) + + builder := services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(builderIndex)) + if builder == nil { + exitData.ValidatorStatus = "Unknown" } else { - exitData.ValidatorStatus = validator.Status.String() + exitData.PublicKey = builder.PublicKey[:] + + // Determine builder status + currentEpoch := chainState.CurrentEpoch() + if builder.WithdrawableEpoch <= currentEpoch { + exitData.ValidatorStatus = "Exited" + } else { + exitData.ValidatorStatus = "Exiting" + } } + } else { + // Regular validator exit + exitData.ValidatorIndex = voluntaryExit.ValidatorIndex + exitData.ValidatorName = services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex) - if exitData.ShowUpcheck { - exitData.UpcheckActivity = uint8(services.GlobalBeaconService.GetValidatorLiveness(validator.Index, 3)) - exitData.UpcheckMaximum = uint8(3) + validator := services.GlobalBeaconService.GetValidatorByIndex(phase0.ValidatorIndex(voluntaryExit.ValidatorIndex), false) + if validator == nil { + exitData.ValidatorStatus = "Unknown" + } else { + exitData.PublicKey = validator.Validator.PublicKey[:] + exitData.WithdrawalCreds = validator.Validator.WithdrawalCredentials + + if strings.HasPrefix(validator.Status.String(), "pending") { + exitData.ValidatorStatus = "Pending" + } else if validator.Status == v1.ValidatorStateActiveOngoing { + exitData.ValidatorStatus = "Active" + exitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateActiveExiting { + exitData.ValidatorStatus = "Exiting" + exitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateActiveSlashed { + exitData.ValidatorStatus = "Slashed" + exitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateExitedUnslashed { + exitData.ValidatorStatus = "Exited" + } else if validator.Status == v1.ValidatorStateExitedSlashed { + exitData.ValidatorStatus = "Slashed" + } else { + exitData.ValidatorStatus = validator.Status.String() + } + + if exitData.ShowUpcheck { + exitData.UpcheckActivity = uint8(services.GlobalBeaconService.GetValidatorLiveness(validator.Index, 3)) + exitData.UpcheckMaximum = uint8(3) + } } } diff --git a/handlers/included_deposits.go b/handlers/included_deposits.go index 76c2effd7..88697d95a 100644 --- a/handlers/included_deposits.go +++ b/handlers/included_deposits.go @@ -194,15 +194,19 @@ func buildFilteredIncludedDepositsPageData(pageIdx uint64, pageSize uint64, minI chainState := services.GlobalBeaconService.GetChainState() for _, deposit := range dbDeposits { + wdCreds := deposit.WithdrawalCredentials() + isBuilder := len(wdCreds) > 0 && wdCreds[0] == 0x03 + depositData := &models.IncludedDepositsPageDataDeposit{ PublicKey: deposit.PublicKey(), - Withdrawalcredentials: deposit.WithdrawalCredentials(), + Withdrawalcredentials: wdCreds, Amount: deposit.Amount(), Time: chainState.SlotToTime(phase0.Slot(deposit.Request.SlotNumber)), SlotNumber: deposit.Request.SlotNumber, SlotRoot: deposit.Request.SlotRoot, Orphaned: deposit.RequestOrphaned, DepositorAddress: deposit.SourceAddress(), + IsBuilder: isBuilder, } if deposit.Request != nil { diff --git a/handlers/initiated_deposits.go b/handlers/initiated_deposits.go index ade3591a1..dca0c62ad 100644 --- a/handlers/initiated_deposits.go +++ b/handlers/initiated_deposits.go @@ -175,6 +175,8 @@ func buildFilteredInitiatedDepositsPageData(pageIdx uint64, pageSize uint64, add } for _, depositTx := range dbDepositTxs { + isBuilder := len(depositTx.WithdrawalCredentials) > 0 && depositTx.WithdrawalCredentials[0] == 0x03 + depositTxData := &models.InitiatedDepositsPageDataDeposit{ Index: depositTx.Index, Address: depositTx.TxSender, @@ -187,6 +189,7 @@ func buildFilteredInitiatedDepositsPageData(pageIdx uint64, pageSize uint64, add Orphaned: depositTx.Orphaned, Valid: depositTx.ValidSignature == 1 || depositTx.ValidSignature == 2, ValidatorStatus: "", + IsBuilder: isBuilder, } if validatorIdx, found := services.GlobalBeaconService.GetValidatorIndexByPubkey(phase0.BLSPubKey(depositTx.PublicKey)); !found { diff --git a/handlers/queued_deposits.go b/handlers/queued_deposits.go index b626acb97..408b1bedd 100644 --- a/handlers/queued_deposits.go +++ b/handlers/queued_deposits.go @@ -206,12 +206,16 @@ func buildQueuedDepositsPageData(pageIdx uint64, pageSize uint64, minIndex uint6 for i := start; i < end; i++ { queueEntry := filteredQueue[i] + wdCreds := queueEntry.PendingDeposit.WithdrawalCredentials[:] + isBuilder := len(wdCreds) > 0 && wdCreds[0] == 0x03 + depositData := &models.QueuedDepositsPageDataDeposit{ QueuePosition: queueEntry.QueuePos, EstimatedTime: chainState.EpochToTime(queueEntry.EpochEstimate), PublicKey: queueEntry.PendingDeposit.Pubkey[:], Amount: uint64(queueEntry.PendingDeposit.Amount), - Withdrawalcredentials: queueEntry.PendingDeposit.WithdrawalCredentials[:], + Withdrawalcredentials: wdCreds, + IsBuilder: isBuilder, } // Get validator status if exists diff --git a/handlers/slot.go b/handlers/slot.go index 6a9ca2efd..7347d864c 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -529,9 +529,16 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV pageData.VoluntaryExits = make([]*models.SlotPageVoluntaryExit, pageData.VoluntaryExitsCount) for i, exit := range voluntaryExits { + validatorIndex := uint64(exit.Message.ValidatorIndex) + isBuilder := validatorIndex&services.BuilderIndexFlag != 0 + displayIndex := validatorIndex + if isBuilder { + displayIndex = validatorIndex &^ services.BuilderIndexFlag + } pageData.VoluntaryExits[i] = &models.SlotPageVoluntaryExit{ - ValidatorIndex: uint64(exit.Message.ValidatorIndex), - ValidatorName: services.GlobalBeaconService.GetValidatorName(uint64(exit.Message.ValidatorIndex)), + ValidatorIndex: displayIndex, + ValidatorName: services.GlobalBeaconService.GetValidatorName(validatorIndex), + IsBuilder: isBuilder, Epoch: uint64(exit.Message.Epoch), Signature: exit.Signature[:], } @@ -647,7 +654,36 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV pageData.SyncAggParticipation = utils.SyncCommitteeParticipation(pageData.SyncAggregateBits, specs.SyncCommitteeSize) } - if executionPayload, _ := blockData.Block.ExecutionPayload(); executionPayload != nil { + if payloadBid, err := blockData.Block.SignedExecutionPayloadBid(); err == nil { + pageData.PayloadHeader = &models.SlotPagePayloadHeader{ + PayloadStatus: uint16(0), + ParentBlockHash: payloadBid.Message.ParentBlockHash[:], + ParentBlockRoot: payloadBid.Message.ParentBlockRoot[:], + BlockHash: payloadBid.Message.BlockHash[:], + GasLimit: uint64(payloadBid.Message.GasLimit), + BuilderIndex: uint64(payloadBid.Message.BuilderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadBid.Message.BuilderIndex) | services.BuilderIndexFlag), + Slot: uint64(payloadBid.Message.Slot), + Value: uint64(payloadBid.Message.Value), + BlobKzgCommitmentsRoot: payloadBid.Message.BlobKZGCommitmentsRoot[:], + Signature: payloadBid.Signature[:], + } + } + + var executionPayload *spec.VersionedExecutionPayload + if blockData.Block.Version >= spec.DataVersionGloas && blockData.Payload != nil { + blobKzgCommitments = blockData.Payload.Message.BlobKZGCommitments + executionPayload = &spec.VersionedExecutionPayload{ + Version: spec.DataVersionGloas, + Gloas: blockData.Payload.Message.Payload, + } + + pageData.PayloadHeader.PayloadStatus = uint16(1) + } else { + executionPayload, _ = blockData.Block.ExecutionPayload() + } + + if executionPayload != nil { pageData.ExecutionData = &models.SlotPageExecutionData{} if parentHash, err := executionPayload.ParentHash(); err == nil { diff --git a/handlers/voluntary_exits.go b/handlers/voluntary_exits.go index 7ab2b0a35..77b3db786 100644 --- a/handlers/voluntary_exits.go +++ b/handlers/voluntary_exits.go @@ -8,6 +8,7 @@ import ( "strings" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" @@ -163,40 +164,67 @@ func buildFilteredVoluntaryExitsPageData(pageIdx uint64, pageSize uint64, minSlo SlotRoot: voluntaryExit.SlotRoot, Time: chainState.SlotToTime(phase0.Slot(voluntaryExit.SlotNumber)), Orphaned: voluntaryExit.Orphaned, - ValidatorIndex: voluntaryExit.ValidatorIndex, - ValidatorName: services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex), ValidatorStatus: "", } - validator := services.GlobalBeaconService.GetValidatorByIndex(phase0.ValidatorIndex(voluntaryExit.ValidatorIndex), false) - if validator == nil { - voluntaryExitData.ValidatorStatus = "Unknown" - } else { - voluntaryExitData.PublicKey = validator.Validator.PublicKey[:] - voluntaryExitData.WithdrawalCreds = validator.Validator.WithdrawalCredentials - - if strings.HasPrefix(validator.Status.String(), "pending") { - voluntaryExitData.ValidatorStatus = "Pending" - } else if validator.Status == v1.ValidatorStateActiveOngoing { - voluntaryExitData.ValidatorStatus = "Active" - voluntaryExitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateActiveExiting { - voluntaryExitData.ValidatorStatus = "Exiting" - voluntaryExitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateActiveSlashed { - voluntaryExitData.ValidatorStatus = "Slashed" - voluntaryExitData.ShowUpcheck = true - } else if validator.Status == v1.ValidatorStateExitedUnslashed { - voluntaryExitData.ValidatorStatus = "Exited" - } else if validator.Status == v1.ValidatorStateExitedSlashed { - voluntaryExitData.ValidatorStatus = "Slashed" + // Check if this is a builder exit (validator index has BuilderIndexFlag set) + if voluntaryExit.ValidatorIndex&services.BuilderIndexFlag != 0 { + builderIndex := voluntaryExit.ValidatorIndex &^ services.BuilderIndexFlag + voluntaryExitData.IsBuilder = true + voluntaryExitData.ValidatorIndex = builderIndex + + // Resolve builder name via validatornames service (with BuilderIndexFlag) + voluntaryExitData.ValidatorName = services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex) + + builder := services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(builderIndex)) + if builder == nil { + voluntaryExitData.ValidatorStatus = "Unknown" } else { - voluntaryExitData.ValidatorStatus = validator.Status.String() + voluntaryExitData.PublicKey = builder.PublicKey[:] + + // Determine builder status + currentEpoch := chainState.CurrentEpoch() + if builder.WithdrawableEpoch <= currentEpoch { + voluntaryExitData.ValidatorStatus = "Exited" + } else { + voluntaryExitData.ValidatorStatus = "Exiting" + } } + } else { + // Regular validator exit + voluntaryExitData.ValidatorIndex = voluntaryExit.ValidatorIndex + voluntaryExitData.ValidatorName = services.GlobalBeaconService.GetValidatorName(voluntaryExit.ValidatorIndex) + + validator := services.GlobalBeaconService.GetValidatorByIndex(phase0.ValidatorIndex(voluntaryExit.ValidatorIndex), false) + if validator == nil { + voluntaryExitData.ValidatorStatus = "Unknown" + } else { + voluntaryExitData.PublicKey = validator.Validator.PublicKey[:] + voluntaryExitData.WithdrawalCreds = validator.Validator.WithdrawalCredentials + + if strings.HasPrefix(validator.Status.String(), "pending") { + voluntaryExitData.ValidatorStatus = "Pending" + } else if validator.Status == v1.ValidatorStateActiveOngoing { + voluntaryExitData.ValidatorStatus = "Active" + voluntaryExitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateActiveExiting { + voluntaryExitData.ValidatorStatus = "Exiting" + voluntaryExitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateActiveSlashed { + voluntaryExitData.ValidatorStatus = "Slashed" + voluntaryExitData.ShowUpcheck = true + } else if validator.Status == v1.ValidatorStateExitedUnslashed { + voluntaryExitData.ValidatorStatus = "Exited" + } else if validator.Status == v1.ValidatorStateExitedSlashed { + voluntaryExitData.ValidatorStatus = "Slashed" + } else { + voluntaryExitData.ValidatorStatus = validator.Status.String() + } - if voluntaryExitData.ShowUpcheck { - voluntaryExitData.UpcheckActivity = uint8(services.GlobalBeaconService.GetValidatorLiveness(validator.Index, 3)) - voluntaryExitData.UpcheckMaximum = uint8(3) + if voluntaryExitData.ShowUpcheck { + voluntaryExitData.UpcheckActivity = uint8(services.GlobalBeaconService.GetValidatorLiveness(validator.Index, 3)) + voluntaryExitData.UpcheckMaximum = uint8(3) + } } } diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 678664f1f..2700545ab 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -9,6 +9,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -532,6 +533,11 @@ func (indexer *Indexer) GetBuilderSetSize() uint64 { return indexer.builderCache.getBuilderSetSize() } +// GetBuilderByIndex returns the builder by index for the canonical head. +func (indexer *Indexer) GetBuilderByIndex(index gloas.BuilderIndex, overrideForkId *ForkKey) *gloas.Builder { + return indexer.builderCache.getBuilderByIndex(index, overrideForkId) +} + // GetRecentBuilderBalances returns the most recent builder balances for the given fork. func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phase0.Gwei { chainState := indexer.consensusPool.GetChainState() diff --git a/services/chainservice_builder.go b/services/chainservice_builder.go index 33819ca9f..6d8dc3a22 100644 --- a/services/chainservice_builder.go +++ b/services/chainservice_builder.go @@ -13,6 +13,10 @@ import ( "github.com/ethpandaops/dora/indexer/beacon" ) +// BuilderIndexFlag separates builder indices from validator indices +// A validator/builder index with this flag set is a builder index +const BuilderIndexFlag = beacon.BuilderIndexFlag + type BuilderWithIndex struct { Index gloas.BuilderIndex Builder *gloas.Builder @@ -241,6 +245,11 @@ func (bs *ChainService) GetFilteredBuilderSet(filter *dbtypes.BuilderFilter, wit return result, matchingCount } +// GetBuilderByIndex returns the builder by index +func (bs *ChainService) GetBuilderByIndex(index gloas.BuilderIndex) *gloas.Builder { + return bs.beaconIndexer.GetBuilderByIndex(index, nil) +} + // getBuilderStatus determines the status of a builder func getBuilderStatus(builder *gloas.Builder, currentEpoch phase0.Epoch, superseded bool) dbtypes.BuilderStatus { if superseded { diff --git a/services/chainservice_deposits.go b/services/chainservice_deposits.go index 5f21ec2ff..0f7b61e82 100644 --- a/services/chainservice_deposits.go +++ b/services/chainservice_deposits.go @@ -305,7 +305,8 @@ func (bs *ChainService) GetDepositOperationsByFilter(filter *dbtypes.DepositFilt if len(txFilter.WithdrawalAddress) > 0 { wdcreds := depositWithTx.WithdrawalCredentials - if wdcreds[0] != 0x01 && wdcreds[0] != 0x02 { + // 0x01 = ETH1, 0x02 = compounding, 0x03 = builder deposit + if wdcreds[0] != 0x01 && wdcreds[0] != 0x02 && wdcreds[0] != 0x03 { continue } @@ -526,11 +527,11 @@ func (bs *ChainService) GetIndexedDepositQueue(headBlock *beacon.Block) *Indexed indexedQueue.QueueEstimation = queueEpoch if lastNormalDeposit != nil && !bytes.Equal(lastNormalDeposit.PendingDeposit.Pubkey[:], lastIncludedDeposit.PublicKey[:]) { - // something is bad, return empty queue - logrus.Warnf("ChainService.GetIndexedDepositQueue: last included deposit not found in queue, %x != %x", lastNormalDeposit.PendingDeposit.Pubkey[:], lastIncludedDeposit.PublicKey[:]) - return &IndexedDepositQueue{ - Queue: []*IndexedDepositQueueEntry{}, - } + // Mismatch between queue and included deposits - this can happen if there are + // builder deposits (0x03) that skip the queue. Log warning but still return + // the queue to show useful information. The deposit indexes might not be perfectly + // matched but the queue itself is still valid. + logrus.Debugf("ChainService.GetIndexedDepositQueue: last included deposit not found in queue (possibly due to builder deposits), %x != %x", lastNormalDeposit.PendingDeposit.Pubkey[:], lastIncludedDeposit.PublicKey[:]) } return indexedQueue @@ -598,7 +599,17 @@ func (bs *ChainService) getLastIncludedDeposit(headRoot phase0.Root) *dbtypes.De } if len(deposits) > 0 { - lastDeposits = deposits + // Filter out builder deposits (0x03) as they skip the queue + filteredDeposits := make([]*dbtypes.Deposit, 0, len(deposits)) + for _, deposit := range deposits { + if len(deposit.WithdrawalCredentials) > 0 && deposit.WithdrawalCredentials[0] == 0x03 { + continue // Skip builder deposits + } + filteredDeposits = append(filteredDeposits, deposit) + } + if len(filteredDeposits) > 0 { + lastDeposits = filteredDeposits + } } } } diff --git a/templates/deposits/deposits.html b/templates/deposits/deposits.html index f9b27d0cb..cd6bd9f77 100644 --- a/templates/deposits/deposits.html +++ b/templates/deposits/deposits.html @@ -162,7 +162,11 @@
    This table displays the deposits received by the Beac
    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} @@ -187,18 +191,18 @@
    This table displays the deposits received by the Beac {{ end }} {{ if $deposit.IsQueued }} - Queued {{ end }} {{ if $deposit.InvalidSignature }} - @@ -295,7 +299,11 @@
    This table displays the deposits made for validators
    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} @@ -410,7 +418,11 @@
    This table displays deposits waiting to be activated
    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} diff --git a/templates/exits/exits.html b/templates/exits/exits.html index 62217b0e6..e552fa0a8 100644 --- a/templates/exits/exits.html +++ b/templates/exits/exits.html @@ -138,7 +138,11 @@
    This table displays the most recent voluntary exit re {{ formatRecentTimeShort $exit.Time }} - {{ formatValidatorWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + {{ if $exit.IsBuilder }} + {{ formatBuilderWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + {{ end }}
    diff --git a/templates/included_deposits/included_deposits.html b/templates/included_deposits/included_deposits.html index 240dd3684..e3a650abb 100644 --- a/templates/included_deposits/included_deposits.html +++ b/templates/included_deposits/included_deposits.html @@ -194,7 +194,11 @@

    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} diff --git a/templates/initiated_deposits/initiated_deposits.html b/templates/initiated_deposits/initiated_deposits.html index 35814a1e2..3e274b914 100644 --- a/templates/initiated_deposits/initiated_deposits.html +++ b/templates/initiated_deposits/initiated_deposits.html @@ -166,7 +166,11 @@

    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} diff --git a/templates/queued_deposits/queued_deposits.html b/templates/queued_deposits/queued_deposits.html index ed52a4250..88980d9a9 100644 --- a/templates/queued_deposits/queued_deposits.html +++ b/templates/queued_deposits/queued_deposits.html @@ -135,7 +135,11 @@

    {{ if $deposit.ValidatorExists }} - {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ if $deposit.IsBuilder }} + {{ formatBuilderWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $deposit.ValidatorIndex $deposit.ValidatorName }} + {{ end }} {{ else }} 0x{{ printf "%x" $deposit.PublicKey }} {{ end }} diff --git a/templates/slot/overview.html b/templates/slot/overview.html index 9ba17e99b..270db1c12 100644 --- a/templates/slot/overview.html +++ b/templates/slot/overview.html @@ -201,9 +201,9 @@
    -
    Builder Index:
    +
    Builder:
    - {{ formatValidator .BuilderIndex .BuilderName }} + {{ formatBuilderWithIndex .BuilderIndex .BuilderName }}
    diff --git a/templates/slot/voluntary_exits.html b/templates/slot/voluntary_exits.html index 0ffbc0333..187999d64 100644 --- a/templates/slot/voluntary_exits.html +++ b/templates/slot/voluntary_exits.html @@ -13,7 +13,13 @@ {{ range $i, $exit := .Block.VoluntaryExits }} {{ $i }} - {{ formatValidatorWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + + {{ if $exit.IsBuilder }} + {{ formatBuilderWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $exit.ValidatorIndex $exit.ValidatorName }} + {{ end }} + {{ $exit.Epoch }} 0x{{ printf "%x" $exit.Signature }} diff --git a/templates/voluntary_exits/voluntary_exits.html b/templates/voluntary_exits/voluntary_exits.html index d7fb14c17..93b48154d 100644 --- a/templates/voluntary_exits/voluntary_exits.html +++ b/templates/voluntary_exits/voluntary_exits.html @@ -139,7 +139,13 @@

    {{ formatAddCommas $voluntaryExit.SlotNumber }} {{ end }} {{ formatRecentTimeShort $voluntaryExit.Time }} - {{ formatValidatorWithIndex $voluntaryExit.ValidatorIndex $voluntaryExit.ValidatorName }} + + {{ if $voluntaryExit.IsBuilder }} + {{ formatBuilderWithIndex $voluntaryExit.ValidatorIndex $voluntaryExit.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $voluntaryExit.ValidatorIndex $voluntaryExit.ValidatorName }} + {{ end }} +
    diff --git a/types/models/deposits.go b/types/models/deposits.go index 83e836d31..f8b027e2a 100644 --- a/types/models/deposits.go +++ b/types/models/deposits.go @@ -43,6 +43,7 @@ type DepositsPageDataInitiatedDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } type DepositsPageDataIncludedDeposit struct { @@ -69,6 +70,7 @@ type DepositsPageDataIncludedDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } type DepositsPageDataIncludedDepositTxDetails struct { @@ -98,6 +100,7 @@ type DepositsPageDataQueuedDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } type DepositsPageDataQueuedDepositTxDetails struct { diff --git a/types/models/exits.go b/types/models/exits.go index 683aa2fa4..340d71f11 100644 --- a/types/models/exits.go +++ b/types/models/exits.go @@ -29,6 +29,7 @@ type ExitsPageDataRecentExit struct { Orphaned bool `json:"orphaned"` ValidatorIndex uint64 `json:"vindex"` ValidatorName string `json:"vname"` + IsBuilder bool `json:"is_builder"` PublicKey []byte `json:"pubkey"` WithdrawalCreds []byte `json:"wdcreds"` ValidatorStatus string `json:"vstatus"` diff --git a/types/models/included_deposits.go b/types/models/included_deposits.go index fd597f1d4..338292e81 100644 --- a/types/models/included_deposits.go +++ b/types/models/included_deposits.go @@ -62,6 +62,7 @@ type IncludedDepositsPageDataDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } type IncludedDepositsPageDataDepositTxDetails struct { diff --git a/types/models/initiated_deposits.go b/types/models/initiated_deposits.go index 5f1c91fc0..8bfb5558f 100644 --- a/types/models/initiated_deposits.go +++ b/types/models/initiated_deposits.go @@ -53,4 +53,5 @@ type InitiatedDepositsPageDataDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } diff --git a/types/models/queued_deposits.go b/types/models/queued_deposits.go index c5bfc90ae..351805744 100644 --- a/types/models/queued_deposits.go +++ b/types/models/queued_deposits.go @@ -54,6 +54,7 @@ type QueuedDepositsPageDataDeposit struct { ValidatorExists bool `json:"validator_exists"` ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } type QueuedDepositsPageDataDepositTxDetails struct { diff --git a/types/models/slot.go b/types/models/slot.go index 6a262a40e..121b8a89b 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -155,6 +155,7 @@ type SlotPageDeposit struct { type SlotPageVoluntaryExit struct { ValidatorIndex uint64 `json:"validatorindex"` ValidatorName string `json:"validatorname"` + IsBuilder bool `json:"is_builder"` Epoch uint64 `json:"epoch"` Signature []byte `json:"signature"` } diff --git a/types/models/voluntary_exits.go b/types/models/voluntary_exits.go index cd5ee2a16..2bfc2fae3 100644 --- a/types/models/voluntary_exits.go +++ b/types/models/voluntary_exits.go @@ -41,6 +41,7 @@ type VoluntaryExitsPageDataExit struct { Orphaned bool `json:"orphaned"` ValidatorIndex uint64 `json:"vindex"` ValidatorName string `json:"vname"` + IsBuilder bool `json:"is_builder"` PublicKey []byte `json:"pubkey"` WithdrawalCreds []byte `json:"wdcreds"` ValidatorStatus string `json:"vstatus"` diff --git a/utils/format.go b/utils/format.go index 3ac100ba5..df3c90f45 100644 --- a/utils/format.go +++ b/utils/format.go @@ -636,6 +636,29 @@ func FormatValidatorNameWithIndex(index uint64, name string) template.HTML { return template.HTML(fmt.Sprintf("%v", index)) } +func FormatBuilder(index uint64, name string) template.HTML { + return formatBuilder(index, name, "fa-hard-hat mr-2", false) +} + +func FormatBuilderWithIndex(index uint64, name string) template.HTML { + return formatBuilder(index, name, "fa-hard-hat mr-2", true) +} + +func formatBuilder(index uint64, name string, icon string, withIndex bool) template.HTML { + if index == math.MaxInt64 { + return template.HTML(fmt.Sprintf(" unknown", icon)) + } else if name != "" { + var nameLabel string + if withIndex { + nameLabel = fmt.Sprintf("%v (%v)", html.EscapeString(name), index) + } else { + nameLabel = html.EscapeString(name) + } + return template.HTML(fmt.Sprintf(" %v", index, icon, index, nameLabel)) + } + return template.HTML(fmt.Sprintf(" %v", icon, index, index)) +} + func FormatRecentTimeShort(ts time.Time) template.HTML { duration := time.Until(ts) var timeStr string diff --git a/utils/templateFucs.go b/utils/templateFucs.go index f1f44510c..d2fc8b779 100644 --- a/utils/templateFucs.go +++ b/utils/templateFucs.go @@ -86,6 +86,8 @@ func GetTemplateFuncs() template.FuncMap { "formatValidatorWithIndex": FormatValidatorWithIndex, "formatValidatorNameWithIndex": FormatValidatorNameWithIndex, "formatSlashedValidator": FormatSlashedValidator, + "formatBuilder": FormatBuilder, + "formatBuilderWithIndex": FormatBuilderWithIndex, "formatWithdawalCredentials": FormatWithdawalCredentials, "formatRecentTimeShort": FormatRecentTimeShort, "formatGraffiti": FormatGraffiti, From b85c01b6d762a0401a7260232b5a77911b63f7e5 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 20 Jan 2026 18:30:48 +0100 Subject: [PATCH 29/99] fix unused function --- indexer/beacon/buildercache.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/indexer/beacon/buildercache.go b/indexer/beacon/buildercache.go index 41f9c12a1..1a8114ae3 100644 --- a/indexer/beacon/buildercache.go +++ b/indexer/beacon/buildercache.go @@ -290,18 +290,6 @@ func (cache *builderCache) getBuilderSetSize() uint64 { return uint64(len(cache.builderSetCache)) } -// getBuilderFlags returns the status flags for a specific builder -func (cache *builderCache) getBuilderFlags(builderIndex gloas.BuilderIndex) uint16 { - cache.cacheMutex.RLock() - defer cache.cacheMutex.RUnlock() - - if uint64(builderIndex) >= uint64(len(cache.builderSetCache)) || cache.builderSetCache[builderIndex] == nil { - return 0 - } - - return cache.builderSetCache[builderIndex].statusFlags -} - // setFinalizedEpoch updates the builder cache when a new epoch is finalized func (cache *builderCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDependentRoot phase0.Root) { cache.cacheMutex.Lock() From 3f6cf115fe3df3b18262e9645e7a5dbd629d0b61 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 20 Jan 2026 21:18:03 +0100 Subject: [PATCH 30/99] add builder details page --- cmd/dora-explorer/main.go | 1 + db/block_bids.go | 37 ++++ handlers/builder.go | 284 ++++++++++++++++++++++++++ handlers/pageData.go | 18 +- templates/builder/builder.html | 235 +++++++++++++++++++++ templates/builder/notfound.html | 27 +++ templates/builder/recentBids.html | 57 ++++++ templates/builder/recentBlocks.html | 58 ++++++ templates/builder/recentDeposits.html | 52 +++++ types/models/builders.go | 70 +++++++ 10 files changed, 837 insertions(+), 2 deletions(-) create mode 100644 handlers/builder.go create mode 100644 templates/builder/builder.html create mode 100644 templates/builder/notfound.html create mode 100644 templates/builder/recentBids.html create mode 100644 templates/builder/recentBlocks.html create mode 100644 templates/builder/recentDeposits.html diff --git a/cmd/dora-explorer/main.go b/cmd/dora-explorer/main.go index eee977e44..443bf7af3 100644 --- a/cmd/dora-explorer/main.go +++ b/cmd/dora-explorer/main.go @@ -232,6 +232,7 @@ func startFrontend(router *mux.Router) { router.HandleFunc("/validator/{idxOrPubKey}", handlers.Validator).Methods("GET") router.HandleFunc("/validator/{index}/slots", handlers.ValidatorSlots).Methods("GET") router.HandleFunc("/builders", handlers.Builders).Methods("GET") + router.HandleFunc("/builder/{idxOrPubKey}", handlers.BuilderDetail).Methods("GET") if utils.Config.Frontend.Pprof { // add pprof handler diff --git a/db/block_bids.go b/db/block_bids.go index bfcfac474..c2ac142b0 100644 --- a/db/block_bids.go +++ b/db/block_bids.go @@ -111,3 +111,40 @@ func DeleteBidsBeforeSlot(minSlot uint64, tx *sqlx.Tx) error { _, err := tx.Exec(`DELETE FROM block_bids WHERE slot < $1`, minSlot) return err } + +// GetBidsByBuilderIndex returns bids submitted by a specific builder, ordered by slot descending +func GetBidsByBuilderIndex(builderIndex uint64, offset uint64, limit uint32) ([]*dbtypes.BlockBid, uint64) { + var sql strings.Builder + args := []any{ + builderIndex, + } + fmt.Fprint(&sql, ` + SELECT + parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment + FROM block_bids + WHERE builder_index = $1 + ORDER BY slot DESC, value DESC + `) + + if limit > 0 { + fmt.Fprintf(&sql, " LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) + args = append(args, limit, offset) + } + + bids := []*dbtypes.BlockBid{} + err := ReaderDb.Select(&bids, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching bids for builder index %d: %v", builderIndex, err) + return nil, 0 + } + + // Get total count + var totalCount uint64 + err = ReaderDb.Get(&totalCount, `SELECT COUNT(*) FROM block_bids WHERE builder_index = $1`, builderIndex) + if err != nil { + logger.Errorf("Error while counting bids for builder index %d: %v", builderIndex, err) + return bids, 0 + } + + return bids, totalCount +} diff --git a/handlers/builder.go b/handlers/builder.go new file mode 100644 index 000000000..aa4f9ff1e --- /dev/null +++ b/handlers/builder.go @@ -0,0 +1,284 @@ +package handlers + +import ( + "encoding/hex" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/dora/indexer/beacon" + "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/dora/templates" + "github.com/ethpandaops/dora/types/models" +) + +// BuilderDetail will return the main "builder" page using a go template +func BuilderDetail(w http.ResponseWriter, r *http.Request) { + var builderTemplateFiles = append(layoutTemplateFiles, + "builder/builder.html", + "builder/recentBlocks.html", + "builder/recentBids.html", + "builder/recentDeposits.html", + "_svg/timeline.html", + ) + var notfoundTemplateFiles = append(layoutTemplateFiles, + "builder/notfound.html", + ) + + var pageTemplate = templates.GetTemplate(builderTemplateFiles...) + data := InitPageData(w, r, "builders", "/builder", "Builder", builderTemplateFiles) + + var builder *gloas.Builder + var builderIndex uint64 + var superseded bool + + vars := mux.Vars(r) + idxOrPubKey := strings.Replace(vars["idxOrPubKey"], "0x", "", -1) + builderPubKey, err := hex.DecodeString(idxOrPubKey) + if err != nil || len(builderPubKey) != 48 { + // search by index + idx, err := strconv.ParseUint(vars["idxOrPubKey"], 10, 64) + if err == nil { + builderIndex = idx + builder = services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(idx)) + if builder == nil { + // Try from DB + dbBuilder := db.GetActiveBuilderByIndex(idx) + if dbBuilder != nil { + builder = beacon.UnwrapDbBuilder(dbBuilder) + superseded = dbBuilder.Superseded + } + } + } + } else { + // search by pubkey + dbBuilder := db.GetBuilderByPubkey(builderPubKey) + if dbBuilder != nil { + builderIndex = dbBuilder.BuilderIndex + superseded = dbBuilder.Superseded + builder = services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(dbBuilder.BuilderIndex)) + if builder == nil { + builder = beacon.UnwrapDbBuilder(dbBuilder) + } + } + } + + if builder == nil { + data := InitPageData(w, r, "builders", "/builder", "Builder not found", notfoundTemplateFiles) + w.Header().Set("Content-Type", "text/html") + handleTemplateError(w, r, "builder.go", "BuilderDetail", "", templates.GetTemplate(notfoundTemplateFiles...).ExecuteTemplate(w, "layout", data)) + return + } + + tabView := "blocks" + if r.URL.Query().Has("v") { + tabView = r.URL.Query().Get("v") + } + + var pageError error + pageError = services.GlobalCallRateLimiter.CheckCallLimit(r, 1) + if pageError == nil { + data.Data, pageError = getBuilderPageData(builderIndex, superseded, tabView) + } + if data.Data == nil { + pageError = errors.New("builder not found") + } + if pageError != nil { + handlePageError(w, r, pageError) + return + } + w.Header().Set("Content-Type", "text/html") + + if r.URL.Query().Has("lazy") { + // return the selected tab content only (lazy loaded) + handleTemplateError(w, r, "builder.go", "BuilderDetail", "", pageTemplate.ExecuteTemplate(w, "lazyPage", data.Data)) + } else { + handleTemplateError(w, r, "builder.go", "BuilderDetail", "", pageTemplate.ExecuteTemplate(w, "layout", data)) + } +} + +func getBuilderPageData(builderIndex uint64, superseded bool, tabView string) (*models.BuilderPageData, error) { + pageData := &models.BuilderPageData{} + pageCacheKey := fmt.Sprintf("builder:%v:%v", builderIndex, tabView) + pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { + pageData, cacheTimeout := buildBuilderPageData(builderIndex, superseded, tabView) + pageCall.CacheTimeout = cacheTimeout + return pageData + }) + if pageErr == nil && pageRes != nil { + resData, resOk := pageRes.(*models.BuilderPageData) + if !resOk { + return nil, ErrInvalidPageModel + } + pageData = resData + } + return pageData, pageErr +} + +func buildBuilderPageData(builderIndex uint64, superseded bool, tabView string) (*models.BuilderPageData, time.Duration) { + logrus.Debugf("builder page called: %v", builderIndex) + + chainState := services.GlobalBeaconService.GetChainState() + specs := chainState.GetSpecs() + currentEpoch := chainState.CurrentEpoch() + + // Get builder data + builder := services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(builderIndex)) + if builder == nil { + // Try from DB + dbBuilder := db.GetActiveBuilderByIndex(builderIndex) + if dbBuilder != nil { + builder = beacon.UnwrapDbBuilder(dbBuilder) + superseded = dbBuilder.Superseded + } + } + if builder == nil { + return nil, 0 + } + + // Determine state + state := "Active" + if superseded { + state = "Superseded" + } else if builder.WithdrawableEpoch <= currentEpoch { + state = "Exited" + } + + pageData := &models.BuilderPageData{ + CurrentEpoch: uint64(currentEpoch), + Index: builderIndex, + Name: services.GlobalBeaconService.GetValidatorName(builderIndex | services.BuilderIndexFlag), + PublicKey: builder.PublicKey[:], + Balance: uint64(builder.Balance), + ExecutionAddress: builder.ExecutionAddress[:], + Version: builder.Version, + State: state, + IsSuperseded: superseded, + TabView: tabView, + GloasIsActive: specs.GloasForkEpoch != nil && uint64(currentEpoch) >= *specs.GloasForkEpoch, + } + + // Deposit epoch + if builder.DepositEpoch < 18446744073709551615 { + pageData.ShowDeposit = true + pageData.DepositEpoch = uint64(builder.DepositEpoch) + pageData.DepositTs = chainState.EpochToTime(builder.DepositEpoch) + } + + // Withdrawable epoch + if builder.WithdrawableEpoch < 18446744073709551615 { + pageData.ShowWithdrawable = true + pageData.WithdrawableEpoch = uint64(builder.WithdrawableEpoch) + pageData.WithdrawableTs = chainState.EpochToTime(builder.WithdrawableEpoch) + } + + // Load tab-specific data + switch tabView { + case "blocks": + pageData.RecentBlocks = buildBuilderRecentBlocks(builderIndex, chainState) + case "bids": + pageData.RecentBids = buildBuilderRecentBids(builderIndex, chainState) + case "deposits": + pageData.RecentDeposits = buildBuilderRecentDeposits(builderIndex, chainState) + } + + return pageData, 10 * time.Minute +} + +func buildBuilderRecentBlocks(builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBlock { + // Get recent bids from this builder that have been included (winning bids) + bids, _ := db.GetBidsByBuilderIndex(builderIndex, 0, 20) + + blocks := make([]*models.BuilderPageDataBlock, 0, len(bids)) + for _, bid := range bids { + // Check if this bid was actually included (payload status) + slots := db.GetSlotsByBlockHash(bid.BlockHash) + for _, slot := range slots { + if slot.PayloadStatus == dbtypes.PayloadStatusCanonical || slot.PayloadStatus == dbtypes.PayloadStatusOrphaned { + blocks = append(blocks, &models.BuilderPageDataBlock{ + Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot.Slot))), + Slot: slot.Slot, + Ts: chainState.SlotToTime(phase0.Slot(slot.Slot)), + BlockRoot: slot.Root, + BlockHash: bid.BlockHash, + Status: uint16(slot.PayloadStatus), + FeeRecipient: bid.FeeRecipient, + GasLimit: bid.GasLimit, + Value: bid.Value, + ElPayment: bid.ElPayment, + }) + break + } + } + } + + return blocks +} + +func buildBuilderRecentBids(builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBid { + bids, _ := db.GetBidsByBuilderIndex(builderIndex, 0, 20) + + result := make([]*models.BuilderPageDataBid, 0, len(bids)) + for _, bid := range bids { + bidData := &models.BuilderPageDataBid{ + Slot: bid.Slot, + Ts: chainState.SlotToTime(phase0.Slot(bid.Slot)), + ParentRoot: bid.ParentRoot, + ParentHash: bid.ParentHash, + BlockHash: bid.BlockHash, + FeeRecipient: bid.FeeRecipient, + GasLimit: bid.GasLimit, + Value: bid.Value, + ElPayment: bid.ElPayment, + IsWinning: false, + } + + // Check if this bid won (payload was included) + slots := db.GetSlotsByBlockHash(bid.BlockHash) + for _, slot := range slots { + if slot.PayloadStatus == dbtypes.PayloadStatusCanonical { + bidData.IsWinning = true + break + } + } + + result = append(result, bidData) + } + + return result +} + +func buildBuilderRecentDeposits(builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataDeposit { + // Builder exits are tracked as voluntary exits with BuilderIndexFlag set + builderIndexWithFlag := builderIndex | services.BuilderIndexFlag + filter := &dbtypes.VoluntaryExitFilter{ + MinIndex: builderIndexWithFlag, + MaxIndex: builderIndexWithFlag, + } + + exits, _ := services.GlobalBeaconService.GetVoluntaryExitsByFilter(filter, 0, 20) + + result := make([]*models.BuilderPageDataDeposit, 0, len(exits)) + for _, exit := range exits { + result = append(result, &models.BuilderPageDataDeposit{ + Type: "exit", + SlotNumber: exit.SlotNumber, + SlotRoot: exit.SlotRoot, + Time: chainState.SlotToTime(phase0.Slot(exit.SlotNumber)), + Orphaned: exit.Orphaned, + }) + } + + return result +} diff --git a/handlers/pageData.go b/handlers/pageData.go index 2dc6b1717..03546fb02 100644 --- a/handlers/pageData.go +++ b/handlers/pageData.go @@ -90,6 +90,8 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st } func createMenuItems(active string) []types.MainMenuItem { + chainState := services.GlobalBeaconService.GetChainState() + specs := chainState.GetSpecs() hiddenFor := []string{"confirmation", "login", "register"} if utils.SliceContains(hiddenFor, active) { @@ -203,6 +205,20 @@ func createMenuItems(active string) []types.MainMenuItem { validatorMenu = append(validatorMenu, types.NavigationGroup{ Links: validatorMenuLinks, }) + + if specs != nil && specs.GloasForkEpoch != nil && uint64(chainState.CurrentEpoch()) >= *specs.GloasForkEpoch { + builderMenu := []types.NavigationLink{ + { + Label: "Builders", + Path: "/builders", + Icon: "fa-building", + }, + } + validatorMenu = append(validatorMenu, types.NavigationGroup{ + Links: builderMenu, + }) + } + validatorMenu = append(validatorMenu, types.NavigationGroup{ Links: []types.NavigationLink{ { @@ -223,8 +239,6 @@ func createMenuItems(active string) []types.MainMenuItem { }, }) - chainState := services.GlobalBeaconService.GetChainState() - specs := chainState.GetSpecs() if specs != nil && specs.ElectraForkEpoch != nil && uint64(chainState.CurrentEpoch()) >= *specs.ElectraForkEpoch { validatorMenu = append(validatorMenu, types.NavigationGroup{ Links: []types.NavigationLink{ diff --git a/templates/builder/builder.html b/templates/builder/builder.html new file mode 100644 index 000000000..dbbf37382 --- /dev/null +++ b/templates/builder/builder.html @@ -0,0 +1,235 @@ +{{ define "page" }} +
    +
    +

    Builder {{ formatBuilderWithIndex .Index .Name }}

    + +
    + +
    +
    + +
    +
    + Status: +
    +
    + +
    +
    +
    +
    Deposited
    +
    + + + +
    +
    +
    +
    + {{ if .ShowDeposit }} + + {{ end }} +
    +
    +
    +
    +
    Active
    +
    + + + + +
    +
    +
    +
    +
    + {{ if .ShowWithdrawable }} + + {{ end }} +
    +
    +
    +
    +
    Exited
    +
    + + +
    +
    +
    +
    +
    +
    +
    + +
    +
    Index:
    +
    + {{ formatBuilderWithIndex .Index .Name }} + +
    +
    +
    +
    Public Key:
    +
    + 0x{{ printf "%x" .PublicKey }} + +
    +
    +
    +
    Execution Address:
    +
    + {{ ethAddressLink .ExecutionAddress }} +
    +
    +
    +
    Status:
    +
    + {{ if eq .State "Active" }} + Active + {{ else if eq .State "Exited" }} + Exited + {{ else if eq .State "Superseded" }} + Superseded + {{ else }} + {{ .State }} + {{ end }} +
    +
    +
    +
    Balance:
    +
    + {{ formatEthFromGwei .Balance }} + +
    +
    +
    +
    Version:
    +
    + {{ .Version }} +
    +
    + {{ if .ShowDeposit }} +
    +
    Deposit Epoch:
    +
    + {{ formatAddCommas .DepositEpoch }} + ({{ formatTimestamp .DepositTs }}) +
    +
    + {{ end }} + {{ if .ShowWithdrawable }} +
    +
    Withdrawable Epoch:
    +
    + {{ formatAddCommas .WithdrawableEpoch }} + ({{ formatTimestamp .WithdrawableTs }}) +
    +
    + {{ end }} + +
    +
    + + + + +
    +
    + {{ if eq .TabView "blocks" }} + {{ template "recentBlocks" . }} + {{ end }} +
    +
    + {{ if eq .TabView "bids" }} + {{ template "recentBids" . }} + {{ end }} +
    +
    + {{ if eq .TabView "deposits" }} + {{ template "recentDeposits" . }} + {{ end }} +
    +
    + +
    +{{ end }} +{{ define "lazyPage" }} + {{ if eq .TabView "blocks" }} + {{ template "recentBlocks" . }} + {{ else if eq .TabView "bids" }} + {{ template "recentBids" . }} + {{ else if eq .TabView "deposits" }} + {{ template "recentDeposits" . }} + {{ else }} + Unknown tab + {{ end }} +{{ end }} +{{ define "js" }} + +{{ end }} +{{ define "css" }} + +{{ end }} diff --git a/templates/builder/notfound.html b/templates/builder/notfound.html new file mode 100644 index 000000000..b92472273 --- /dev/null +++ b/templates/builder/notfound.html @@ -0,0 +1,27 @@ +{{ define "js" }} +{{ end }} + +{{ define "css" }} +{{ end }} + +{{ define "page" }} +
    +
    +
    +

    Builder not found

    + +
    +
    +
    +
    +
    Sorry but we could not find the builder you are looking for. The builder may not exist or has not been indexed yet.
    +
    +
    +
    +{{ end }} diff --git a/templates/builder/recentBids.html b/templates/builder/recentBids.html new file mode 100644 index 000000000..d3cbddb7b --- /dev/null +++ b/templates/builder/recentBids.html @@ -0,0 +1,57 @@ +{{ define "recentBids" }} +
    +
    +
    + + + + + + + + + + + + + + {{ if gt (len .RecentBids) 0 }} + {{ range $i, $bid := .RecentBids }} + + + + + + + + + + {{ end }} + {{ else }} + + + + + + {{ end }} + +
    SlotTimeBlock HashGas LimitValueEL PaymentStatus
    {{ formatAddCommas $bid.Slot }}{{ formatRecentTimeShort $bid.Ts }} + + 0x{{ printf "%x" $bid.BlockHash }} + + + {{ formatAddCommas $bid.GasLimit }}{{ formatEthFromGwei $bid.Value }}{{ formatEthFromGwei $bid.ElPayment }} + {{ if $bid.IsWinning }} + Won + {{ else }} + - + {{ end }} +
    +
    + {{ template "timeline_svg" }} +
    +
    +
    +
    +
    +{{ end }} diff --git a/templates/builder/recentBlocks.html b/templates/builder/recentBlocks.html new file mode 100644 index 000000000..c03749e4a --- /dev/null +++ b/templates/builder/recentBlocks.html @@ -0,0 +1,58 @@ +{{ define "recentBlocks" }} +
    +
    +
    + + + + + + + + + + + + + {{ if gt (len .RecentBlocks) 0 }} + {{ range $i, $block := .RecentBlocks }} + + + + + + + + + {{ end }} + {{ else }} + + + + + + {{ end }} + +
    EpochSlotBlock HashStatusTimeValue
    {{ formatAddCommas $block.Epoch }}{{ formatAddCommas $block.Slot }} + + 0x{{ printf "%x" $block.BlockHash }} + + + {{ if eq $block.Status 0 }} + Missing + {{ else if eq $block.Status 1 }} + Canonical + {{ else if eq $block.Status 2 }} + Orphaned + {{ else }} + Unknown + {{ end }} + {{ formatRecentTimeShort $block.Ts }}{{ formatEthFromGwei $block.Value }}
    +
    + {{ template "timeline_svg" }} +
    +
    +
    +
    +
    +{{ end }} diff --git a/templates/builder/recentDeposits.html b/templates/builder/recentDeposits.html new file mode 100644 index 000000000..19f3177f5 --- /dev/null +++ b/templates/builder/recentDeposits.html @@ -0,0 +1,52 @@ +{{ define "recentDeposits" }} +
    +
    +
    + + + + + + + + + + + {{ if gt (len .RecentDeposits) 0 }} + {{ range $i, $deposit := .RecentDeposits }} + + + + + + + {{ end }} + {{ else }} + + + + + + {{ end }} + +
    TypeSlotTimeStatus
    + {{ if eq $deposit.Type "exit" }} + Voluntary Exit + {{ else }} + {{ $deposit.Type }} + {{ end }} + {{ formatAddCommas $deposit.SlotNumber }}{{ formatRecentTimeShort $deposit.Time }} + {{ if $deposit.Orphaned }} + Orphaned + {{ else }} + Included + {{ end }} +
    +
    + {{ template "timeline_svg" }} +
    +
    +
    +
    +
    +{{ end }} diff --git a/types/models/builders.go b/types/models/builders.go index 8f705a48a..5bb02a877 100644 --- a/types/models/builders.go +++ b/types/models/builders.go @@ -48,3 +48,73 @@ type BuildersPageDataBuilder struct { WithdrawableTs time.Time `json:"withdrawable_ts"` WithdrawableEpoch uint64 `json:"withdrawable_epoch"` } + +// BuilderPageData holds data for the builder details page +type BuilderPageData struct { + CurrentEpoch uint64 `json:"current_epoch"` + Index uint64 `json:"index"` + Name string `json:"name"` + PublicKey []byte `json:"pubkey"` + Balance uint64 `json:"balance"` + ExecutionAddress []byte `json:"execution_address"` + Version uint8 `json:"version"` + State string `json:"state"` // "Active", "Exited", "Superseded" + + // Deposit lifecycle + ShowDeposit bool `json:"show_deposit"` + DepositEpoch uint64 `json:"deposit_epoch"` + DepositTs time.Time `json:"deposit_ts"` + + // Withdrawable lifecycle + ShowWithdrawable bool `json:"show_withdrawable"` + WithdrawableEpoch uint64 `json:"withdrawable_epoch"` + WithdrawableTs time.Time `json:"withdrawable_ts"` + + IsSuperseded bool `json:"is_superseded"` + + // Tab control + TabView string `json:"tab_view"` + GloasIsActive bool `json:"gloas_is_active"` + + // Tab data (loaded conditionally) + RecentBlocks []*BuilderPageDataBlock `json:"recent_blocks"` + RecentBids []*BuilderPageDataBid `json:"recent_bids"` + RecentDeposits []*BuilderPageDataDeposit `json:"recent_deposits"` +} + +// BuilderPageDataBlock represents a block/payload built by this builder +type BuilderPageDataBlock struct { + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + Ts time.Time `json:"ts"` + BlockRoot []byte `json:"block_root"` + BlockHash []byte `json:"block_hash"` + Status uint16 `json:"status"` // PayloadStatus + FeeRecipient []byte `json:"fee_recipient"` + GasLimit uint64 `json:"gas_limit"` + Value uint64 `json:"value"` + ElPayment uint64 `json:"el_payment"` +} + +// BuilderPageDataBid represents a bid submitted by this builder +type BuilderPageDataBid struct { + Slot uint64 `json:"slot"` + Ts time.Time `json:"ts"` + ParentRoot []byte `json:"parent_root"` + ParentHash []byte `json:"parent_hash"` + BlockHash []byte `json:"block_hash"` + FeeRecipient []byte `json:"fee_recipient"` + GasLimit uint64 `json:"gas_limit"` + Value uint64 `json:"value"` + ElPayment uint64 `json:"el_payment"` + IsWinning bool `json:"is_winning"` +} + +// BuilderPageDataDeposit represents a builder deposit or voluntary exit +type BuilderPageDataDeposit struct { + Type string `json:"type"` // "exit" + SlotNumber uint64 `json:"slot"` + SlotRoot []byte `json:"slot_root"` + Time time.Time `json:"time"` + Orphaned bool `json:"orphaned"` +} From ce769e374e82d4540cdc41d8c46cf735de36e713 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 25 Jan 2026 21:17:05 +0100 Subject: [PATCH 31/99] add builder credential type to deposit builder --- .../components/SubmitDepositsForm/DepositGenerator.ts | 10 +++++----- .../SubmitDepositsForm/DepositGeneratorModal.tsx | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ui-package/src/components/SubmitDepositsForm/DepositGenerator.ts b/ui-package/src/components/SubmitDepositsForm/DepositGenerator.ts index 9e292326f..f440f8f08 100644 --- a/ui-package/src/components/SubmitDepositsForm/DepositGenerator.ts +++ b/ui-package/src/components/SubmitDepositsForm/DepositGenerator.ts @@ -30,7 +30,7 @@ const SigningData = new ContainerType({ domain: new ByteVectorType(32), }); -export type CredentialType = '00' | '01' | '02'; +export type CredentialType = '00' | '01' | '02' | '03'; export interface WithdrawalCredentialConfig { type: CredentialType; @@ -66,10 +66,10 @@ export function validateMnemonicWords(mnemonic: string): boolean { /** * Build withdrawal credentials from type and ETH address - * @param credType - '01' for execution, '02' for compounding + * @param credType - '01' for execution, '02' for compounding, '03' for builder * @param address - 20-byte ETH address (0x prefixed) */ -export function buildWithdrawalCredentialsFromAddress(credType: '01' | '02', address: string): string { +export function buildWithdrawalCredentialsFromAddress(credType: '01' | '02' | '03', address: string): string { const cleanAddress = address.startsWith('0x') ? address.slice(2) : address; if (cleanAddress.length !== 40) { throw new Error("Invalid address length"); @@ -113,9 +113,9 @@ export async function buildWithdrawalCredentials( return buildBLSWithdrawalCredentials(withdrawalPubkey); } else { if (!config.address) { - throw new Error("Address required for 0x01/0x02 credentials"); + throw new Error("Address required for 0x01/0x02/0x03 credentials"); } - return buildWithdrawalCredentialsFromAddress(config.type, config.address); + return buildWithdrawalCredentialsFromAddress(config.type as '01' | '02' | '03', config.address); } } diff --git a/ui-package/src/components/SubmitDepositsForm/DepositGeneratorModal.tsx b/ui-package/src/components/SubmitDepositsForm/DepositGeneratorModal.tsx index c71180f85..ad2b073e8 100644 --- a/ui-package/src/components/SubmitDepositsForm/DepositGeneratorModal.tsx +++ b/ui-package/src/components/SubmitDepositsForm/DepositGeneratorModal.tsx @@ -30,8 +30,8 @@ interface IValidatorOverrideState { useCustomAmount: boolean; // Credential override fields credentialInputMode: CredentialInputMode; - credentialType: CredentialType; // '00', '01', '02' - withdrawalAddress: string; // For 0x01/0x02 + credentialType: CredentialType; // '00', '01', '02', '03' + withdrawalAddress: string; // For 0x01/0x02/0x03 rawCredentials: string; // For raw mode useCustomCredentials: boolean; } @@ -403,6 +403,7 @@ const DepositGeneratorModal: React.FC = (props) => +
    {credentialType !== '00' && ( @@ -534,8 +535,9 @@ const DepositGeneratorModal: React.FC = (props) => + - {/* Address input (only for 0x01/0x02) */} + {/* Address input (only for 0x01/0x02/0x03) */} {override.credentialType !== '00' && ( Date: Sun, 25 Jan 2026 21:26:03 +0100 Subject: [PATCH 32/99] add highlighting for 0x03 credentials --- utils/format.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/format.go b/utils/format.go index f5a7c7e4c..0bbc88357 100644 --- a/utils/format.go +++ b/utils/format.go @@ -758,6 +758,8 @@ func formatWithdrawalHash(hash []byte) template.HTML { colorClass = "text-success" } else if hash[0] == 0x02 { colorClass = "text-info" + } else if hash[0] == 0x03 { + colorClass = "text-primary" } else { colorClass = "text-warning" } @@ -770,8 +772,8 @@ func FormatWithdawalCredentials(hash []byte) template.HTML { return "INVALID CREDENTIALS" } - // For 0x01 or 0x02 credentials, link to the address - if hash[0] == 0x01 || hash[0] == 0x02 { + // For 0x01, 0x02 or 0x03 credentials, link to the address + if hash[0] == 0x01 || hash[0] == 0x02 || hash[0] == 0x03 { addr := fmt.Sprintf("0x%x", hash[12:]) // Use local link when execution indexer is enabled From 217a4aaa583d48afb351f40cbde8b2e934aa337f Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 25 Jan 2026 23:13:29 +0100 Subject: [PATCH 33/99] show ptc votes on slot details page --- clients/consensus/chainspec.go | 8 +-- handlers/slot.go | 115 ++++++++++++++++++++++++++++++++ indexer/beacon/duties/duties.go | 59 ++++++++++++++++ indexer/beacon/epochstats.go | 38 +++++++++++ templates/slot/ptc_votes.html | 106 +++++++++++++++++++++++++++++ templates/slot/slot.html | 27 ++++++-- types/models/slot.go | 23 +++++++ 7 files changed, 366 insertions(+), 10 deletions(-) create mode 100644 templates/slot/ptc_votes.html diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 27226da10..98fd9ba67 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -231,11 +231,9 @@ type ChainSpecDomainTypes struct { DomainSyncCommitteeSelectionProof phase0.DomainType `yaml:"DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF"` DomainContributionAndProof phase0.DomainType `yaml:"DOMAIN_CONTRIBUTION_AND_PROOF"` DomainBlsToExecutionChange phase0.DomainType `yaml:"DOMAIN_BLS_TO_EXECUTION_CHANGE"` - - // Gloas - DomainBeaconBuilder phase0.DomainType `yaml:"DOMAIN_BEACON_BUILDER" check-if-fork:"GloasForkEpoch"` - DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"GloasForkEpoch"` - DomainProposerPreferences phase0.DomainType `yaml:"DOMAIN_PROPOSER_PREFERENCES" check-if-fork:"GloasForkEpoch"` + DomainBeaconBuilder phase0.DomainType `yaml:"DOMAIN_BEACON_BUILDER" check-if-fork:"GloasForkEpoch"` + DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"GloasForkEpoch"` + DomainProposerPreferences phase0.DomainType `yaml:"DOMAIN_PROPOSER_PREFERENCES" check-if-fork:"GloasForkEpoch"` } type ChainSpec struct { diff --git a/handlers/slot.go b/handlers/slot.go index bb3adea51..49fe4947d 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -46,6 +46,7 @@ func Slot(w http.ResponseWriter, r *http.Request) { "slot/withdrawal_requests.html", "slot/consolidation_requests.html", "slot/bids.html", + "slot/ptc_votes.html", ) var notfoundTemplateFiles = append(layoutTemplateFiles, "slot/notfound.html", @@ -915,6 +916,7 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV // Load execution payload bids for ePBS (gloas+) blocks if blockData.Block.Version >= spec.DataVersionGloas { getSlotPageBids(pageData) + getSlotPagePtcVotes(pageData, blockData, blockData.Header.Message.Slot) } return pageData @@ -1172,6 +1174,119 @@ func getSlotPageBids(pageData *models.SlotPageBlockData) { pageData.BidsCount = uint64(len(pageData.Bids)) } +// getSlotPagePtcVotes extracts PTC (Payload Timeliness Committee) votes from a Gloas block. +// PTC votes are included in blocks as payload attestations for the PREVIOUS slot. +func getSlotPagePtcVotes(pageData *models.SlotPageBlockData, blockData *services.CombinedBlockResponse, blockSlot phase0.Slot) { + // Only Gloas+ blocks have payload attestations + if blockData.Block.Version < spec.DataVersionGloas || blockData.Block.Gloas == nil { + return + } + + payloadAttestations := blockData.Block.Gloas.Message.Body.PayloadAttestations + if len(payloadAttestations) == 0 { + return + } + + chainState := services.GlobalBeaconService.GetChainState() + specs := chainState.GetSpecs() + + // PTC votes are for the previous slot + votedSlot := blockSlot - 1 + votedEpoch := chainState.EpochOfSlot(votedSlot) + + // Get epoch stats for the voted slot to retrieve PTC duties + var ptcDuties []phase0.ValidatorIndex + beaconIndexer := services.GlobalBeaconService.GetBeaconIndexer() + epochStats := beaconIndexer.GetEpochStatsByEpoch(votedEpoch) + for _, es := range epochStats { + values := es.GetValues(true) + if values != nil && values.PtcDuties != nil { + slotInEpoch := uint64(votedSlot) % specs.SlotsPerEpoch + if slotInEpoch < uint64(len(values.PtcDuties)) && values.PtcDuties[slotInEpoch] != nil { + // Convert from active indice indices to validator indices + ptcDuties = make([]phase0.ValidatorIndex, len(values.PtcDuties[slotInEpoch])) + for i, activeIdx := range values.PtcDuties[slotInEpoch] { + if int(activeIdx) < len(values.ActiveIndices) { + ptcDuties[i] = values.ActiveIndices[activeIdx] + } + } + break + } + } + } + + // Build PTC votes structure + ptcVotes := &models.SlotPagePtcVotes{ + VotedSlot: uint64(votedSlot), + TotalPtcSize: specs.PtcSize, + Aggregates: make([]*models.SlotPagePtcAggregate, 0, len(payloadAttestations)), + } + + // Track participating validators across all aggregates + participatingValidators := make(map[uint64]bool) + totalVotes := uint64(0) + + for _, pa := range payloadAttestations { + if pa == nil || pa.Data == nil { + continue + } + + // Set voted block root from first attestation + if ptcVotes.VotedBlockRoot == nil { + ptcVotes.VotedBlockRoot = pa.Data.BeaconBlockRoot[:] + } + + aggregate := &models.SlotPagePtcAggregate{ + PayloadPresent: pa.Data.PayloadPresent, + BlobDataAvailable: pa.Data.BlobDataAvailable, + AggregationBits: pa.AggregationBits, + Signature: pa.Signature[:], + Validators: make([]uint64, 0), + } + + // Map aggregation bits to validator indices + if len(ptcDuties) > 0 { + for i := 0; i < len(ptcDuties) && i < len(pa.AggregationBits)*8; i++ { + byteIdx := i / 8 + bitIdx := i % 8 + if byteIdx < len(pa.AggregationBits) && (pa.AggregationBits[byteIdx]>>bitIdx)&1 == 1 { + validatorIdx := uint64(ptcDuties[i]) + aggregate.Validators = append(aggregate.Validators, validatorIdx) + participatingValidators[validatorIdx] = true + } + } + } + + aggregate.VoteCount = uint64(len(aggregate.Validators)) + totalVotes += aggregate.VoteCount + + ptcVotes.Aggregates = append(ptcVotes.Aggregates, aggregate) + } + + // Build PTC committee list + ptcVotes.PtcCommittee = make([]types.NamedValidator, len(ptcDuties)) + for i, vidx := range ptcDuties { + ptcVotes.PtcCommittee[i] = types.NamedValidator{ + Index: uint64(vidx), + Name: services.GlobalBeaconService.GetValidatorName(uint64(vidx)), + } + + // Add to validator names map + if pageData.ValidatorNames == nil { + pageData.ValidatorNames = make(map[uint64]string) + } + pageData.ValidatorNames[uint64(vidx)] = ptcVotes.PtcCommittee[i].Name + } + + // Calculate participation rate + if specs.PtcSize > 0 { + ptcVotes.Participation = float64(len(participatingValidators)) / float64(specs.PtcSize) + } + + pageData.PtcVotes = ptcVotes + pageData.PtcVotesCount = totalVotes +} + func handleSlotDownload(ctx context.Context, w http.ResponseWriter, blockSlot int64, blockRoot []byte, downloadType string) error { chainState := services.GlobalBeaconService.GetChainState() currentSlot := chainState.CurrentSlot() diff --git a/indexer/beacon/duties/duties.go b/indexer/beacon/duties/duties.go index 9a6ee6402..dd9647edc 100644 --- a/indexer/beacon/duties/duties.go +++ b/indexer/beacon/duties/duties.go @@ -357,3 +357,62 @@ func swapOrNot(buf []byte, byteV byte, i ActiveIndiceIndex, input []ActiveIndice } return byteV, source } + +// GetPtcDuties returns the Payload Timeliness Committee (PTC) members for a given slot. +// The PTC is selected from the concatenated attestation committees for the slot using +// balance-weighted selection without shuffling. +func GetPtcDuties( + spec *consensus.ChainSpec, + state *BeaconState, + attesterDuties [][]ActiveIndiceIndex, + slot phase0.Slot, +) ([]ActiveIndiceIndex, error) { + if spec.PtcSize == 0 { + return nil, nil + } + + epoch := phase0.Epoch(slot / phase0.Slot(spec.SlotsPerEpoch)) + + // Derive PTC seed: hash(get_seed(state, epoch, DOMAIN_PTC_ATTESTER) + uint_to_bytes(slot)) + seedData := []byte{} + seedHash := GetSeed(spec, state, epoch, spec.DomainPtcAttester) + seedData = append(seedData, seedHash[:]...) + seedData = append(seedData, UintToBytes(uint64(slot))...) + seed := Hash(seedData) + + // Concatenate all committee indices for the slot (in order) + indices := make([]ActiveIndiceIndex, 0) + for _, committee := range attesterDuties { + indices = append(indices, committee...) + } + + if len(indices) == 0 { + return nil, errors.New("empty committee indices") + } + + // Balance-weighted selection without shuffling (shuffle_indices=false) + // Uses same acceptance logic as GetProposerIndex (Electra-style 16-bit random values) + maxRandomValue := uint64(1<<16 - 1) + total := uint64(len(indices)) + selected := make([]ActiveIndiceIndex, 0, spec.PtcSize) + + for i := uint64(0); uint64(len(selected)) < spec.PtcSize; i++ { + // No shuffling - traverse indices in order + nextIndex := i % total + candidateIndex := indices[nextIndex] + + // Balance-weighted acceptance check (same as proposer selection) + b := append(seed[:], UintToBytes(i/16)...) + offset := (i % 16) * 2 + hash := Hash(b) + randomValue := BytesToUint(hash[offset : offset+2]) + + effectiveBal := uint64(state.GetEffectiveBalance(candidateIndex)) + + if effectiveBal*maxRandomValue >= spec.MaxEffectiveBalanceElectra*randomValue { + selected = append(selected, candidateIndex) + } + } + + return selected, nil +} diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index c3f084943..e207c8380 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -51,6 +51,7 @@ type EpochStatsValues struct { ProposerDuties []phase0.ValidatorIndex AttesterDuties [][][]duties.ActiveIndiceIndex SyncCommitteeDuties []phase0.ValidatorIndex + PtcDuties [][]duties.ActiveIndiceIndex // [slot_index][ptc_member_index] - PTC duties for Gloas+ epochs ActiveValidators uint64 TotalBalance phase0.Gwei ActiveBalance phase0.Gwei @@ -273,6 +274,17 @@ func (es *EpochStats) parsePackedSSZ(chainState *consensus.ChainState, ssz []byt // compute committees attesterDuties, _ := duties.GetAttesterDuties(chainState.GetSpecs(), beaconState, es.epoch) values.AttesterDuties = attesterDuties + + // compute PTC duties (Gloas+ only) + if chainState.IsEip7732Enabled(es.epoch) && attesterDuties != nil { + ptcDuties := make([][]duties.ActiveIndiceIndex, chainState.GetSpecs().SlotsPerEpoch) + for slotIndex := uint64(0); slotIndex < chainState.GetSpecs().SlotsPerEpoch; slotIndex++ { + slot := chainState.EpochToSlot(es.epoch) + phase0.Slot(slotIndex) + ptc, _ := duties.GetPtcDuties(chainState.GetSpecs(), beaconState, attesterDuties[slotIndex], slot) + ptcDuties[slotIndex] = ptc + } + values.PtcDuties = ptcDuties + } } return values, nil @@ -291,6 +303,7 @@ func (es *EpochStats) pruneValues() { ProposerDuties: es.values.ProposerDuties, AttesterDuties: nil, // prune SyncCommitteeDuties: es.values.SyncCommitteeDuties, + PtcDuties: nil, // prune - only needed for recent epochs ActiveValidators: es.values.ActiveValidators, TotalBalance: es.values.TotalBalance, ActiveBalance: es.values.ActiveBalance, @@ -450,6 +463,20 @@ func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Vali } values.AttesterDuties = attesterDuties + // compute PTC duties (Gloas+ only) + if chainState.IsEip7732Enabled(es.epoch) && attesterDuties != nil { + ptcDuties := make([][]duties.ActiveIndiceIndex, chainState.GetSpecs().SlotsPerEpoch) + for slotIndex := uint64(0); slotIndex < chainState.GetSpecs().SlotsPerEpoch; slotIndex++ { + slot := chainState.EpochToSlot(es.epoch) + phase0.Slot(slotIndex) + ptc, ptcErr := duties.GetPtcDuties(chainState.GetSpecs(), beaconState, attesterDuties[slotIndex], slot) + if ptcErr != nil { + indexer.logger.Warnf("failed computing PTC duties for slot %v: %v", slot, ptcErr) + } + ptcDuties[slotIndex] = ptc + } + values.PtcDuties = ptcDuties + } + if beaconState.RandaoMix != nil { values.RandaoMix = *beaconState.RandaoMix values.NextRandaoMix = *beaconState.NextRandaoMix @@ -571,6 +598,17 @@ func (es *EpochStats) precomputeFromParentState(indexer *Indexer, parentState *E attesterDuties, _ := duties.GetAttesterDuties(chainState.GetSpecs(), beaconState, es.epoch) values.AttesterDuties = attesterDuties + // compute PTC duties (Gloas+ only) + if chainState.IsEip7732Enabled(es.epoch) && attesterDuties != nil { + ptcDuties := make([][]duties.ActiveIndiceIndex, chainState.GetSpecs().SlotsPerEpoch) + for slotIndex := uint64(0); slotIndex < chainState.GetSpecs().SlotsPerEpoch; slotIndex++ { + slot := chainState.EpochToSlot(es.epoch) + phase0.Slot(slotIndex) + ptc, _ := duties.GetPtcDuties(chainState.GetSpecs(), beaconState, attesterDuties[slotIndex], slot) + ptcDuties[slotIndex] = ptc + } + values.PtcDuties = ptcDuties + } + es.precalcValues = values indexer.logger.Infof( diff --git a/templates/slot/ptc_votes.html b/templates/slot/ptc_votes.html new file mode 100644 index 000000000..6f3d13f5e --- /dev/null +++ b/templates/slot/ptc_votes.html @@ -0,0 +1,106 @@ +{{ define "block_ptc_votes" }} +
    + {{ if .Block.PtcVotes }} +
    + + PTC (Payload Timeliness Committee) votes included in this block are for + slot {{ .Block.PtcVotes.VotedSlot }} (the previous slot). + {{ if .Block.PtcVotes.VotedBlockRoot }} +
    Voted block root: 0x{{ printf "%x" .Block.PtcVotes.VotedBlockRoot }} + {{ end }} +
    + +
    +
    +
    +
    +
    {{ formatParticipation .Block.PtcVotes.Participation }}
    + Participation +
    +
    +
    +
    +
    +
    +
    {{ len .Block.PtcVotes.Aggregates }}
    + Aggregates +
    +
    +
    +
    +
    +
    +
    {{ .Block.PtcVotes.TotalPtcSize }}
    + Committee Size +
    +
    +
    +
    + +
    Vote Aggregates
    + {{ range $i, $agg := .Block.PtcVotes.Aggregates }} +
    +
    +
    + + {{ if and $agg.PayloadPresent $agg.BlobDataAvailable }} + Payload + Blob Available + {{ else if $agg.PayloadPresent }} + Payload Only + {{ else if $agg.BlobDataAvailable }} + Blob Only + {{ else }} + Unavailable + {{ end }} + + + {{ $agg.VoteCount }} vote{{ if ne $agg.VoteCount 1 }}s{{ end }} + +
    +
    + + PayloadPresent: {{ if $agg.PayloadPresent }}Yes{{ else }}No{{ end }} + + + BlobDataAvailable: {{ if $agg.BlobDataAvailable }}Yes{{ else }}No{{ end }} + +
    +
    +
    + {{ if gt (len $agg.Validators) 0 }} +
    + Participating Validators ({{ len $agg.Validators }}): +
    +
    + {{ range $j, $vidx := $agg.Validators }} + + {{ formatValidator $vidx (index $.Block.ValidatorNames $vidx) }} + + {{ end }} +
    + {{ else }} +

    No validators in this aggregate

    + {{ end }} +
    +
    + {{ end }} + + {{ if gt (len .Block.PtcVotes.PtcCommittee) 0 }} +
    Full PTC Committee
    +
    +
    +
    + {{ range $i, $member := .Block.PtcVotes.PtcCommittee }} + + {{ formatValidator $member.Index $member.Name }} + + {{ end }} +
    +
    +
    + {{ end }} + {{ else }} +

    No PTC vote data available.

    + {{ end }} +
    +{{ end }} diff --git a/templates/slot/slot.html b/templates/slot/slot.html index 0b4762b5d..be05b617f 100644 --- a/templates/slot/slot.html +++ b/templates/slot/slot.html @@ -43,9 +43,19 @@

    Transactions {{ .Block.TransactionsCount }} {{ end }} + {{ if gt .Block.BidsCount 0 }} +
  • + Bids {{ .Block.BidsCount }} +
    • + {{ end }}
  • Attestations {{ .Block.AttestationsCount }}
    • + {{ if gt .Block.PtcVotesCount 0 }} +
  • + PTC Votes {{ .Block.PtcVotesCount }} +
    • + {{ end }} {{ if gt .Block.DepositsCount 0 }}
  • Deposits {{ .Block.DepositsCount }} @@ -96,11 +106,6 @@

    Consolidation Requests {{ .Block.ConsolidationRequestsCount }}

    • {{ end }} - {{ if gt .Block.BidsCount 0 }} -
  • - Bids {{ .Block.BidsCount }} -
    • - {{ end }} {{ if .Block }}
  • @@ -275,6 +280,18 @@

    Showing {{ .Block.BidsCount }} Execution

    • {{ end }} + {{ if gt .Block.PtcVotesCount 0 }} +
    +
    +
    +
    +

    Showing {{ .Block.PtcVotesCount }} PTC Votes (for slot {{ .Block.PtcVotes.VotedSlot }})

    +
    +
    + {{ template "block_ptc_votes" . }} +
    +
    + {{ end }} {{ if .Block }}
    diff --git a/types/models/slot.go b/types/models/slot.go index 39a0d55a8..817373b12 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -78,6 +78,7 @@ type SlotPageBlockData struct { WithdrawalRequestsCount uint64 `json:"withdrawal_requests_count"` ConsolidationRequestsCount uint64 `json:"consolidation_requests_count"` BidsCount uint64 `json:"bids_count"` + PtcVotesCount uint64 `json:"ptc_votes_count"` PayloadHeader *SlotPagePayloadHeader `json:"payload_header"` ExecutionData *SlotPageExecutionData `json:"execution_data"` @@ -95,6 +96,7 @@ type SlotPageBlockData struct { WithdrawalRequests []*SlotPageWithdrawalRequest `json:"withdrawal_requests"` // WithdrawalRequests included in this block ConsolidationRequests []*SlotPageConsolidationRequest `json:"consolidation_requests"` // ConsolidationRequests included in this block Bids []*SlotPageBid `json:"bids"` // Execution payload bids for this block (ePBS) + PtcVotes *SlotPagePtcVotes `json:"ptc_votes"` // PTC votes included in this block (for previous slot) } type SlotPageExecutionData struct { @@ -312,3 +314,24 @@ type SlotPageBid struct { TotalValue uint64 `json:"total_value"` IsWinning bool `json:"is_winning"` } + +// SlotPagePtcVotes holds PTC (Payload Timeliness Committee) vote information for a slot. +// These are payload attestations included in this block for the PREVIOUS slot. +type SlotPagePtcVotes struct { + VotedSlot uint64 `json:"voted_slot"` // The slot the votes are for (previous slot) + VotedBlockRoot []byte `json:"voted_block_root"` // The block root being voted on + TotalPtcSize uint64 `json:"total_ptc_size"` // Total PTC committee size + Aggregates []*SlotPagePtcAggregate `json:"aggregates"` // Up to 4 aggregates for different vote flag combinations + PtcCommittee []types.NamedValidator `json:"ptc_committee"` // Full PTC committee with participation status + Participation float64 `json:"participation"` // Overall participation rate +} + +// SlotPagePtcAggregate represents a single PTC vote aggregate for a specific vote flag combination. +type SlotPagePtcAggregate struct { + PayloadPresent bool `json:"payload_present"` // Whether the payload was present + BlobDataAvailable bool `json:"blob_data_available"` // Whether blob data was available + AggregationBits []byte `json:"aggregation_bits"` // Bitfield of participating validators + Validators []uint64 `json:"validators"` // Validator indices that voted + Signature []byte `json:"signature"` // Aggregate signature + VoteCount uint64 `json:"vote_count"` // Number of votes in this aggregate +} From f558128a0d83f947072f44f4778d4889a12ee212 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 30 Jan 2026 03:45:37 +0100 Subject: [PATCH 34/99] rework state loading strategy for fuku+ states (load first block state instead of dependent block state) --- clients/consensus/chainstate.go | 8 ++++ indexer/beacon/block.go | 4 ++ indexer/beacon/block_helper.go | 79 ++++++++++++++++++++++++++++----- indexer/beacon/client.go | 56 ++++++++++++----------- indexer/beacon/epochcache.go | 54 +++++++++++----------- indexer/beacon/epochstate.go | 58 +++++++++++++++++++----- indexer/beacon/epochstats.go | 8 ++-- indexer/beacon/finalization.go | 6 ++- indexer/beacon/indexer.go | 5 ++- indexer/beacon/precalc.go | 2 +- indexer/beacon/pruning.go | 2 +- indexer/beacon/requests.go | 8 +++- indexer/beacon/synchronizer.go | 4 +- 13 files changed, 208 insertions(+), 86 deletions(-) diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index a8bc252a1..cfa3d08ef 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -480,6 +480,14 @@ func (cs *ChainState) IsEip7732Enabled(epoch phase0.Epoch) bool { return cs.specs.GloasForkEpoch != nil && phase0.Epoch(*cs.specs.GloasForkEpoch) <= epoch } +func (cs *ChainState) IsFuluEnabled(epoch phase0.Epoch) bool { + if cs.specs == nil { + return false + } + + return cs.specs.FuluForkEpoch != nil && phase0.Epoch(*cs.specs.FuluForkEpoch) <= epoch +} + func (cs *ChainState) GetBalanceChurnLimit(totalActiveBalance uint64) uint64 { if cs.specs == nil { return 0 diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 4a7a5d21f..64f5192d9 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -185,6 +185,10 @@ func (block *Block) AwaitBlock(ctx context.Context, timeout time.Duration) *spec return nil } + if block.block != nil { + return block.block + } + if ctx == nil { ctx = context.Background() } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index bafec4128..c24d8d212 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -515,19 +515,19 @@ func getStatePendingDeposits(v *spec.VersionedBeaconState) ([]*electra.PendingDe case spec.DataVersionDeneb: return nil, errors.New("no pending deposits in deneb") case spec.DataVersionElectra: - if v.Electra == nil || v.Electra.PendingDeposits == nil { + if v.Electra == nil { return nil, errors.New("no electra block") } return v.Electra.PendingDeposits, nil case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.PendingDeposits == nil { + if v.Fulu == nil { return nil, errors.New("no fulu block") } return v.Fulu.PendingDeposits, nil case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.PendingDeposits == nil { + if v.Gloas == nil { return nil, errors.New("no gloas block") } @@ -551,19 +551,19 @@ func getStatePendingWithdrawals(v *spec.VersionedBeaconState) ([]*electra.Pendin case spec.DataVersionDeneb: return nil, errors.New("no pending withdrawals in deneb") case spec.DataVersionElectra: - if v.Electra == nil || v.Electra.PendingPartialWithdrawals == nil { + if v.Electra == nil { return nil, errors.New("no electra block") } return v.Electra.PendingPartialWithdrawals, nil case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.PendingPartialWithdrawals == nil { + if v.Fulu == nil { return nil, errors.New("no fulu block") } return v.Fulu.PendingPartialWithdrawals, nil case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.PendingPartialWithdrawals == nil { + if v.Gloas == nil { return nil, errors.New("no gloas block") } @@ -587,19 +587,19 @@ func getStatePendingConsolidations(v *spec.VersionedBeaconState) ([]*electra.Pen case spec.DataVersionDeneb: return nil, errors.New("no pending consolidations in deneb") case spec.DataVersionElectra: - if v.Electra == nil || v.Electra.PendingConsolidations == nil { + if v.Electra == nil { return nil, errors.New("no electra block") } return v.Electra.PendingConsolidations, nil case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.PendingConsolidations == nil { + if v.Fulu == nil { return nil, errors.New("no fulu block") } return v.Fulu.PendingConsolidations, nil case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.PendingConsolidations == nil { + if v.Gloas == nil { return nil, errors.New("no gloas block") } @@ -625,13 +625,13 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator case spec.DataVersionElectra: return nil, errors.New("no proposer lookahead in electra") case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.ProposerLookahead == nil { + if v.Fulu == nil { return nil, errors.New("no fulu block") } return v.Fulu.ProposerLookahead, nil case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.ProposerLookahead == nil { + if v.Gloas == nil { return nil, errors.New("no gloas block") } @@ -641,6 +641,63 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator } } +// getStateProposerLookahead returns the proposer lookahead from a versioned beacon state. +func getStateBlockRoots(v *spec.VersionedBeaconState) ([]phase0.Root, error) { + switch v.Version { + + case spec.DataVersionPhase0: + if v.Phase0 == nil || v.Phase0.BlockRoots == nil { + return nil, errors.New("no phase0 block") + } + + return v.Phase0.BlockRoots, nil + case spec.DataVersionAltair: + if v.Altair == nil || v.Altair.BlockRoots == nil { + return nil, errors.New("no altair block") + } + + return v.Altair.BlockRoots, nil + case spec.DataVersionBellatrix: + if v.Bellatrix == nil || v.Bellatrix.BlockRoots == nil { + return nil, errors.New("no bellatrix block") + } + + return v.Bellatrix.BlockRoots, nil + case spec.DataVersionCapella: + if v.Capella == nil || v.Capella.BlockRoots == nil { + return nil, errors.New("no capella block") + } + + return v.Capella.BlockRoots, nil + case spec.DataVersionDeneb: + if v.Deneb == nil || v.Deneb.BlockRoots == nil { + return nil, errors.New("no deneb block") + } + + return v.Deneb.BlockRoots, nil + case spec.DataVersionElectra: + if v.Electra == nil || v.Electra.BlockRoots == nil { + return nil, errors.New("no electra block") + } + + return v.Electra.BlockRoots, nil + case spec.DataVersionFulu: + if v.Fulu == nil || v.Fulu.BlockRoots == nil { + return nil, errors.New("no fulu block") + } + + return v.Fulu.BlockRoots, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.BlockRoots == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.BlockRoots, nil + default: + return nil, errors.New("unknown version") + } +} + // getBlockSize returns the block size from a versioned beacon block. func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) (int, error) { switch block.Version { diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 895e8c6d8..2d26df48a 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -254,50 +254,54 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { chainState := c.client.GetPool().GetChainState() dependentRoot := headEvent.CurrentDutyDependentRoot - - var dependentBlock *Block if !bytes.Equal(dependentRoot[:], consensus.NullRoot[:]) { block.dependentRoot = &dependentRoot - - dependentBlock = c.indexer.blockCache.getBlockByRoot(dependentRoot) - if dependentBlock == nil { - c.logger.Warnf("dependent block (%v) not found after backfilling", dependentRoot.String()) - } - } else { - dependentBlock = c.indexer.blockCache.getDependentBlock(chainState, block, c) } // walk back the chain of epoch stats to ensure we have all duties & epoch specific data for the clients chain currentBlock := block - currentEpoch := chainState.EpochOfSlot(currentBlock.Slot) + headEpoch := chainState.EpochOfSlot(currentBlock.Slot) + currentEpoch := headEpoch minInMemorySlot := c.indexer.getMinInMemorySlot() absoluteMinInMemoryEpoch := c.indexer.getAbsoluteMinInMemoryEpoch() for { - if dependentBlock != nil && currentBlock.Slot >= minInMemorySlot { - epoch := chainState.EpochOfSlot(currentBlock.Slot) + parentRoot := currentBlock.GetParentRoot() + if parentRoot == nil { + break + } - // only request state for epochs that are allowed in memory by configuration - // we accept some gaps here, these will be fixed by the pruning/finalization process - requestState := epoch >= absoluteMinInMemoryEpoch + isEpochStart := false + parentBlock := c.indexer.blockCache.getBlockByRoot(*parentRoot) + + if currentBlock.Slot == 0 { + isEpochStart = true + } else if currentBlock.dependentRoot != nil && *parentRoot == *currentBlock.dependentRoot && (parentBlock == nil || parentBlock.Slot > 0) { + isEpochStart = true + } else if parentBlock != nil && chainState.EpochOfSlot(parentBlock.Slot) < currentEpoch { + isEpochStart = true + } + + if isEpochStart { + epoch := chainState.EpochOfSlot(currentBlock.Slot) + dependentRoot := *parentRoot // ensure epoch stats for the epoch - epochStats := c.indexer.epochCache.createOrGetEpochStats(epoch, dependentBlock.Root, requestState) + epochStats := c.indexer.epochCache.createOrGetEpochStats(epoch, dependentRoot) + + if epoch >= absoluteMinInMemoryEpoch { + c.indexer.epochCache.ensureEpochDependentState(epochStats, currentBlock.Root) + } if !epochStats.addRequestedBy(c) { break } - if epochStats.dependentState == nil && epoch == currentEpoch { - // always load most recent dependent state to ensure we have the latest validator set - c.indexer.epochCache.addEpochStateRequest(epochStats) - } - } else { - if dependentBlock == nil { - c.logger.Debugf("epoch stats check failed: dependent block for %v:%v (%v) not found", currentBlock.Slot, chainState.EpochOfSlot(currentBlock.Slot), currentBlock.Root.String()) - } + } + + if parentBlock == nil || parentBlock.Slot < minInMemorySlot { break } - currentBlock = dependentBlock - dependentBlock = c.indexer.blockCache.getDependentBlock(chainState, currentBlock, c) + currentBlock = parentBlock + currentEpoch = chainState.EpochOfSlot(currentBlock.Slot) } c.headRoot = block.Root diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index ce62a81b9..b71d7201d 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -65,7 +65,7 @@ func newEpochCache(indexer *Indexer) *epochCache { } // createOrGetEpochStats gets an existing EpochStats entry for the given epoch and dependentRoot or creates a new instance if not found. -func (cache *epochCache) createOrGetEpochStats(epoch phase0.Epoch, dependentRoot phase0.Root, createStateRequest bool) *EpochStats { +func (cache *epochCache) createOrGetEpochStats(epoch phase0.Epoch, dependentRoot phase0.Root) *EpochStats { cache.cacheMutex.Lock() defer cache.cacheMutex.Unlock() @@ -77,43 +77,40 @@ func (cache *epochCache) createOrGetEpochStats(epoch phase0.Epoch, dependentRoot cache.statsMap[statsKey] = epochStats } - // get or create beacon state which the epoch status depends on (dependentRoot beacon state) - epochState := cache.stateMap[dependentRoot] - if epochState == nil && !epochStats.ready && createStateRequest { - epochState = newEpochState(dependentRoot) - cache.stateMap[dependentRoot] = epochState - - cache.indexer.logger.Infof("added epoch state request for epoch %v (%v) to queue", epoch, dependentRoot.String()) - } - - if epochState != nil { - epochStats.dependentState = epochState - - if epochState.loadingStatus == 2 && !epochStats.ready { - // dependent state is already loaded, process it - go epochStats.processState(cache.indexer, nil) - } - } - return epochStats } -func (cache *epochCache) addEpochStateRequest(epochStats *EpochStats) { +func (cache *epochCache) ensureEpochDependentState(epochStats *EpochStats, firstBlockRoot phase0.Root) { + cache.cacheMutex.Lock() + defer cache.cacheMutex.Unlock() + if epochStats.dependentState != nil { return } - cache.cacheMutex.Lock() - defer cache.cacheMutex.Unlock() - + // get or create beacon state which the epoch status depends on (dependentRoot beacon state) epochState := cache.stateMap[epochStats.dependentRoot] - if epochState == nil { - epochState = newEpochState(epochStats.dependentRoot) + if epochState == nil && !epochStats.ready { + stateRoot := epochStats.dependentRoot + chainState := cache.indexer.consensusPool.GetChainState() + if chainState.IsFuluEnabled(epochStats.epoch) { + stateRoot = firstBlockRoot + } + + epochState = newEpochState(stateRoot) cache.stateMap[epochStats.dependentRoot] = epochState cache.indexer.logger.Infof("added epoch state request for epoch %v (%v) to queue", epochStats.epoch, epochStats.dependentRoot.String()) } - epochStats.dependentState = epochState + + if epochState != nil { + epochStats.dependentState = epochState + + if epochState.loadingStatus == 2 && !epochStats.ready { + // dependent state is already loaded, process it + go epochStats.processState(cache.indexer, nil, 0) + } + } } func (cache *epochCache) getEpochStats(epoch phase0.Epoch, dependentRoot phase0.Root) *EpochStats { @@ -468,11 +465,14 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { log.Infof("loading epoch %v stats (dep: %v, req: %v)", epochStats.epoch, epochStats.dependentRoot.String(), len(epochStats.requestedBy)) + t1 := time.Now() state, err := epochStats.dependentState.loadState(client.getContext(), client, cache) if err != nil && epochStats.dependentState.loadingStatus == 0 { client.logger.Warnf("failed loading epoch %v stats (dep: %v): %v", epochStats.epoch, epochStats.dependentRoot.String(), err) } + loadDuration := time.Since(t1) + if epochStats.dependentState.loadingStatus != 2 { // epoch state could not be loaded epochStats.dependentState.retryCount++ @@ -497,7 +497,7 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { cache.cacheMutex.Unlock() for _, stats := range dependentStats { - go stats.processState(cache.indexer, validatorSet) + go stats.processState(cache.indexer, validatorSet, loadDuration) } return true diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index db65ec134..f6878847f 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -91,7 +91,6 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } s.loadingStatus = 1 - client.logger.Debugf("loading state for slot %v", s.slotRoot.String()) ctx, cancel := context.WithTimeout(ctx, beaconStateRequestTimeout+(beaconHeaderRequestTimeout*2)) s.loadingCancel = cancel @@ -105,29 +104,38 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } }() - var blockHeader *phase0.SignedBeaconBlockHeader + var beaconBlock *spec.VersionedSignedBeaconBlock block := client.indexer.blockCache.getBlockByRoot(s.slotRoot) if block != nil { - blockHeader = block.AwaitHeader(ctx, beaconHeaderRequestTimeout) + beaconBlock = block.AwaitBlock(ctx, beaconHeaderRequestTimeout) } - if blockHeader == nil { + if beaconBlock == nil { var err error - blockHeader, err = LoadBeaconHeader(ctx, client, s.slotRoot) + beaconBlock, err = LoadBeaconBlock(ctx, client, s.slotRoot) if err != nil { return nil, err } } - s.stateRoot = blockHeader.Message.StateRoot + if beaconBlock != nil { + slot, _ := beaconBlock.Slot() + client.logger.Infof("loading state for block root %v (slot %v)", s.slotRoot.String(), slot) - resState, err := LoadBeaconState(ctx, client, blockHeader.Message.StateRoot) + var err error + s.stateRoot, err = beaconBlock.StateRoot() + if err != nil { + return nil, fmt.Errorf("error getting state root from beacon block %v: %v", s.slotRoot.String(), err) + } + } + + resState, err := LoadBeaconState(ctx, client, s.stateRoot) if err != nil { return nil, err } - err = s.processState(resState, cache) + err = s.processState(resState, beaconBlock, cache) if err != nil { return nil, err } @@ -145,7 +153,7 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch // processState processes the state and updates the epochState instance. // the function extracts and unifies all relevant information from the beacon state, so the full beacon state can be dropped from memory afterwards. -func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epochCache) error { +func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, cache *epochCache) error { slot, err := state.Slot() if err != nil { return fmt.Errorf("error getting slot from state %v: %v", s.slotRoot.String(), err) @@ -153,19 +161,30 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epoch s.stateSlot = slot + dependentRoot := s.slotRoot + if state.Version >= spec.DataVersionFulu { + blockRoots, err := getStateBlockRoots(state) + if err != nil { + return fmt.Errorf("error getting block roots from state %v: %v", s.slotRoot.String(), err) + } + + specs := cache.indexer.consensusPool.GetChainState().GetSpecs() + dependentRoot = blockRoots[slot%phase0.Slot(specs.SlotsPerHistoricalRoot)] + } + validatorList, err := state.Validators() if err != nil { return fmt.Errorf("error getting validators from state %v: %v", s.slotRoot.String(), err) } if cache != nil { - cache.indexer.validatorCache.updateValidatorSet(slot, s.slotRoot, validatorList) + cache.indexer.validatorCache.updateValidatorSet(slot, dependentRoot, validatorList) } // Process builder set for Gloas if state.Version >= spec.DataVersionGloas && state.Gloas != nil { if cache != nil { - cache.indexer.builderCache.updateBuilderSet(slot, s.slotRoot, state.Gloas.Builders) + cache.indexer.builderCache.updateBuilderSet(slot, dependentRoot, state.Gloas.Builders) } // Extract builder balances @@ -194,7 +213,22 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epoch } s.randaoMixes = randaoMixes - s.depositIndex = getStateDepositIndex(state) + + if state.Version >= spec.DataVersionFulu { + // subtract the deposit indexes from the current block + blockRequests, err := beaconBlock.ExecutionRequests() + if err != nil { + return fmt.Errorf("error getting execution requests from block %v: %v", s.slotRoot.String(), err) + } + + if len(blockRequests.Deposits) > 0 { + s.depositIndex = blockRequests.Deposits[0].Index + } else { + s.depositIndex = getStateDepositIndex(state) + } + } else { + s.depositIndex = getStateDepositIndex(state) + } if state.Version >= spec.DataVersionAltair { currentSyncCommittee, err := getStateCurrentSyncCommittee(state) diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index e207c8380..34288b7a7 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -23,6 +23,7 @@ import ( type EpochStats struct { epoch phase0.Epoch dependentRoot phase0.Root + firstBlockRoot phase0.Root dependentState *epochState requestedMutex sync.Mutex @@ -47,7 +48,7 @@ type EpochStatsValues struct { RandaoMix phase0.Hash32 NextRandaoMix phase0.Hash32 ActiveIndices []phase0.ValidatorIndex - EffectiveBalances []uint32 + EffectiveBalances []uint32 // effective balance in full ETH of last epoch for pre-fulu stats, effective balance in full ETH of current epoch for fulu+ stats ProposerDuties []phase0.ValidatorIndex AttesterDuties [][][]duties.ActiveIndiceIndex SyncCommitteeDuties []phase0.ValidatorIndex @@ -335,7 +336,7 @@ func (es *EpochStats) loadValuesFromDb(chainState *consensus.ChainState) *EpochS } // processState processes the epoch state and computes proposer and attester duties. -func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Validator) { +func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Validator, loadDuration time.Duration) { if es.dependentState == nil || es.dependentState.loadingStatus != 2 { return } @@ -502,12 +503,13 @@ func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Vali es.isInDb = true indexer.logger.Infof( - "processed epoch %v stats (root: %v / state: %v, validators: %v/%v, %v ms), %v bytes", + "processed epoch %v stats (root: %v / state: %v, validators: %v/%v, load: %v ms, process: %v ms), %v bytes", es.epoch, es.dependentRoot.String(), dependentState.stateRoot.String(), values.ActiveValidators, len(validatorSet), + loadDuration.Milliseconds(), time.Since(t1).Milliseconds(), len(packedSsz), ) diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 1a3604531..29db7af13 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -217,6 +217,10 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R } } + if firstBlock.Slot == 0 { + dependentRoot = phase0.Root{} + } + if !isValid { return false, fmt.Errorf("first canonical block %v (%v) is not the first block of epoch %v", firstBlock.Slot, firstBlock.Root.String(), epoch) } @@ -279,7 +283,7 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R // if the state is not yet loaded, we set it to high priority and wait for it to be loaded if !epochStats.ready { if epochStats.dependentState == nil { - indexer.epochCache.addEpochStateRequest(epochStats) + indexer.epochCache.ensureEpochDependentState(epochStats, canonicalBlocks[0].Root) } if epochStats.dependentState != nil && epochStats.dependentState.loadingStatus != 2 && epochStats.dependentState.retryCount < 10 { indexer.logger.Infof("epoch %d state (%v) not yet loaded, waiting for state to be loaded", epoch, dependentRoot.String()) diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 89141c959..c024bec37 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -302,7 +302,7 @@ func (indexer *Indexer) StartIndexer() { processingWaitGroup.Done() }() - epochStats := indexer.epochCache.createOrGetEpochStats(phase0.Epoch(dbDuty.Epoch), phase0.Root(dbDuty.DependentRoot), false) + epochStats := indexer.epochCache.createOrGetEpochStats(phase0.Epoch(dbDuty.Epoch), phase0.Root(dbDuty.DependentRoot)) pruneStats := dbDuty.Epoch < uint64(indexer.lastPrunedEpoch) err := epochStats.restoreFromDb(dbDuty, chainState, !pruneStats) @@ -451,7 +451,8 @@ func (indexer *Indexer) StartIndexer() { if len(genesisBlock) == 0 { indexer.logger.Warnf("genesis block not found in cache") } else { - indexer.epochCache.createOrGetEpochStats(0, genesisBlock[0].Root, true) + epochStats := indexer.epochCache.createOrGetEpochStats(0, genesisBlock[0].Root) + indexer.epochCache.ensureEpochDependentState(epochStats, genesisBlock[0].Root) } } diff --git a/indexer/beacon/precalc.go b/indexer/beacon/precalc.go index 078370eda..ada710034 100644 --- a/indexer/beacon/precalc.go +++ b/indexer/beacon/precalc.go @@ -32,7 +32,7 @@ func (indexer *Indexer) precalcNextEpochStats(epoch phase0.Epoch) error { } // precompute epoch stats for the epoch if we have the parent epoch stats ready - epochStats := indexer.epochCache.createOrGetEpochStats(epoch, dependentBlock.Root, false) + epochStats := indexer.epochCache.createOrGetEpochStats(epoch, dependentBlock.Root) if !epochStats.ready { var parentDependentBlock *Block if chainState.EpochOfSlot(dependentBlock.Slot) == epoch-1 { diff --git a/indexer/beacon/pruning.go b/indexer/beacon/pruning.go index 94896ba78..88f2276b0 100644 --- a/indexer/beacon/pruning.go +++ b/indexer/beacon/pruning.go @@ -118,7 +118,7 @@ func (indexer *Indexer) processEpochPruning(pruneEpoch phase0.Epoch) (uint64, ui // if the state is not yet loaded, we set it to high priority and wait for it to be loaded if epochStats != nil && !epochStats.ready { if epochStats.dependentState == nil { - indexer.epochCache.addEpochStateRequest(epochStats) + indexer.epochCache.ensureEpochDependentState(epochStats, blocks[0].Root) } if epochStats.dependentState != nil && epochStats.dependentState.loadingStatus != 2 && epochStats.dependentState.retryCount < 10 { indexer.logger.Infof("epoch %d state (%v) not yet loaded, waiting for state to be loaded", pruneEpoch, dependentRoot.String()) diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index f8182a79c..c5058014f 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -72,7 +72,13 @@ func LoadBeaconState(ctx context.Context, client *Client, root phase0.Root) (*sp ctx, cancel := context.WithTimeout(ctx, beaconStateRequestTimeout) defer cancel() - resState, err := client.client.GetRPCClient().GetState(ctx, fmt.Sprintf("0x%x", root[:])) + stateRef := fmt.Sprintf("0x%x", root[:]) + nullRoot := phase0.Root{} + if root == nullRoot { + stateRef = "genesis" + } + + resState, err := client.client.GetRPCClient().GetState(ctx, stateRef) if err != nil { return nil, err } diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 3336b1717..a8d07b7d0 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -383,7 +383,9 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry } epochState := newEpochState(dependentRoot) + t1 := time.Now() state, err := epochState.loadState(s.syncCtx, client, nil) + loadDuration := time.Since(t1) if (err != nil || epochState.loadingStatus != 2) && !lastTry { return false, fmt.Errorf("error fetching epoch %v state: %v", syncEpoch, err) } @@ -403,7 +405,7 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry if epochState != nil && epochState.loadingStatus == 2 { epochStats = newEpochStats(syncEpoch, dependentRoot) epochStats.dependentState = epochState - epochStats.processState(s.indexer, validatorSet) + epochStats.processState(s.indexer, validatorSet, loadDuration) epochStatsValues = epochStats.GetValues(false) } From 576577cea0ffd7ea29ece7d6338c15ba79af96e9 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 30 Jan 2026 03:56:03 +0100 Subject: [PATCH 35/99] use proposer lookahead for epoch state precalculation --- indexer/beacon/epochstats.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index 34288b7a7..ecfe42654 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -267,7 +267,9 @@ func (es *EpochStats) parsePackedSSZ(chainState *consensus.ChainState, ssz []byt proposerDuties = append(proposerDuties, proposerIndex) } - values.ProposerDuties = proposerDuties + if len(values.ProposerDuties) == 0 { + values.ProposerDuties = proposerDuties + } if beaconState.RandaoMix != nil { values.RandaoMix = *beaconState.RandaoMix } @@ -438,7 +440,7 @@ func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Vali offset = slotsPerEpoch } - values.ProposerDuties = dependentState.proposerLookahead[offset : offset+slotsPerEpoch] + values.ProposerDuties = dependentState.proposerLookahead[offset:] } else { proposerDuties := []phase0.ValidatorIndex{} for slot := chainState.EpochToSlot(es.epoch); slot < chainState.EpochToSlot(es.epoch+1); slot++ { @@ -584,14 +586,20 @@ func (es *EpochStats) precomputeFromParentState(indexer *Indexer, parentState *E // compute proposers proposerDuties := []phase0.ValidatorIndex{} - for slot := chainState.EpochToSlot(es.epoch); slot < chainState.EpochToSlot(es.epoch+1); slot++ { - proposer, err := duties.GetProposerIndex(chainState.GetSpecs(), beaconState, slot) - proposerIndex := phase0.ValidatorIndex(math.MaxInt64) - if err == nil { - proposerIndex = values.ActiveIndices[proposer] - } - proposerDuties = append(proposerDuties, proposerIndex) + specs := chainState.GetSpecs() + if uint64(len(parentState.dependentState.proposerLookahead)) > specs.SlotsPerEpoch { + proposerDuties = parentState.dependentState.proposerLookahead[specs.SlotsPerEpoch:] + } else { + for slot := chainState.EpochToSlot(es.epoch); slot < chainState.EpochToSlot(es.epoch+1); slot++ { + proposer, err := duties.GetProposerIndex(chainState.GetSpecs(), beaconState, slot) + proposerIndex := phase0.ValidatorIndex(math.MaxInt64) + if err == nil { + proposerIndex = values.ActiveIndices[proposer] + } + + proposerDuties = append(proposerDuties, proposerIndex) + } } values.ProposerDuties = proposerDuties From 33ba82fbd95b6fa04f47520d6fad343a42ed7a49 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 30 Jan 2026 04:22:47 +0100 Subject: [PATCH 36/99] subscribe to new topics in separate event stream to avoid compatibility issues --- clients/consensus/clientlogic.go | 2 +- clients/consensus/rpc/beaconstream.go | 157 +++++++++++++++++++------- 2 files changed, 116 insertions(+), 43 deletions(-) diff --git a/clients/consensus/clientlogic.go b/clients/consensus/clientlogic.go index 59b938a62..ff12f3c03 100644 --- a/clients/consensus/clientlogic.go +++ b/clients/consensus/clientlogic.go @@ -137,7 +137,7 @@ func (client *Client) runClientLogic() error { blockStream := client.rpcClient.NewBlockStream( client.clientCtx, client.logger, - rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent|rpc.StreamExecutionPayloadEvent, + rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent|rpc.StreamExecutionPayloadEvent|rpc.StreamExecutionPayloadBidEvent, ) defer blockStream.Close() diff --git a/clients/consensus/rpc/beaconstream.go b/clients/consensus/rpc/beaconstream.go index 8c91b7aef..9f699ac49 100644 --- a/clients/consensus/rpc/beaconstream.go +++ b/clients/consensus/rpc/beaconstream.go @@ -11,7 +11,6 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/donovanhide/eventsource" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/consensus/rpc/eventstream" @@ -74,52 +73,126 @@ func (bs *BeaconStream) startStream() { bs.running = false }() - stream := bs.subscribeStream(bs.client.endpoint, bs.events) - if stream != nil { - defer stream.Close() + // Subscribe to basic events (block, head, finalized_checkpoint) + basicEvents := bs.events & (StreamBlockEvent | StreamHeadEvent | StreamFinalizedEvent) + basicStream := bs.subscribeStream(bs.client.endpoint, basicEvents) + if basicStream != nil { + defer basicStream.Close() + } - for { + // Subscribe to advanced events (execution_payload_available, execution_payload_bid) + // These are in a separate stream because clients may not support them yet, + // and subscribing to unsupported topics can cause the entire subscription to fail. + // Run in a separate goroutine so it doesn't block the basic stream. + advancedEvents := bs.events & (StreamExecutionPayloadEvent | StreamExecutionPayloadBidEvent) + advancedStreamChan := make(chan *eventstream.Stream, 1) + if advancedEvents > 0 { + go func() { + stream := bs.subscribeStream(bs.client.endpoint, advancedEvents) select { + case advancedStreamChan <- stream: case <-bs.ctx.Done(): - return - case evt := <-stream.Events: - switch evt.Event() { - case "block": - bs.processBlockEvent(evt) - case "head": - bs.processHeadEvent(evt) - case "finalized_checkpoint": - bs.processFinalizedEvent(evt) - case "execution_payload_available": - bs.processExecutionPayloadAvailableEvent(evt) - case "execution_payload_bid": - bs.processExecutionPayloadBidEvent(evt) - } - case <-stream.Ready: - bs.ReadyChan <- &BeaconStreamStatus{ - Ready: true, - } - case err := <-stream.Errors: - if strings.Contains(err.Error(), "INTERNAL_ERROR; received from peer") { - // this seems to be a go bug, silently reconnect to the stream - time.Sleep(10 * time.Millisecond) - stream.RetryNow() - } else { - bs.logger.Warnf("beacon block stream error: %v", err) + if stream != nil { + stream.Close() } + } + }() + } - select { - case bs.ReadyChan <- &BeaconStreamStatus{ - Ready: false, - Error: err, - }: - case <-bs.ctx.Done(): - } + var advancedStream *eventstream.Stream + defer func() { + if advancedStream != nil { + advancedStream.Close() + } + }() + + for { + select { + case <-bs.ctx.Done(): + return + + // Basic stream events + case evt := <-basicStream.Events: + switch evt.Event() { + case "block": + bs.processBlockEvent(evt) + case "head": + bs.processHeadEvent(evt) + case "finalized_checkpoint": + bs.processFinalizedEvent(evt) + } + case <-basicStream.Ready: + bs.ReadyChan <- &BeaconStreamStatus{ + Ready: true, } + case err := <-basicStream.Errors: + bs.handleStreamError(basicStream, err) + + // Advanced stream connection established + case stream := <-advancedStreamChan: + advancedStream = stream + + // Advanced stream events (no Ready/Error forwarding) + case evt := <-bs.getAdvancedStreamEvents(advancedStream): + switch evt.Event() { + case "execution_payload_available": + bs.processExecutionPayloadAvailableEvent(evt) + case "execution_payload_bid": + bs.processExecutionPayloadBidEvent(evt) + } + case <-bs.getAdvancedStreamReady(advancedStream): + // Don't forward ready events from advanced stream + case <-bs.getAdvancedStreamErrors(advancedStream): + // Silently retry - clients may not support these events yet + time.Sleep(10 * time.Millisecond) + advancedStream.RetryNow() } } } +// getAdvancedStreamEvents returns the events channel or a nil channel if stream is nil. +func (bs *BeaconStream) getAdvancedStreamEvents(stream *eventstream.Stream) chan eventstream.StreamEvent { + if stream == nil { + return nil + } + return stream.Events +} + +// getAdvancedStreamReady returns the ready channel or a nil channel if stream is nil. +func (bs *BeaconStream) getAdvancedStreamReady(stream *eventstream.Stream) chan bool { + if stream == nil { + return nil + } + return stream.Ready +} + +// getAdvancedStreamErrors returns the errors channel or a nil channel if stream is nil. +func (bs *BeaconStream) getAdvancedStreamErrors(stream *eventstream.Stream) chan error { + if stream == nil { + return nil + } + return stream.Errors +} + +// handleStreamError handles stream errors and forwards them to the ReadyChan. +func (bs *BeaconStream) handleStreamError(stream *eventstream.Stream, err error) { + if strings.Contains(err.Error(), "INTERNAL_ERROR; received from peer") { + // this seems to be a go bug, silently reconnect to the stream + time.Sleep(10 * time.Millisecond) + stream.RetryNow() + } else { + bs.logger.Warnf("beacon block stream error: %v", err) + } + + select { + case bs.ReadyChan <- &BeaconStreamStatus{ + Ready: false, + Error: err, + }: + case <-bs.ctx.Done(): + } +} + func (bs *BeaconStream) subscribeStream(endpoint string, events uint16) *eventstream.Stream { var topics strings.Builder @@ -206,7 +279,7 @@ func (bs *BeaconStream) subscribeStream(endpoint string, events uint16) *eventst } } -func (bs *BeaconStream) processBlockEvent(evt eventsource.Event) { +func (bs *BeaconStream) processBlockEvent(evt eventstream.StreamEvent) { var parsed v1.BlockEvent err := json.Unmarshal([]byte(evt.Data()), &parsed) @@ -221,7 +294,7 @@ func (bs *BeaconStream) processBlockEvent(evt eventsource.Event) { } } -func (bs *BeaconStream) processHeadEvent(evt eventsource.Event) { +func (bs *BeaconStream) processHeadEvent(evt eventstream.StreamEvent) { var parsed v1.HeadEvent err := json.Unmarshal([]byte(evt.Data()), &parsed) @@ -237,7 +310,7 @@ func (bs *BeaconStream) processHeadEvent(evt eventsource.Event) { } } -func (bs *BeaconStream) processFinalizedEvent(evt eventsource.Event) { +func (bs *BeaconStream) processFinalizedEvent(evt eventstream.StreamEvent) { var parsed v1.FinalizedCheckpointEvent err := json.Unmarshal([]byte(evt.Data()), &parsed) @@ -252,7 +325,7 @@ func (bs *BeaconStream) processFinalizedEvent(evt eventsource.Event) { } } -func (bs *BeaconStream) processExecutionPayloadAvailableEvent(evt eventsource.Event) { +func (bs *BeaconStream) processExecutionPayloadAvailableEvent(evt eventstream.StreamEvent) { var parsed v1.ExecutionPayloadAvailableEvent err := json.Unmarshal([]byte(evt.Data()), &parsed) @@ -267,7 +340,7 @@ func (bs *BeaconStream) processExecutionPayloadAvailableEvent(evt eventsource.Ev } } -func (bs *BeaconStream) processExecutionPayloadBidEvent(evt eventsource.Event) { +func (bs *BeaconStream) processExecutionPayloadBidEvent(evt eventstream.StreamEvent) { var parsed gloas.SignedExecutionPayloadBid err := json.Unmarshal([]byte(evt.Data()), &parsed) From b14a70f662878c245e74a2a342adb81a1f277330 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 30 Jan 2026 05:58:09 +0100 Subject: [PATCH 37/99] track builder index in db and improve builder details --- db/block_bids.go | 42 ++++++++++ .../pgsql/20260108202212_epbs-payload.sql | 9 ++- .../sqlite/20260108202212_epbs-payload.sql | 2 + db/slots.go | 27 ++++--- dbtypes/dbtypes.go | 1 + dbtypes/other.go | 1 + handlers/builder.go | 77 +++++++++++++------ indexer/beacon/block.go | 7 ++ indexer/beacon/block_helper.go | 26 +++++++ indexer/beacon/writedb.go | 11 +++ services/chainservice_blocks.go | 17 +++- templates/builder/builder.html | 4 +- utils/format.go | 4 +- 13 files changed, 188 insertions(+), 40 deletions(-) diff --git a/db/block_bids.go b/db/block_bids.go index c2ac142b0..cb1efc69a 100644 --- a/db/block_bids.go +++ b/db/block_bids.go @@ -112,6 +112,48 @@ func DeleteBidsBeforeSlot(minSlot uint64, tx *sqlx.Tx) error { return err } +// GetBidsByBlockHashes returns bids for multiple block hashes and a specific builder index +// Returns a map keyed by block hash (hex string) for easy lookup +func GetBidsByBlockHashes(blockHashes [][]byte, builderIndex uint64) map[string]*dbtypes.BlockBid { + result := make(map[string]*dbtypes.BlockBid, len(blockHashes)) + if len(blockHashes) == 0 { + return result + } + + var sql strings.Builder + args := make([]any, 0, len(blockHashes)+1) + + fmt.Fprint(&sql, ` + SELECT + parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment + FROM block_bids + WHERE builder_index = $1 AND block_hash IN (`) + + args = append(args, builderIndex) + for i, hash := range blockHashes { + if i > 0 { + fmt.Fprint(&sql, ", ") + } + fmt.Fprintf(&sql, "$%d", i+2) + args = append(args, hash) + } + fmt.Fprint(&sql, ")") + + bids := []*dbtypes.BlockBid{} + err := ReaderDb.Select(&bids, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching bids by block hashes: %v", err) + return result + } + + for _, bid := range bids { + key := fmt.Sprintf("%x", bid.BlockHash) + result[key] = bid + } + + return result +} + // GetBidsByBuilderIndex returns bids submitted by a specific builder, ordered by slot descending func GetBidsByBuilderIndex(builderIndex uint64, offset uint64, limit uint32) ([]*dbtypes.BlockBid, uint64) { var sql strings.Builder diff --git a/db/schema/pgsql/20260108202212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql index 10de8df5f..89dd2442a 100644 --- a/db/schema/pgsql/20260108202212_epbs-payload.sql +++ b/db/schema/pgsql/20260108202212_epbs-payload.sql @@ -10,12 +10,17 @@ ALTER TABLE public."orphaned_blocks" ADD "payload_ssz" bytea NULL; ALTER TABLE public."slots" ADD - "payload_status" smallint NOT NULL DEFAULT 0; + "payload_status" smallint NOT NULL DEFAULT 0, + "builder_index" bigint NOT NULL DEFAULT -1; CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" - ON public."slots" + ON public."slots" ("payload_status" ASC NULLS LAST); +CREATE INDEX IF NOT EXISTS "slots_builder_index_idx" + ON public."slots" + ("builder_index" ASC NULLS LAST); + ALTER TABLE public."epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/schema/sqlite/20260108202212_epbs-payload.sql b/db/schema/sqlite/20260108202212_epbs-payload.sql index 4535d7cb6..731fbd07d 100644 --- a/db/schema/sqlite/20260108202212_epbs-payload.sql +++ b/db/schema/sqlite/20260108202212_epbs-payload.sql @@ -8,8 +8,10 @@ ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "slots" ADD "payload_status" smallint NOT NULL DEFAULT 0; +ALTER TABLE "slots" ADD "builder_index" BIGINT NOT NULL DEFAULT -1; CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON "slots" ("payload_status" ASC); +CREATE INDEX IF NOT EXISTS "slots_builder_index_idx" ON "slots" ("builder_index" ASC); ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/slots.go b/db/slots.go index 5f4884c85..eb1fbaf5d 100644 --- a/db/slots.go +++ b/db/slots.go @@ -21,8 +21,8 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35) + exec_times, block_uid, payload_status, builder_index + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, @@ -36,15 +36,15 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35)`, + exec_times, block_uid, payload_status, builder_index + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.BlobCount, slot.EthGasUsed, slot.EthGasLimit, slot.EthBaseFee, slot.EthFeeRecipient, slot.BlockSize, slot.RecvDelay, slot.MinExecTime, slot.MaxExecTime, - slot.ExecTimes, slot.BlockUid, slot.PayloadStatus) + slot.ExecTimes, slot.BlockUid, slot.PayloadStatus, slot.BuilderIndex) if err != nil { return err } @@ -101,7 +101,7 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", "eth_gas_used", "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", "max_exec_time", - "exec_times", "block_uid", "payload_status", + "exec_times", "block_uid", "payload_status", "builder_index", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -135,7 +135,7 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status + exec_times, block_uid, payload_status, builder_index FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -156,7 +156,7 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status + exec_times, block_uid, payload_status, builder_index FROM slots WHERE root = $1 `, root) @@ -184,7 +184,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status + exec_times, block_uid, payload_status, builder_index FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -260,7 +260,7 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status + exec_times, block_uid, payload_status, builder_index FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -322,7 +322,7 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", "eth_gas_used", "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", "max_exec_time", - "exec_times", "block_uid", "payload_status", + "exec_times", "block_uid", "payload_status", "builder_index", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -477,6 +477,11 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint fmt.Fprintf(&sql, ` AND slots.eth_block_hash = $%v `, argIdx) args = append(args, filter.EthBlockHash) } + if filter.BuilderIndex != nil { + argIdx++ + fmt.Fprintf(&sql, ` AND slots.builder_index = $%v `, argIdx) + args = append(args, *filter.BuilderIndex) + } fmt.Fprintf(&sql, ` ORDER BY slots.slot DESC `) fmt.Fprintf(&sql, ` LIMIT $%v OFFSET $%v `, argIdx+1, argIdx+2) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 74b7daea8..aad14f6ac 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -68,6 +68,7 @@ type Slot struct { ExecTimes []byte `db:"exec_times"` PayloadStatus PayloadStatus `db:"payload_status"` BlockUid uint64 `db:"block_uid"` + BuilderIndex int64 `db:"builder_index"` // Builder index, -1 for self-built blocks (MaxUint64) } type Epoch struct { diff --git a/dbtypes/other.go b/dbtypes/other.go index 8871329fa..daca67df6 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -67,6 +67,7 @@ type BlockFilter struct { ForkIds []uint64 // Filter by fork IDs EthBlockNumber *uint64 // Filter by EL block number EthBlockHash []byte // Filter by EL block hash + BuilderIndex *int64 // Filter by builder index (-1 for self-built blocks) } type MevBlockFilter struct { diff --git a/handlers/builder.go b/handlers/builder.go index aa4f9ff1e..040ff88e5 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -197,30 +197,63 @@ func buildBuilderPageData(builderIndex uint64, superseded bool, tabView string) } func buildBuilderRecentBlocks(builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBlock { - // Get recent bids from this builder that have been included (winning bids) - bids, _ := db.GetBidsByBuilderIndex(builderIndex, 0, 20) + // Filter blocks by builder index using the new DB filter + builderIndexInt64 := int64(builderIndex) + filter := &dbtypes.BlockFilter{ + BuilderIndex: &builderIndexInt64, + WithOrphaned: 1, // Include both canonical and orphaned + WithMissing: 0, // Exclude missing blocks + } - blocks := make([]*models.BuilderPageDataBlock, 0, len(bids)) - for _, bid := range bids { - // Check if this bid was actually included (payload status) - slots := db.GetSlotsByBlockHash(bid.BlockHash) - for _, slot := range slots { - if slot.PayloadStatus == dbtypes.PayloadStatusCanonical || slot.PayloadStatus == dbtypes.PayloadStatusOrphaned { - blocks = append(blocks, &models.BuilderPageDataBlock{ - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot.Slot))), - Slot: slot.Slot, - Ts: chainState.SlotToTime(phase0.Slot(slot.Slot)), - BlockRoot: slot.Root, - BlockHash: bid.BlockHash, - Status: uint16(slot.PayloadStatus), - FeeRecipient: bid.FeeRecipient, - GasLimit: bid.GasLimit, - Value: bid.Value, - ElPayment: bid.ElPayment, - }) - break - } + // Get blocks built by this builder + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(filter, 0, 20, 0) + + // Collect block hashes for batch bid lookup + blockHashes := make([][]byte, 0, len(dbBlocks)) + validBlocks := make([]*dbtypes.Slot, 0, len(dbBlocks)) + + for _, assignedSlot := range dbBlocks { + if assignedSlot.Block == nil { + continue } + slot := assignedSlot.Block + + // Only include blocks with actual payloads + if slot.PayloadStatus != dbtypes.PayloadStatusCanonical && slot.PayloadStatus != dbtypes.PayloadStatusOrphaned { + continue + } + + if len(slot.EthBlockHash) > 0 { + blockHashes = append(blockHashes, slot.EthBlockHash) + validBlocks = append(validBlocks, slot) + } + } + + // Batch fetch all bids for these block hashes + bidsMap := db.GetBidsByBlockHashes(blockHashes, builderIndex) + + // Build result + blocks := make([]*models.BuilderPageDataBlock, 0, len(validBlocks)) + for _, slot := range validBlocks { + block := &models.BuilderPageDataBlock{ + Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot.Slot))), + Slot: slot.Slot, + Ts: chainState.SlotToTime(phase0.Slot(slot.Slot)), + BlockRoot: slot.Root, + BlockHash: slot.EthBlockHash, + Status: uint16(slot.PayloadStatus), + FeeRecipient: slot.EthFeeRecipient, + GasLimit: slot.EthGasLimit, + } + + // Look up bid info for Value and ElPayment from the batch result + blockHashKey := fmt.Sprintf("%x", slot.EthBlockHash) + if bid, ok := bidsMap[blockHashKey]; ok { + block.Value = bid.Value + block.ElPayment = bid.ElPayment + } + + blocks = append(blocks, block) } return blocks diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 64f5192d9..2567029c4 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -3,6 +3,7 @@ package beacon import ( "context" "fmt" + "math" "math/rand/v2" "sync" "time" @@ -67,6 +68,7 @@ type BlockBodyIndex struct { SyncParticipation float32 EthTransactionCount uint64 BlobCount uint64 + BuilderIndex uint64 } // newBlock creates a new Block instance. @@ -409,6 +411,11 @@ func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload if blobKzgCommitments, err := body.BlobKZGCommitments(); err == nil { blockIndex.BlobCount = uint64(len(blobKzgCommitments)) } + if builderIndex, err := getBlockPayloadBuilderIndex(body); err == nil { + blockIndex.BuilderIndex = uint64(builderIndex) + } else { + blockIndex.BuilderIndex = math.MaxUint64 + } } if payload != nil { blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index c24d8d212..621cb5caf 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -334,6 +334,32 @@ func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, err } } +// getBlockPayloadBuilderIndex returns the builder index from the execution payload of a versioned signed beacon block. +func getBlockPayloadBuilderIndex(v *spec.VersionedSignedBeaconBlock) (gloas.BuilderIndex, error) { + switch v.Version { + case spec.DataVersionPhase0: + return 0, errors.New("no builder index in phase0 block") + case spec.DataVersionAltair: + return 0, errors.New("no builder index in altair block") + case spec.DataVersionBellatrix: + return 0, errors.New("no builder index in bellatrix block") + case spec.DataVersionCapella: + return 0, errors.New("no builder index in capella block") + case spec.DataVersionDeneb: + return 0, errors.New("no builder index in deneb block") + case spec.DataVersionElectra: + return 0, errors.New("no builder index in electra block") + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.Message == nil || v.Gloas.Message.Body == nil || v.Gloas.Message.Body.SignedExecutionPayloadBid == nil || v.Gloas.Message.Body.SignedExecutionPayloadBid.Message == nil { + return 0, errors.New("no gloas block") + } + + return v.Gloas.Message.Body.SignedExecutionPayloadBid.Message.BuilderIndex, nil + default: + return 0, errors.New("unknown version") + } +} + // getStateRandaoMixes returns the RANDAO mixes from a versioned beacon state. func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { switch v.Version { diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index d5e9f6087..3e2ee0791 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -287,6 +287,16 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override } } + // Get builder index from block, default to -1 (self-built/MaxUint64) + var builderIndexInt64 int64 = -1 + if blockIndex := block.GetBlockIndex(); blockIndex != nil { + if blockIndex.BuilderIndex == math.MaxUint64 { + builderIndexInt64 = -1 + } else { + builderIndexInt64 = int64(blockIndex.BuilderIndex) + } + } + dbBlock := dbtypes.Slot{ Slot: uint64(block.header.Message.Slot), Proposer: uint64(block.header.Message.ProposerIndex), @@ -307,6 +317,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override RecvDelay: block.recvDelay, PayloadStatus: payloadStatus, BlockUid: block.BlockUID, + BuilderIndex: builderIndexInt64, } blockSize, err := getBlockSize(block.dynSsz, blockBody) diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 1cce33123..db76a3b6b 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -840,6 +840,21 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } + // filter by builder index + if filter.BuilderIndex != nil { + builderIndex := blockIndex.BuilderIndex + // Convert uint64 to int64 for comparison (-1 means self-built/MaxUint64) + var builderIndexInt64 int64 + if builderIndex == math.MaxUint64 { + builderIndexInt64 = -1 + } else { + builderIndexInt64 = int64(builderIndex) + } + if builderIndexInt64 != *filter.BuilderIndex { + continue + } + } + cachedMatches = append(cachedMatches, cachedDbBlock{ slot: uint64(block.Slot), proposer: uint64(blockHeader.Message.ProposerIndex), @@ -850,7 +865,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx // reconstruct missing blocks from epoch duties // For slot/root filtering, we still need to check if we need missing blocks for that specific slot - shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 + shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 && filter.BuilderIndex == nil // If filtering by slot, only check missing for that specific slot if filter.Slot != nil { diff --git a/templates/builder/builder.html b/templates/builder/builder.html index dbbf37382..b67384236 100644 --- a/templates/builder/builder.html +++ b/templates/builder/builder.html @@ -134,7 +134,7 @@

    Builder {{ form
    Deposit Epoch:
    {{ formatAddCommas .DepositEpoch }} - ({{ formatTimestamp .DepositTs }}) + ({{ formatRecentTimeShort .DepositTs }})

    {{ end }} @@ -143,7 +143,7 @@

    Builder {{ form
    Withdrawable Epoch:
    {{ formatAddCommas .WithdrawableEpoch }} - ({{ formatTimestamp .WithdrawableTs }}) + ({{ formatRecentTimeShort .WithdrawableTs }})

    {{ end }} diff --git a/utils/format.go b/utils/format.go index 0bbc88357..167e64b36 100644 --- a/utils/format.go +++ b/utils/format.go @@ -712,8 +712,8 @@ func FormatBuilderWithIndex(index uint64, name string) template.HTML { } func formatBuilder(index uint64, name string, icon string, withIndex bool) template.HTML { - if index == math.MaxInt64 { - return template.HTML(fmt.Sprintf(" unknown", icon)) + if index == math.MaxUint64 { + return template.HTML(fmt.Sprintf(" Self-built", icon)) } else if name != "" { var nameLabel string if withIndex { From 1ca75f4adc8f67ad6968d8a25bf62110570e70cd Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 30 Jan 2026 06:06:17 +0100 Subject: [PATCH 38/99] fix staticcheck issue --- indexer/beacon/epochstats.go | 1 - 1 file changed, 1 deletion(-) diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index ecfe42654..f31660f53 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -23,7 +23,6 @@ import ( type EpochStats struct { epoch phase0.Epoch dependentRoot phase0.Root - firstBlockRoot phase0.Root dependentState *epochState requestedMutex sync.Mutex From 7c57b2c4a56c9165d93a228a9afb3bcb1d855b20 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 6 Feb 2026 22:04:17 +0100 Subject: [PATCH 39/99] add orphaned payload detection --- .../pgsql/20260108202212_epbs-payload.sql | 7 +- .../sqlite/20260108202212_epbs-payload.sql | 2 + db/slots.go | 68 +++++++++++-------- dbtypes/dbtypes.go | 1 + dbtypes/other.go | 2 + indexer/beacon/block.go | 6 ++ indexer/beacon/block_helper.go | 42 ++++++++++++ indexer/beacon/finalization.go | 60 ++++++++++++++++ indexer/beacon/pruning.go | 30 ++++++++ indexer/beacon/synchronizer.go | 30 ++++++++ indexer/beacon/writedb.go | 11 +++ services/chainservice_blocks.go | 47 ++++++++++++- 12 files changed, 275 insertions(+), 31 deletions(-) diff --git a/db/schema/pgsql/20260108202212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql index 89dd2442a..fdec29e44 100644 --- a/db/schema/pgsql/20260108202212_epbs-payload.sql +++ b/db/schema/pgsql/20260108202212_epbs-payload.sql @@ -11,12 +11,17 @@ ALTER TABLE public."orphaned_blocks" ADD ALTER TABLE public."slots" ADD "payload_status" smallint NOT NULL DEFAULT 0, - "builder_index" bigint NOT NULL DEFAULT -1; + "builder_index" bigint NOT NULL DEFAULT -1, + "eth_block_parent_hash" bytea NULL; CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON public."slots" ("payload_status" ASC NULLS LAST); +CREATE INDEX IF NOT EXISTS "slots_eth_block_parent_hash_idx" + ON public."slots" + ("eth_block_parent_hash" ASC NULLS LAST); + CREATE INDEX IF NOT EXISTS "slots_builder_index_idx" ON public."slots" ("builder_index" ASC NULLS LAST); diff --git a/db/schema/sqlite/20260108202212_epbs-payload.sql b/db/schema/sqlite/20260108202212_epbs-payload.sql index 731fbd07d..2bf22624a 100644 --- a/db/schema/sqlite/20260108202212_epbs-payload.sql +++ b/db/schema/sqlite/20260108202212_epbs-payload.sql @@ -9,8 +9,10 @@ ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "slots" ADD "payload_status" smallint NOT NULL DEFAULT 0; ALTER TABLE "slots" ADD "builder_index" BIGINT NOT NULL DEFAULT -1; +ALTER TABLE "slots" ADD "eth_block_parent_hash" BLOB NULL; CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON "slots" ("payload_status" ASC); +CREATE INDEX IF NOT EXISTS "slots_eth_block_parent_hash_idx" ON "slots" ("eth_block_parent_hash" ASC); CREATE INDEX IF NOT EXISTS "slots_builder_index_idx" ON "slots" ("builder_index" ASC); ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/slots.go b/db/slots.go index eb1fbaf5d..fdf827aa6 100644 --- a/db/slots.go +++ b/db/slots.go @@ -19,10 +19,10 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36) + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, @@ -34,17 +34,17 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, - slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.BlobCount, slot.EthGasUsed, - slot.EthGasLimit, slot.EthBaseFee, slot.EthFeeRecipient, slot.BlockSize, slot.RecvDelay, slot.MinExecTime, slot.MaxExecTime, - slot.ExecTimes, slot.BlockUid, slot.PayloadStatus, slot.BuilderIndex) + slot.EthBlockParentHash, slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.BlobCount, + slot.EthGasUsed, slot.EthGasLimit, slot.EthBaseFee, slot.EthFeeRecipient, slot.BlockSize, slot.RecvDelay, slot.MinExecTime, + slot.MaxExecTime, slot.ExecTimes, slot.BlockUid, slot.PayloadStatus, slot.BuilderIndex) if err != nil { return err } @@ -99,9 +99,9 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", "eth_gas_used", - "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", "max_exec_time", - "exec_times", "block_uid", "payload_status", "builder_index", + "eth_block_parent_hash", "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", + "eth_gas_used", "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", + "max_exec_time", "exec_times", "block_uid", "payload_status", "builder_index", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -133,9 +133,9 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -154,9 +154,9 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index FROM slots WHERE root = $1 `, root) @@ -182,9 +182,9 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -258,9 +258,9 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, eth_gas_used, - eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, max_exec_time, - exec_times, block_uid, payload_status, builder_index + eth_block_parent_hash, eth_block_extra, eth_block_extra_text, sync_participation, fork_id, blob_count, + eth_gas_used, eth_gas_limit, eth_base_fee, eth_fee_recipient, block_size, recv_delay, min_exec_time, + max_exec_time, exec_times, block_uid, payload_status, builder_index FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -320,9 +320,9 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", "eth_gas_used", - "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", "max_exec_time", - "exec_times", "block_uid", "payload_status", "builder_index", + "eth_block_parent_hash", "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "blob_count", + "eth_gas_used", "eth_gas_limit", "eth_base_fee", "eth_fee_recipient", "block_size", "recv_delay", "min_exec_time", + "max_exec_time", "exec_times", "block_uid", "payload_status", "builder_index", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -482,6 +482,16 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint fmt.Fprintf(&sql, ` AND slots.builder_index = $%v `, argIdx) args = append(args, *filter.BuilderIndex) } + if filter.WithPayloadOrphaned == 0 { + fmt.Fprintf(&sql, ` AND slots.payload_status != 2 `) + } else if filter.WithPayloadOrphaned == 2 { + fmt.Fprintf(&sql, ` AND slots.payload_status = 2 `) + } + if len(filter.EthBlockParentHash) > 0 { + argIdx++ + fmt.Fprintf(&sql, ` AND slots.eth_block_parent_hash = $%v `, argIdx) + args = append(args, filter.EthBlockParentHash) + } fmt.Fprintf(&sql, ` ORDER BY slots.slot DESC `) fmt.Fprintf(&sql, ` LIMIT $%v OFFSET $%v `, argIdx+1, argIdx+2) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index aad14f6ac..69ea81cd1 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -57,6 +57,7 @@ type Slot struct { EthFeeRecipient []byte `db:"eth_fee_recipient"` EthBlockNumber *uint64 `db:"eth_block_number"` EthBlockHash []byte `db:"eth_block_hash"` + EthBlockParentHash []byte `db:"eth_block_parent_hash"` EthBlockExtra []byte `db:"eth_block_extra"` EthBlockExtraText string `db:"eth_block_extra_text"` SyncParticipation float32 `db:"sync_participation"` diff --git a/dbtypes/other.go b/dbtypes/other.go index daca67df6..6583d5be3 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -53,6 +53,7 @@ type BlockFilter struct { InvertProposer bool WithOrphaned uint8 WithMissing uint8 + WithPayloadOrphaned uint8 // 0: only canonical payloads, 1: all, 2: only orphaned payloads MinSyncParticipation *float32 MaxSyncParticipation *float32 MinExecTime *uint32 @@ -67,6 +68,7 @@ type BlockFilter struct { ForkIds []uint64 // Filter by fork IDs EthBlockNumber *uint64 // Filter by EL block number EthBlockHash []byte // Filter by EL block hash + EthBlockParentHash []byte // Filter by EL block parent hash BuilderIndex *int64 // Filter by builder index (-1 for self-built blocks) } diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 2567029c4..b2b1f2f4b 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -49,6 +49,7 @@ type Block struct { isInFinalizedDb bool // block is in finalized table (slots) isInUnfinalizedDb bool // block is in unfinalized table (unfinalized_blocks) hasExecutionPayload bool // block has an execution payload (either in cache or db) + isPayloadOrphaned bool // payload is orphaned (next block doesn't build on it) isDisposed bool // block is disposed processingStatus dbtypes.UnfinalizedBlockStatus seenMutex sync.RWMutex @@ -64,6 +65,7 @@ type BlockBodyIndex struct { Graffiti [32]byte ExecutionExtraData []byte ExecutionHash phase0.Hash32 + ExecutionParentHash phase0.Hash32 ExecutionNumber uint64 SyncParticipation float32 EthTransactionCount uint64 @@ -416,9 +418,13 @@ func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload } else { blockIndex.BuilderIndex = math.MaxUint64 } + if parentHash, err := getBlockExecutionParentHash(body); err == nil { + blockIndex.ExecutionParentHash = parentHash + } } if payload != nil { blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) + blockIndex.ExecutionParentHash = payload.Message.Payload.ParentHash // Calculate transaction count executionTransactions := payload.Message.Payload.Transactions diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 621cb5caf..979579a0c 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -360,6 +360,48 @@ func getBlockPayloadBuilderIndex(v *spec.VersionedSignedBeaconBlock) (gloas.Buil } } +// getBlockExecutionParentHash returns the parent hash from the execution payload of a versioned signed beacon block. +func getBlockExecutionParentHash(v *spec.VersionedSignedBeaconBlock) (phase0.Hash32, error) { + switch v.Version { + case spec.DataVersionPhase0: + return phase0.Hash32{}, errors.New("no parent hash in phase0 block") + case spec.DataVersionAltair: + return phase0.Hash32{}, errors.New("no parent hash in altair block") + case spec.DataVersionBellatrix: + if v.Bellatrix == nil || v.Bellatrix.Message == nil || v.Bellatrix.Message.Body == nil || v.Bellatrix.Message.Body.ExecutionPayload == nil { + return phase0.Hash32{}, errors.New("no bellatrix block") + } + + return v.Bellatrix.Message.Body.ExecutionPayload.ParentHash, nil + case spec.DataVersionCapella: + if v.Capella == nil || v.Capella.Message == nil || v.Capella.Message.Body == nil || v.Capella.Message.Body.ExecutionPayload == nil { + return phase0.Hash32{}, errors.New("no capella block") + } + + return v.Capella.Message.Body.ExecutionPayload.ParentHash, nil + case spec.DataVersionDeneb: + if v.Deneb == nil || v.Deneb.Message == nil || v.Deneb.Message.Body == nil || v.Deneb.Message.Body.ExecutionPayload == nil { + return phase0.Hash32{}, errors.New("no deneb block") + } + + return v.Deneb.Message.Body.ExecutionPayload.ParentHash, nil + case spec.DataVersionElectra: + if v.Electra == nil || v.Electra.Message == nil || v.Electra.Message.Body == nil || v.Electra.Message.Body.ExecutionPayload == nil { + return phase0.Hash32{}, errors.New("no electra block") + } + + return v.Electra.Message.Body.ExecutionPayload.ParentHash, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.Message == nil || v.Gloas.Message.Body == nil || v.Gloas.Message.Body.SignedExecutionPayloadBid == nil || v.Gloas.Message.Body.SignedExecutionPayloadBid.Message == nil { + return phase0.Hash32{}, errors.New("no gloas block") + } + + return v.Gloas.Message.Body.SignedExecutionPayloadBid.Message.ParentBlockHash, nil + default: + return phase0.Hash32{}, errors.New("unknown version") + } +} + // getStateRandaoMixes returns the RANDAO mixes from a versioned beacon state. func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { switch v.Version { diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 29db7af13..e53154cdd 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -329,6 +329,36 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R finalizedForkIds[block.GetForkId()] = true } + // Determine payload status for canonical blocks (ePBS only) + // A payload is orphaned if the next canonical block doesn't build on it + allCanonicalBlocks := append(canonicalBlocks, nextEpochCanonicalBlocks...) + for i, block := range canonicalBlocks { + if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + continue + } + + blockIndex := block.GetBlockIndex() + if blockIndex == nil || blockIndex.ExecutionNumber == 0 { + continue // no execution payload + } + + // Find the next canonical block + var nextBlock *Block + if i+1 < len(allCanonicalBlocks) { + nextBlock = allCanonicalBlocks[i+1] + } + + if nextBlock != nil { + nextBlockIndex := nextBlock.GetBlockIndex() + if nextBlockIndex != nil { + // Check if next block builds on this block's payload + if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + block.isPayloadOrphaned = true + } + } + } + } + dependentGroups := map[phase0.Root][]*Block{} for _, block := range orphanedBlocks { var dependentRoot phase0.Root @@ -398,6 +428,36 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R } } + // Determine payload status for orphaned chain blocks (ePBS only) + // A payload is orphaned if the next block in the chain doesn't build on it + allChainBlocks := append(chain, nextBlocks...) + for i, block := range chain { + if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + continue + } + + blockIndex := block.GetBlockIndex() + if blockIndex == nil || blockIndex.ExecutionNumber == 0 { + continue // no execution payload + } + + // Find the next block in this orphaned chain + var nextBlock *Block + if i+1 < len(allChainBlocks) { + nextBlock = allChainBlocks[i+1] + } + + if nextBlock != nil { + nextBlockIndex := nextBlock.GetBlockIndex() + if nextBlockIndex != nil { + // Check if next block builds on this block's payload + if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + block.isPayloadOrphaned = true + } + } + } + } + // compute votes for canonical blocks votingBlocks := make([]*Block, len(chain)+len(nextBlocks)) copy(votingBlocks, chain) diff --git a/indexer/beacon/pruning.go b/indexer/beacon/pruning.go index 88f2276b0..b2e5a2849 100644 --- a/indexer/beacon/pruning.go +++ b/indexer/beacon/pruning.go @@ -170,6 +170,36 @@ func (indexer *Indexer) processEpochPruning(pruneEpoch phase0.Epoch) (uint64, ui } } + // Determine payload status for chain blocks (ePBS only) + // A payload is orphaned if the next block in the chain doesn't build on it + allChainBlocks := append(chain, nextBlocks...) + for i, block := range chain { + if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + continue + } + + blockIndex := block.GetBlockIndex() + if blockIndex == nil || blockIndex.ExecutionNumber == 0 { + continue // no execution payload + } + + // Find the next block in this chain + var nextBlock *Block + if i+1 < len(allChainBlocks) { + nextBlock = allChainBlocks[i+1] + } + + if nextBlock != nil { + nextBlockIndex := nextBlock.GetBlockIndex() + if nextBlockIndex != nil { + // Check if next block builds on this block's payload + if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + block.isPayloadOrphaned = true + } + } + } + } + // compute votes for canonical blocks votingBlocks := make([]*Block, len(chain)+len(nextBlocks)) copy(votingBlocks, chain) diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index a8d07b7d0..eb5956592 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -430,6 +430,36 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry sim.validatorSet = validatorSet } + // Determine payload status for canonical blocks (ePBS only) + // A payload is orphaned if the next canonical block doesn't build on it + allCanonicalBlocks := append(canonicalBlocks, nextEpochCanonicalBlocks...) + for i, block := range canonicalBlocks { + if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + continue + } + + blockIndex := block.GetBlockIndex() + if blockIndex == nil || blockIndex.ExecutionNumber == 0 { + continue // no execution payload + } + + // Find the next canonical block + var nextBlock *Block + if i+1 < len(allCanonicalBlocks) { + nextBlock = allCanonicalBlocks[i+1] + } + + if nextBlock != nil { + nextBlockIndex := nextBlock.GetBlockIndex() + if nextBlockIndex != nil { + // Check if next block builds on this block's payload + if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + block.isPayloadOrphaned = true + } + } + } + } + // save blocks err = db.RunDBTransaction(func(tx *sqlx.Tx) error { err = s.indexer.dbWriter.persistEpochData(tx, syncEpoch, canonicalBlocks, epochStats, epochVotes, sim) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 3e2ee0791..6dcfdb672 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -72,6 +72,11 @@ func (dbw *dbWriter) persistBlockData(tx *sqlx.Tx, block *Block, epochStats *Epo dbBlock.Status = dbtypes.Orphaned } + // Apply payload orphaned status from block flag (set during finalization/sync) + if block.isPayloadOrphaned { + dbBlock.PayloadStatus = dbtypes.PayloadStatusOrphaned + } + err := db.InsertSlot(dbBlock, tx) if err != nil { return nil, fmt.Errorf("error inserting slot: %v", err) @@ -251,6 +256,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionBlockHash, _ := blockBody.ExecutionBlockHash() var executionBlockNumber uint64 + var executionBlockParentHash []byte var executionExtraData []byte var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal @@ -262,6 +268,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override blockPayload := block.GetExecutionPayload() if blockPayload != nil { executionBlockNumber = blockPayload.Message.Payload.BlockNumber + executionBlockParentHash = blockPayload.Message.Payload.ParentHash[:] executionExtraData = blockPayload.Message.Payload.ExtraData executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals @@ -279,6 +286,9 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionExtraData, _ = executionPayload.ExtraData() executionTransactions, _ = executionPayload.Transactions() executionWithdrawals, _ = executionPayload.Withdrawals() + if parentHash, err := executionPayload.ParentHash(); err == nil { + executionBlockParentHash = parentHash[:] + } } blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() @@ -353,6 +363,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override dbBlock.EthTransactionCount = uint64(len(executionTransactions)) dbBlock.EthBlockNumber = &executionBlockNumber dbBlock.EthBlockHash = executionBlockHash[:] + dbBlock.EthBlockParentHash = executionBlockParentHash dbBlock.EthBlockExtra = executionExtraData dbBlock.EthBlockExtraText = utils.GraffitiToString(executionExtraData[:]) dbBlock.WithdrawCount = uint64(len(executionWithdrawals)) diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index db76a3b6b..5f771aac4 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -855,6 +855,51 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } + // filter by EL block parent hash + if len(filter.EthBlockParentHash) > 0 { + if !bytes.Equal(blockIndex.ExecutionParentHash[:], filter.EthBlockParentHash) { + continue + } + } + + // filter by payload status (runtime computation for unfinalized blocks) + if filter.WithPayloadOrphaned != 1 { + // Compute payload status by checking if any child block in the canonical chain + // builds on this block's execution payload + payloadIsCanonical := false + if blockIndex.ExecutionNumber > 0 { + // Get child blocks and check if any canonical child builds on this payload + childBlocks := bs.beaconIndexer.GetBlockByParentRoot(block.Root) + for _, child := range childBlocks { + childIndex := child.GetBlockIndex() + if childIndex == nil { + continue + } + // Check if child is in the canonical chain + if !bs.beaconIndexer.IsCanonicalBlockByHead(child, lastCanonicalBlock) { + continue + } + // Check if child builds on this block's execution payload + if bytes.Equal(childIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + payloadIsCanonical = true + break + } + } + } else { + // No execution payload, treat as canonical for filtering purposes + payloadIsCanonical = true + } + + if filter.WithPayloadOrphaned == 0 && !payloadIsCanonical { + // only canonical payloads, skip orphaned + continue + } + if filter.WithPayloadOrphaned == 2 && payloadIsCanonical { + // only orphaned payloads, skip canonical + continue + } + } + cachedMatches = append(cachedMatches, cachedDbBlock{ slot: uint64(block.Slot), proposer: uint64(blockHeader.Message.ProposerIndex), @@ -865,7 +910,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx // reconstruct missing blocks from epoch duties // For slot/root filtering, we still need to check if we need missing blocks for that specific slot - shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 && filter.BuilderIndex == nil + shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 && filter.BuilderIndex == nil && filter.WithPayloadOrphaned != 2 && len(filter.EthBlockParentHash) == 0 // If filtering by slot, only check missing for that specific slot if filter.Slot != nil { From acdf5e0d31b8295812c286924fe1475de803a187 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 13 Feb 2026 13:43:42 +0100 Subject: [PATCH 40/99] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c049e187..d44ee843e 100644 --- a/go.mod +++ b/go.mod @@ -261,4 +261,4 @@ require ( modernc.org/sqlite v1.38.2 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab diff --git a/go.sum b/go.sum index ed554c5a1..cab9ec200 100644 --- a/go.sum +++ b/go.sum @@ -567,8 +567,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.1 h1:84eNMiiOYDiNC2Y1m5A/UtIPs6u/9SsvG4RVSBRGE5U= github.com/pk910/dynamic-ssz v1.2.1/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d h1:BRJcO8QD00qTwDX4doVLvDgpzkECmVjb3HQ3P2GJF5g= -github.com/pk910/go-eth2-client v0.0.0-20260120171652-f54f2961d32d/go.mod h1:mKrNtB6iRgMN+gLXwvko19uSdivFVX4/mNYUlrCLwNQ= +github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab h1:JgraAl78YBypoFGwYiKoZXKnXTUJWXpFErWBBVyYCs0= +github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab/go.mod h1:1MBvz1DVe6jZduHJDJuJ/1S4LzYmDmvx+r5alo4pyrM= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= From e29fbf58a50c20353e9b51fc7e07ca811da15498 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Mon, 16 Feb 2026 11:11:48 +0100 Subject: [PATCH 41/99] fix: execution requests --- indexer/beacon/epochstate.go | 58 +++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index f6878847f..715e09555 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -8,6 +8,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -135,7 +136,17 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch return nil, err } - err = s.processState(resState, beaconBlock, cache) + var executionPayload *gloas.SignedExecutionPayloadEnvelope + if beaconBlock != nil && beaconBlock.Version >= spec.DataVersionGloas { + if block != nil { + executionPayload = block.GetExecutionPayload() + } + if executionPayload == nil { + executionPayload, _ = LoadExecutionPayload(ctx, client, s.slotRoot) + } + } + + err = s.processState(resState, beaconBlock, executionPayload, cache) if err != nil { return nil, err } @@ -153,7 +164,7 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch // processState processes the state and updates the epochState instance. // the function extracts and unifies all relevant information from the beacon state, so the full beacon state can be dropped from memory afterwards. -func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, cache *epochCache) error { +func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, executionPayload *gloas.SignedExecutionPayloadEnvelope, cache *epochCache) error { slot, err := state.Slot() if err != nil { return fmt.Errorf("error getting slot from state %v: %v", s.slotRoot.String(), err) @@ -215,16 +226,27 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock s.randaoMixes = randaoMixes if state.Version >= spec.DataVersionFulu { - // subtract the deposit indexes from the current block - blockRequests, err := beaconBlock.ExecutionRequests() - if err != nil { - return fmt.Errorf("error getting execution requests from block %v: %v", s.slotRoot.String(), err) - } - - if len(blockRequests.Deposits) > 0 { - s.depositIndex = blockRequests.Deposits[0].Index + if state.Version >= spec.DataVersionGloas { + isPostPayload := isGloasPostPayloadState(state, slot) + if isPostPayload && executionPayload != nil && + executionPayload.Message != nil && + executionPayload.Message.ExecutionRequests != nil && + len(executionPayload.Message.ExecutionRequests.Deposits) > 0 { + s.depositIndex = executionPayload.Message.ExecutionRequests.Deposits[0].Index + } else { + s.depositIndex = getStateDepositIndex(state) + } } else { - s.depositIndex = getStateDepositIndex(state) + blockRequests, err := beaconBlock.ExecutionRequests() + if err != nil { + return fmt.Errorf("error getting execution requests from block %v: %v", + s.slotRoot.String(), err) + } + if len(blockRequests.Deposits) > 0 { + s.depositIndex = blockRequests.Deposits[0].Index + } else { + s.depositIndex = getStateDepositIndex(state) + } } } else { s.depositIndex = getStateDepositIndex(state) @@ -281,3 +303,17 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock return nil } + +// isGloasPostPayloadState checks whether the Gloas state is post-payload +// (i.e. execution payload deposits have been applied) for the given slot. +func isGloasPostPayloadState(state *spec.VersionedBeaconState, slot phase0.Slot) bool { + if state.Gloas == nil { + return false + } + bitfieldLen := uint64(len(state.Gloas.ExecutionPayloadAvailability)) * 8 + if bitfieldLen == 0 { + return false + } + idx := uint64(slot) % bitfieldLen + return state.Gloas.ExecutionPayloadAvailability[idx/8]&(1<<(idx%8)) != 0 +} From 98047f80afc857644c7252c52d5917413ca8a30f Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Mon, 16 Feb 2026 11:41:06 +0100 Subject: [PATCH 42/99] feat(beacon): load payload based on EIP-7737 enablement in client process This change modifies how blocks are processed in the beacon client. It checks the chain state for EIP-7737 enablement based on the block slot's epoch to determine whether to load the execution payload when processing a block from the stream. Additionally, it updates `GetSlotDetailsByBlockroot` and `GetSlotDetailsBySlot` to include the payload in the response if it exists, further integrating payload handling into block detail retrieval. --- indexer/beacon/client.go | 5 ++++- services/chainservice_blocks.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 2d26df48a..a3b9b0f58 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -310,7 +310,10 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { // processStreamBlock processes a block received from the stream (either via block or head events). func (c *Client) processStreamBlock(slot phase0.Slot, root phase0.Root) (*Block, error) { - block, isNew, processingTimes, err := c.processBlock(slot, root, nil, true, false) + chainState := c.client.GetPool().GetChainState() + loadPayload := chainState.IsEip7732Enabled(chainState.EpochOfSlot(slot)) + + block, isNew, processingTimes, err := c.processBlock(slot, root, nil, true, loadPayload) if err != nil { return nil, err } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 4dbab17a7..7d8e0307f 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -183,12 +183,16 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) }) if err == nil && blockData != nil && blockData.Body != nil { - result = &CombinedBlockResponse{ + resp := &CombinedBlockResponse{ Root: blockroot, Header: header, Block: blockData.Body.(*spec.VersionedSignedBeaconBlock), Orphaned: false, } + if blockData.Payload != nil { + resp.Payload = blockData.Payload.(*gloas.SignedExecutionPayloadEnvelope) + } + result = resp } } @@ -330,12 +334,16 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl return nil, err } - result = &CombinedBlockResponse{ + resp := &CombinedBlockResponse{ Root: blockRoot, Header: header, Block: blockData.Body.(*spec.VersionedSignedBeaconBlock), Orphaned: false, } + if blockData.Payload != nil { + resp.Payload = blockData.Payload.(*gloas.SignedExecutionPayloadEnvelope) + } + result = resp } } From ca65050a5c7d21f21629ac98995dc26df39004ba Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Mon, 16 Feb 2026 12:04:39 +0100 Subject: [PATCH 43/99] refactor(beacon): simplify block processing logic by removing EIP-7732 check in processStreamBlock Removes the logic to determine if EIP-7732 payload loading is needed when processing a block from the head event stream because blocks received via head events are assumed to not require immediate execution payload processing for caching purposes initially. Updates `processExecutionPayloadAvailableEvent` to use `createOrGetBlock` when a block is not found in the cache during an execution payload availability event, ensuring the block object exists before proceeding. --- indexer/beacon/client.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index a3b9b0f58..c217f9a60 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -310,10 +310,7 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { // processStreamBlock processes a block received from the stream (either via block or head events). func (c *Client) processStreamBlock(slot phase0.Slot, root phase0.Root) (*Block, error) { - chainState := c.client.GetPool().GetChainState() - loadPayload := chainState.IsEip7732Enabled(chainState.EpochOfSlot(slot)) - - block, isNew, processingTimes, err := c.processBlock(slot, root, nil, true, loadPayload) + block, isNew, processingTimes, err := c.processBlock(slot, root, nil, true, false) if err != nil { return nil, err } @@ -621,7 +618,7 @@ func (c *Client) processExecutionPayloadAvailableEvent(executionPayloadEvent *v1 } } else { - block = c.indexer.blockCache.getBlockByRoot(executionPayloadEvent.BlockRoot) + block, _ = c.indexer.blockCache.createOrGetBlock(executionPayloadEvent.BlockRoot, executionPayloadEvent.Slot) } if block == nil { From c74876185a8aae31c7887d077d6e03d4928b5de4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 19 Feb 2026 12:46:01 +0100 Subject: [PATCH 44/99] fix gloas types --- dbtypes/dbtypes.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- handlers/slot.go | 32 ++++++++++++++++++-------------- indexer/beacon/bidcache.go | 2 +- indexer/beacon/block.go | 4 ---- indexer/beacon/client.go | 2 +- indexer/beacon/writedb.go | 9 ++------- templates/slot/overview.html | 19 ++++++++++++++++--- types/models/slot.go | 22 +++++++++++----------- 10 files changed, 53 insertions(+), 45 deletions(-) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index e0dfb7d7b..dd0ee58ce 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -573,7 +573,7 @@ type BlockBid struct { BlockHash []byte `db:"block_hash"` FeeRecipient []byte `db:"fee_recipient"` GasLimit uint64 `db:"gas_limit"` - BuilderIndex uint64 `db:"builder_index"` + BuilderIndex int64 `db:"builder_index"` Slot uint64 `db:"slot"` Value uint64 `db:"value"` ElPayment uint64 `db:"el_payment"` diff --git a/go.mod b/go.mod index bdc700b11..4f57bb594 100644 --- a/go.mod +++ b/go.mod @@ -261,4 +261,4 @@ require ( modernc.org/sqlite v1.38.2 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f diff --git a/go.sum b/go.sum index d1a6a8e6c..0eb297d75 100644 --- a/go.sum +++ b/go.sum @@ -567,8 +567,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.1 h1:84eNMiiOYDiNC2Y1m5A/UtIPs6u/9SsvG4RVSBRGE5U= github.com/pk910/dynamic-ssz v1.2.1/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab h1:JgraAl78YBypoFGwYiKoZXKnXTUJWXpFErWBBVyYCs0= -github.com/pk910/go-eth2-client v0.0.0-20260213124136-f6ee250facab/go.mod h1:1MBvz1DVe6jZduHJDJuJ/1S4LzYmDmvx+r5alo4pyrM= +github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f h1:OjX1YemePnLes4JqeDFIJOZN9YPCc4R6cyvyXFtVA6c= +github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f/go.mod h1:8fpxrIBBVbOcVG3vcHe5ubOHIeqW3N5t7kS4oU5EeJU= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/handlers/slot.go b/handlers/slot.go index fc2e3d4f2..96dfc06f9 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -739,24 +739,28 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } if payloadBid, err := blockData.Block.SignedExecutionPayloadBid(); err == nil { + commitments := make([][]byte, len(payloadBid.Message.BlobKZGCommitments)) + for i := range payloadBid.Message.BlobKZGCommitments { + commitments[i] = payloadBid.Message.BlobKZGCommitments[i][:] + } + pageData.PayloadHeader = &models.SlotPagePayloadHeader{ - PayloadStatus: uint16(0), - ParentBlockHash: payloadBid.Message.ParentBlockHash[:], - ParentBlockRoot: payloadBid.Message.ParentBlockRoot[:], - BlockHash: payloadBid.Message.BlockHash[:], - GasLimit: uint64(payloadBid.Message.GasLimit), - BuilderIndex: uint64(payloadBid.Message.BuilderIndex), - BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadBid.Message.BuilderIndex) | services.BuilderIndexFlag), - Slot: uint64(payloadBid.Message.Slot), - Value: uint64(payloadBid.Message.Value), - BlobKzgCommitmentsRoot: payloadBid.Message.BlobKZGCommitmentsRoot[:], - Signature: payloadBid.Signature[:], + PayloadStatus: uint16(0), + ParentBlockHash: payloadBid.Message.ParentBlockHash[:], + ParentBlockRoot: payloadBid.Message.ParentBlockRoot[:], + BlockHash: payloadBid.Message.BlockHash[:], + GasLimit: uint64(payloadBid.Message.GasLimit), + BuilderIndex: uint64(payloadBid.Message.BuilderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadBid.Message.BuilderIndex) | services.BuilderIndexFlag), + Slot: uint64(payloadBid.Message.Slot), + Value: uint64(payloadBid.Message.Value), + BlobKZGCommitments: commitments, + Signature: payloadBid.Signature[:], } } var executionPayload *spec.VersionedExecutionPayload if blockData.Block.Version >= spec.DataVersionGloas && blockData.Payload != nil { - blobKzgCommitments = blockData.Payload.Message.BlobKZGCommitments executionPayload = &spec.VersionedExecutionPayload{ Version: spec.DataVersionGloas, Gloas: blockData.Payload.Message.Payload, @@ -1159,8 +1163,8 @@ func getSlotPageBids(pageData *models.SlotPageBlockData) { BlockHash: bid.BlockHash, FeeRecipient: bid.FeeRecipient, GasLimit: bid.GasLimit, - BuilderIndex: bid.BuilderIndex, - BuilderName: services.GlobalBeaconService.GetValidatorName(bid.BuilderIndex), + BuilderIndex: uint64(bid.BuilderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(bid.BuilderIndex)), Slot: bid.Slot, Value: bid.Value, ElPayment: bid.ElPayment, diff --git a/indexer/beacon/bidcache.go b/indexer/beacon/bidcache.go index ce4c5ac65..32d23ab41 100644 --- a/indexer/beacon/bidcache.go +++ b/indexer/beacon/bidcache.go @@ -23,7 +23,7 @@ type bidCacheKey struct { ParentRoot phase0.Root ParentHash phase0.Hash32 BlockHash phase0.Hash32 - BuilderIndex uint64 + BuilderIndex int64 } // blockBidCache caches execution payload bids for recent blocks. diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 4415a68b7..434767714 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -444,10 +444,6 @@ func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload executionTransactions := payload.Message.Payload.Transactions blockIndex.EthTransactionCount = uint64(len(executionTransactions)) - // Calculate blob count - blobKzgCommitments := payload.Message.BlobKZGCommitments - blockIndex.BlobCount = uint64(len(blobKzgCommitments)) - // Get gas used and gas limit blockIndex.GasUsed = payload.Message.Payload.GasUsed blockIndex.GasLimit = payload.Message.Payload.GasLimit diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index d3cd18554..49e7659ee 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -673,7 +673,7 @@ func (c *Client) processExecutionPayloadBidEvent(executionPayloadBidEvent *gloas BlockHash: executionPayloadBidEvent.Message.BlockHash[:], FeeRecipient: executionPayloadBidEvent.Message.FeeRecipient[:], GasLimit: uint64(executionPayloadBidEvent.Message.GasLimit), - BuilderIndex: uint64(executionPayloadBidEvent.Message.BuilderIndex), + BuilderIndex: int64(executionPayloadBidEvent.Message.BuilderIndex), Slot: uint64(executionPayloadBidEvent.Message.Slot), Value: uint64(executionPayloadBidEvent.Message.Value), ElPayment: uint64(executionPayloadBidEvent.Message.ExecutionPayment), diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 30543d9bf..5b274ef62 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -7,7 +7,6 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" @@ -262,6 +261,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() executionBlockHash, _ := blockBody.ExecutionBlockHash() + blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionBlockNumber uint64 var executionBlockParentHash []byte @@ -269,7 +269,6 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - var blobKzgCommitments []deneb.KZGCommitment var payloadStatus dbtypes.PayloadStatus if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { @@ -281,7 +280,6 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits - blobKzgCommitments = blockPayload.Message.BlobKZGCommitments payloadStatus = dbtypes.PayloadStatusCanonical } else { payloadStatus = dbtypes.PayloadStatusMissing @@ -298,7 +296,6 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionBlockParentHash = parentHash[:] } } - blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits @@ -532,11 +529,11 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() + blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - var blobKzgCommitments []deneb.KZGCommitment if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -545,12 +542,10 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits - blobKzgCommitments = blockPayload.Message.BlobKZGCommitments } } else { executionTransactions, _ = blockBody.ExecutionTransactions() executionWithdrawals, _ = blockBody.Withdrawals() - blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits diff --git a/templates/slot/overview.html b/templates/slot/overview.html index 96a3453ef..1753593c1 100644 --- a/templates/slot/overview.html +++ b/templates/slot/overview.html @@ -265,10 +265,23 @@

    -
    Blob KZG Root:
    +
    Blob KZG Commitments:
    - 0x{{ printf "%x" .BlobKzgCommitmentsRoot }} - + {{ len .BlobKZGCommitments }} blob{{ if ne (len .BlobKZGCommitments) 1 }}s{{ end }} + {{ if .BlobKZGCommitments }} + +
    + {{ range $i, $c := .BlobKZGCommitments }} +
    + {{ $i }}: + 0x{{ printf "%x" $c }} + +
    + {{ end }} +
    + {{ end }}
    diff --git a/types/models/slot.go b/types/models/slot.go index 45f31d7de..86fe0ba33 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -125,17 +125,17 @@ type SlotPageExecutionData struct { } type SlotPagePayloadHeader struct { - PayloadStatus uint16 `json:"payload_status"` - ParentBlockHash []byte `json:"parent_block_hash"` - ParentBlockRoot []byte `json:"parent_block_root"` - BlockHash []byte `json:"block_hash"` - GasLimit uint64 `json:"gas_limit"` - BuilderIndex uint64 `json:"builder_index"` - BuilderName string `json:"builder_name"` - Slot uint64 `json:"slot"` - Value uint64 `json:"value"` - BlobKzgCommitmentsRoot []byte `json:"blob_kzg_commitments_root"` - Signature []byte `json:"signature"` + PayloadStatus uint16 `json:"payload_status"` + ParentBlockHash []byte `json:"parent_block_hash"` + ParentBlockRoot []byte `json:"parent_block_root"` + BlockHash []byte `json:"block_hash"` + GasLimit uint64 `json:"gas_limit"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` + Slot uint64 `json:"slot"` + Value uint64 `json:"value"` + BlobKZGCommitments [][]byte `json:"blob_kzg_commitments"` + Signature []byte `json:"signature"` } type SlotPageAttestation struct { From f934ab711300b18cb255bec2836537fb76b8e195 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 19 Feb 2026 12:54:54 +0100 Subject: [PATCH 45/99] improve bid rendering --- handlers/slot.go | 1 + templates/slot/bids.html | 15 ++++++++++++--- templates/slot/overview.html | 2 +- types/models/slot.go | 1 + 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/handlers/slot.go b/handlers/slot.go index 96dfc06f9..e558320aa 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -1165,6 +1165,7 @@ func getSlotPageBids(pageData *models.SlotPageBlockData) { GasLimit: bid.GasLimit, BuilderIndex: uint64(bid.BuilderIndex), BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(bid.BuilderIndex)), + IsSelfBuilt: bid.BuilderIndex < 0, Slot: bid.Slot, Value: bid.Value, ElPayment: bid.ElPayment, diff --git a/templates/slot/bids.html b/templates/slot/bids.html index e0e1770c6..37f6184f8 100644 --- a/templates/slot/bids.html +++ b/templates/slot/bids.html @@ -14,12 +14,21 @@ {{ range $i, $bid := .Block.Bids }} - + - {{ formatValidatorWithIndex $bid.BuilderIndex $bid.BuilderName }} + {{ if $bid.IsSelfBuilt }} + Self-built + {{ else }} + {{ formatValidatorWithIndex $bid.BuilderIndex $bid.BuilderName }} + {{ end }} {{ if $bid.IsWinning }}Winner{{ end }} - 0x{{ printf "%.16x" $bid.BlockHash }}... + +
    + 0x{{ printf "%x" $bid.BlockHash }} + +
    + {{ ethAddressLink $bid.FeeRecipient }} {{ formatAddCommas $bid.GasLimit }} {{ formatEthFromGwei $bid.Value }} diff --git a/templates/slot/overview.html b/templates/slot/overview.html index 1753593c1..eb5505275 100644 --- a/templates/slot/overview.html +++ b/templates/slot/overview.html @@ -218,7 +218,7 @@ {{ if eq .PayloadStatus 0 }} Missing {{ else if eq .PayloadStatus 1 }} - Proposed + Revealed {{ else if eq .PayloadStatus 2 }} Orphaned {{ else }} diff --git a/types/models/slot.go b/types/models/slot.go index 86fe0ba33..d94c6dc4f 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -309,6 +309,7 @@ type SlotPageBid struct { GasLimit uint64 `json:"gas_limit"` BuilderIndex uint64 `json:"builder_index"` BuilderName string `json:"builder_name"` + IsSelfBuilt bool `json:"is_self_built"` Slot uint64 `json:"slot"` Value uint64 `json:"value"` ElPayment uint64 `json:"el_payment"` From c07a098c87aea15d5cc80383c31cc01770793515 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 19 Feb 2026 13:44:06 +0100 Subject: [PATCH 46/99] fix orphaned payload filter --- services/chainservice_blocks.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 7d8e0307f..307fb077c 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -667,7 +667,8 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx // get blocks from cache // iterate from current slot to finalized slot - lastCanonicalBlock := bs.beaconIndexer.GetCanonicalHead(nil) + canonicalHead := bs.beaconIndexer.GetCanonicalHead(nil) + lastCanonicalBlock := canonicalHead // apply epoch filter to slot range cacheStartSlot := startSlot @@ -930,7 +931,9 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } // filter by payload status (runtime computation for unfinalized blocks) - if filter.WithPayloadOrphaned != 1 { + // Only applies to gloas/ePBS blocks where payloads are separate from beacon blocks + blockEpoch := chainState.EpochOfSlot(block.Slot) + if filter.WithPayloadOrphaned != 1 && chainState.IsEip7732Enabled(blockEpoch) { // Compute payload status by checking if any child block in the canonical chain // builds on this block's execution payload payloadIsCanonical := false @@ -942,8 +945,9 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx if childIndex == nil { continue } - // Check if child is in the canonical chain - if !bs.beaconIndexer.IsCanonicalBlockByHead(child, lastCanonicalBlock) { + // Check if child is in the canonical chain (use original head since + // children are at higher slots than the updated lastCanonicalBlock) + if !bs.beaconIndexer.IsCanonicalBlockByHead(child, canonicalHead) { continue } // Check if child builds on this block's execution payload From 295ee8ac72080f79b2158b957597756929a195f5 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 23 Feb 2026 12:48:36 +0100 Subject: [PATCH 47/99] make gloas db operations & page handlers context aware --- db/block_bids.go | 19 ++++++------ db/builders.go | 43 +++++++++++++------------- handlers/builder.go | 10 +++--- handlers/builders.go | 7 +++-- handlers/search.go | 52 ++++++++++++++++---------------- indexer/beacon/bidcache.go | 2 +- indexer/beacon/buildercache.go | 8 ++--- indexer/beacon/indexer_getter.go | 2 +- services/chainservice_builder.go | 7 +++-- 9 files changed, 77 insertions(+), 73 deletions(-) diff --git a/db/block_bids.go b/db/block_bids.go index cb1efc69a..7b2b9c6ca 100644 --- a/db/block_bids.go +++ b/db/block_bids.go @@ -1,6 +1,7 @@ package db import ( + "context" "fmt" "strings" @@ -63,7 +64,7 @@ func InsertBids(bids []*dbtypes.BlockBid, tx *sqlx.Tx) error { return nil } -func GetBidsForBlockRoot(blockRoot []byte) []*dbtypes.BlockBid { +func GetBidsForBlockRoot(ctx context.Context, blockRoot []byte) []*dbtypes.BlockBid { var sql strings.Builder args := []any{ blockRoot, @@ -77,7 +78,7 @@ func GetBidsForBlockRoot(blockRoot []byte) []*dbtypes.BlockBid { `) bids := []*dbtypes.BlockBid{} - err := ReaderDb.Select(&bids, sql.String(), args...) + err := ReaderDb.SelectContext(ctx, &bids, sql.String(), args...) if err != nil { logger.Errorf("Error while fetching bids for block root: %v", err) return nil @@ -85,7 +86,7 @@ func GetBidsForBlockRoot(blockRoot []byte) []*dbtypes.BlockBid { return bids } -func GetBidsForSlotRange(minSlot uint64) []*dbtypes.BlockBid { +func GetBidsForSlotRange(ctx context.Context, minSlot uint64) []*dbtypes.BlockBid { var sql strings.Builder args := []any{ minSlot, @@ -99,7 +100,7 @@ func GetBidsForSlotRange(minSlot uint64) []*dbtypes.BlockBid { `) bids := []*dbtypes.BlockBid{} - err := ReaderDb.Select(&bids, sql.String(), args...) + err := ReaderDb.SelectContext(ctx, &bids, sql.String(), args...) if err != nil { logger.Errorf("Error while fetching bids for slot range: %v", err) return nil @@ -114,7 +115,7 @@ func DeleteBidsBeforeSlot(minSlot uint64, tx *sqlx.Tx) error { // GetBidsByBlockHashes returns bids for multiple block hashes and a specific builder index // Returns a map keyed by block hash (hex string) for easy lookup -func GetBidsByBlockHashes(blockHashes [][]byte, builderIndex uint64) map[string]*dbtypes.BlockBid { +func GetBidsByBlockHashes(ctx context.Context, blockHashes [][]byte, builderIndex uint64) map[string]*dbtypes.BlockBid { result := make(map[string]*dbtypes.BlockBid, len(blockHashes)) if len(blockHashes) == 0 { return result @@ -140,7 +141,7 @@ func GetBidsByBlockHashes(blockHashes [][]byte, builderIndex uint64) map[string] fmt.Fprint(&sql, ")") bids := []*dbtypes.BlockBid{} - err := ReaderDb.Select(&bids, sql.String(), args...) + err := ReaderDb.SelectContext(ctx, &bids, sql.String(), args...) if err != nil { logger.Errorf("Error while fetching bids by block hashes: %v", err) return result @@ -155,7 +156,7 @@ func GetBidsByBlockHashes(blockHashes [][]byte, builderIndex uint64) map[string] } // GetBidsByBuilderIndex returns bids submitted by a specific builder, ordered by slot descending -func GetBidsByBuilderIndex(builderIndex uint64, offset uint64, limit uint32) ([]*dbtypes.BlockBid, uint64) { +func GetBidsByBuilderIndex(ctx context.Context, builderIndex uint64, offset uint64, limit uint32) ([]*dbtypes.BlockBid, uint64) { var sql strings.Builder args := []any{ builderIndex, @@ -174,7 +175,7 @@ func GetBidsByBuilderIndex(builderIndex uint64, offset uint64, limit uint32) ([] } bids := []*dbtypes.BlockBid{} - err := ReaderDb.Select(&bids, sql.String(), args...) + err := ReaderDb.SelectContext(ctx, &bids, sql.String(), args...) if err != nil { logger.Errorf("Error while fetching bids for builder index %d: %v", builderIndex, err) return nil, 0 @@ -182,7 +183,7 @@ func GetBidsByBuilderIndex(builderIndex uint64, offset uint64, limit uint32) ([] // Get total count var totalCount uint64 - err = ReaderDb.Get(&totalCount, `SELECT COUNT(*) FROM block_bids WHERE builder_index = $1`, builderIndex) + err = ReaderDb.GetContext(ctx, &totalCount, `SELECT COUNT(*) FROM block_bids WHERE builder_index = $1`, builderIndex) if err != nil { logger.Errorf("Error while counting bids for builder index %d: %v", builderIndex, err) return bids, 0 diff --git a/db/builders.go b/db/builders.go index cb7d314d8..26d67bfa1 100644 --- a/db/builders.go +++ b/db/builders.go @@ -1,6 +1,7 @@ package db import ( + "context" "fmt" "strings" @@ -93,9 +94,9 @@ func InsertBuilderBatch(builders []*dbtypes.Builder, tx *sqlx.Tx) error { } // GetBuilderByPubkey returns a builder by pubkey (primary key) -func GetBuilderByPubkey(pubkey []byte) *dbtypes.Builder { +func GetBuilderByPubkey(ctx context.Context, pubkey []byte) *dbtypes.Builder { builder := dbtypes.Builder{} - err := ReaderDb.Get(&builder, ` + err := ReaderDb.GetContext(ctx, &builder, ` SELECT * FROM builders WHERE pubkey = $1 `, pubkey) if err != nil { @@ -105,9 +106,9 @@ func GetBuilderByPubkey(pubkey []byte) *dbtypes.Builder { } // GetActiveBuilderByIndex returns the active (non-superseded) builder for a given index -func GetActiveBuilderByIndex(index uint64) *dbtypes.Builder { +func GetActiveBuilderByIndex(ctx context.Context, index uint64) *dbtypes.Builder { builder := dbtypes.Builder{} - err := ReaderDb.Get(&builder, ` + err := ReaderDb.GetContext(ctx, &builder, ` SELECT * FROM builders WHERE builder_index = $1 AND superseded = false `, index) if err != nil { @@ -117,9 +118,9 @@ func GetActiveBuilderByIndex(index uint64) *dbtypes.Builder { } // GetBuildersByIndex returns all builders (including superseded) for a given index -func GetBuildersByIndex(index uint64) []*dbtypes.Builder { +func GetBuildersByIndex(ctx context.Context, index uint64) []*dbtypes.Builder { builders := []*dbtypes.Builder{} - err := ReaderDb.Select(&builders, ` + err := ReaderDb.SelectContext(ctx, &builders, ` SELECT * FROM builders WHERE builder_index = $1 ORDER BY superseded ASC `, index) if err != nil { @@ -130,9 +131,9 @@ func GetBuildersByIndex(index uint64) []*dbtypes.Builder { } // GetBuilderRange returns builders in a given index range (only active builders) -func GetBuilderRange(startIndex uint64, endIndex uint64) []*dbtypes.Builder { +func GetBuilderRange(ctx context.Context, startIndex uint64, endIndex uint64) []*dbtypes.Builder { builders := []*dbtypes.Builder{} - err := ReaderDb.Select(&builders, ` + err := ReaderDb.SelectContext(ctx, &builders, ` SELECT * FROM builders WHERE builder_index >= $1 AND builder_index <= $2 AND superseded = false ORDER BY builder_index ASC @@ -145,9 +146,9 @@ func GetBuilderRange(startIndex uint64, endIndex uint64) []*dbtypes.Builder { } // GetMaxBuilderIndex returns the highest builder index in the database -func GetMaxBuilderIndex() (uint64, error) { +func GetMaxBuilderIndex(ctx context.Context) (uint64, error) { var maxIndex uint64 - err := ReaderDb.Get(&maxIndex, "SELECT COALESCE(MAX(builder_index), 0) FROM builders") + err := ReaderDb.GetContext(ctx, &maxIndex, "SELECT COALESCE(MAX(builder_index), 0) FROM builders") if err != nil { return 0, fmt.Errorf("error getting max builder index: %w", err) } @@ -155,13 +156,13 @@ func GetMaxBuilderIndex() (uint64, error) { } // GetBuilderCount returns the count of builders (optionally only active) -func GetBuilderCount(activeOnly bool) (uint64, error) { +func GetBuilderCount(ctx context.Context, activeOnly bool) (uint64, error) { var count uint64 var err error if activeOnly { - err = ReaderDb.Get(&count, "SELECT COUNT(*) FROM builders WHERE superseded = false") + err = ReaderDb.GetContext(ctx, &count, "SELECT COUNT(*) FROM builders WHERE superseded = false") } else { - err = ReaderDb.Get(&count, "SELECT COUNT(*) FROM builders") + err = ReaderDb.GetContext(ctx, &count, "SELECT COUNT(*) FROM builders") } if err != nil { return 0, fmt.Errorf("error getting builder count: %w", err) @@ -207,7 +208,7 @@ func SetBuildersSuperseded(pubkeys [][]byte, tx *sqlx.Tx) error { } // StreamBuildersByPubkeys streams builders by pubkeys in batches -func StreamBuildersByPubkeys(pubkeys [][]byte, cb func(builder *dbtypes.Builder) bool) error { +func StreamBuildersByPubkeys(ctx context.Context, pubkeys [][]byte, cb func(builder *dbtypes.Builder) bool) error { const batchSize = 1000 for i := 0; i < len(pubkeys); i += batchSize { @@ -240,7 +241,7 @@ func StreamBuildersByPubkeys(pubkeys [][]byte, cb func(builder *dbtypes.Builder) // Fetch all builders for this batch builders := make([]*dbtypes.Builder, len(batch)) - rows, err := ReaderDb.Query(sql.String(), args...) + rows, err := ReaderDb.QueryContext(ctx, sql.String(), args...) if err != nil { return fmt.Errorf("error querying builders: %w", err) } @@ -280,9 +281,9 @@ func StreamBuildersByPubkeys(pubkeys [][]byte, cb func(builder *dbtypes.Builder) } // GetBuildersByExecutionAddress returns builders with a specific execution address -func GetBuildersByExecutionAddress(address []byte) []*dbtypes.Builder { +func GetBuildersByExecutionAddress(ctx context.Context, address []byte) []*dbtypes.Builder { builders := []*dbtypes.Builder{} - err := ReaderDb.Select(&builders, ` + err := ReaderDb.SelectContext(ctx, &builders, ` SELECT * FROM builders WHERE execution_address = $1 ORDER BY builder_index ASC `, address) if err != nil { @@ -293,7 +294,7 @@ func GetBuildersByExecutionAddress(address []byte) []*dbtypes.Builder { } // GetBuilderIndexesByFilter returns builder indexes matching a filter -func GetBuilderIndexesByFilter(filter dbtypes.BuilderFilter, currentEpoch uint64) ([]uint64, error) { +func GetBuilderIndexesByFilter(ctx context.Context, filter dbtypes.BuilderFilter, currentEpoch uint64) ([]uint64, error) { var sql strings.Builder args := []interface{}{} fmt.Fprint(&sql, ` @@ -324,7 +325,7 @@ func GetBuilderIndexesByFilter(filter dbtypes.BuilderFilter, currentEpoch uint64 } builderIds := []uint64{} - err := ReaderDb.Select(&builderIds, sql.String(), args...) + err := ReaderDb.SelectContext(ctx, &builderIds, sql.String(), args...) if err != nil { logger.Errorf("Error while fetching builders by filter: %v", err) return nil, err @@ -379,7 +380,7 @@ func buildBuilderFilterSql(filter dbtypes.BuilderFilter, currentEpoch uint64, sq } // StreamBuildersByIndexes streams builders by indexes -func StreamBuildersByIndexes(indexes []uint64, cb func(builder *dbtypes.Builder) bool) { +func StreamBuildersByIndexes(ctx context.Context, indexes []uint64, cb func(builder *dbtypes.Builder) bool) { const batchSize = 1000 for i := 0; i < len(indexes); i += batchSize { @@ -412,7 +413,7 @@ func StreamBuildersByIndexes(indexes []uint64, cb func(builder *dbtypes.Builder) // Fetch all builders for this batch builders := make([]*dbtypes.Builder, len(batch)) - rows, err := ReaderDb.Query(sql.String(), args...) + rows, err := ReaderDb.QueryContext(ctx, sql.String(), args...) if err != nil { logger.Errorf("Error querying builders: %v", err) return diff --git a/handlers/builder.go b/handlers/builder.go index e13f45c17..19feb83fd 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -55,7 +55,7 @@ func BuilderDetail(w http.ResponseWriter, r *http.Request) { builder = services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(idx)) if builder == nil { // Try from DB - dbBuilder := db.GetActiveBuilderByIndex(idx) + dbBuilder := db.GetActiveBuilderByIndex(r.Context(), idx) if dbBuilder != nil { builder = beacon.UnwrapDbBuilder(dbBuilder) superseded = dbBuilder.Superseded @@ -64,7 +64,7 @@ func BuilderDetail(w http.ResponseWriter, r *http.Request) { } } else { // search by pubkey - dbBuilder := db.GetBuilderByPubkey(builderPubKey) + dbBuilder := db.GetBuilderByPubkey(r.Context(), builderPubKey) if dbBuilder != nil { builderIndex = dbBuilder.BuilderIndex superseded = dbBuilder.Superseded @@ -138,7 +138,7 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b builder := services.GlobalBeaconService.GetBuilderByIndex(gloas.BuilderIndex(builderIndex)) if builder == nil { // Try from DB - dbBuilder := db.GetActiveBuilderByIndex(builderIndex) + dbBuilder := db.GetActiveBuilderByIndex(ctx, builderIndex) if dbBuilder != nil { builder = beacon.UnwrapDbBuilder(dbBuilder) superseded = dbBuilder.Superseded @@ -231,7 +231,7 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta } // Batch fetch all bids for these block hashes - bidsMap := db.GetBidsByBlockHashes(blockHashes, builderIndex) + bidsMap := db.GetBidsByBlockHashes(ctx, blockHashes, builderIndex) // Build result blocks := make([]*models.BuilderPageDataBlock, 0, len(validBlocks)) @@ -261,7 +261,7 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta } func buildBuilderRecentBids(ctx context.Context, builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBid { - bids, _ := db.GetBidsByBuilderIndex(builderIndex, 0, 20) + bids, _ := db.GetBidsByBuilderIndex(ctx, builderIndex, 0, 20) result := make([]*models.BuilderPageDataBid, 0, len(bids)) for _, bid := range bids { diff --git a/handlers/builders.go b/handlers/builders.go index ad1d90075..b1ae7a563 100644 --- a/handlers/builders.go +++ b/handlers/builders.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -96,7 +97,7 @@ func getBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, f pageData := &models.BuildersPageData{} pageCacheKey := fmt.Sprintf("builders:%v:%v:%v:%v:%v:%v:%v", pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { - pageData, cacheTimeout := buildBuildersPageData(pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) + pageData, cacheTimeout := buildBuildersPageData(pageCall.CallCtx, pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) pageCall.CacheTimeout = cacheTimeout return pageData }) @@ -110,7 +111,7 @@ func getBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, f return pageData, pageErr } -func buildBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, filterPubKey string, filterIndex string, filterExecutionAddr string, filterStatus string) (*models.BuildersPageData, time.Duration) { +func buildBuildersPageData(ctx context.Context, pageNumber uint64, pageSize uint64, sortOrder string, filterPubKey string, filterIndex string, filterExecutionAddr string, filterStatus string) (*models.BuildersPageData, time.Duration) { logrus.Debugf("builders page called: %v:%v:%v:%v:%v:%v:%v", pageNumber, pageSize, sortOrder, filterPubKey, filterIndex, filterExecutionAddr, filterStatus) pageData := &models.BuildersPageData{} cacheTime := 10 * time.Minute @@ -189,7 +190,7 @@ func buildBuildersPageData(pageNumber uint64, pageSize uint64, sortOrder string, pageData.Sorting = sortOrder // get latest builder set - builderSetRsp, builderSetLen := services.GlobalBeaconService.GetFilteredBuilderSet(&builderFilter, true) + builderSetRsp, builderSetLen := services.GlobalBeaconService.GetFilteredBuilderSet(ctx, &builderFilter, true) if len(builderSetRsp) == 0 { cacheTime = 5 * time.Minute } diff --git a/handlers/search.go b/handlers/search.go index a0e9e7818..627d3a628 100644 --- a/handlers/search.go +++ b/handlers/search.go @@ -92,9 +92,9 @@ func buildSearchResolverResult(ctx context.Context, searchQuery string) (searchR } blockResult := &dbtypes.SearchBlockResult{} - err = db.ReaderDb.Get(blockResult, ` - SELECT slot, root, status - FROM slots + err = db.ReaderDb.GetContext(ctx, blockResult, ` + SELECT slot, root, status + FROM slots WHERE slot = $1 AND status != 0 LIMIT 1`, searchQuery) if err == nil { @@ -128,9 +128,9 @@ func buildSearchResolverResult(ctx context.Context, searchQuery string) (searchR blockHash, err := hex.DecodeString(hashQuery) if err == nil { blockResult := &dbtypes.SearchBlockResult{} - err = db.ReaderDb.Get(blockResult, ` - SELECT slot, root, orphaned - FROM slots + err = db.ReaderDb.GetContext(ctx, blockResult, ` + SELECT slot, root, orphaned + FROM slots WHERE root = $1 OR state_root = $1 LIMIT 1`, blockHash) @@ -149,7 +149,7 @@ func buildSearchResolverResult(ctx context.Context, searchQuery string) (searchR } names := &dbtypes.SearchNameResult{} - err = db.ReaderDb.Get(names, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.GetContext(ctx, names, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT name FROM validator_names @@ -166,7 +166,7 @@ func buildSearchResolverResult(ctx context.Context, searchQuery string) (searchR } graffiti := &dbtypes.SearchGraffitiResult{} - err = db.ReaderDb.Get(graffiti, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.GetContext(ctx, graffiti, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT graffiti FROM slots @@ -261,7 +261,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se switch searchType { case "epochs": dbres := &dbtypes.SearchAheadEpochsResult{} - err = db.ReaderDb.Select(dbres, "SELECT epoch FROM epochs WHERE CAST(epoch AS text) LIKE $1 ORDER BY epoch LIMIT 10", search+"%") + err = db.ReaderDb.SelectContext(ctx, dbres, "SELECT epoch FROM epochs WHERE CAST(epoch AS text) LIKE $1 ORDER BY epoch LIMIT 10", search+"%") if err == nil { model := make([]models.SearchAheadEpochsResult, len(*dbres)) for idx, entry := range *dbres { @@ -299,9 +299,9 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } } else { dbres := &dbtypes.SearchAheadSlotsResult{} - err = db.ReaderDb.Select(dbres, ` - SELECT slot, root, status - FROM slots + err = db.ReaderDb.SelectContext(ctx, dbres, ` + SELECT slot, root, status + FROM slots WHERE slot < $1 AND (root = $2 OR state_root = $2) ORDER BY slot LIMIT 1`, minSlotIdx, blockHash) if err != nil { @@ -335,9 +335,9 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se result = res } else { dbres := &dbtypes.SearchAheadSlotsResult{} - err = db.ReaderDb.Select(dbres, ` - SELECT slot, root, status - FROM slots + err = db.ReaderDb.SelectContext(ctx, dbres, ` + SELECT slot, root, status + FROM slots WHERE slot = $1 AND status != 0 ORDER BY slot LIMIT 10`, blockNumber) if err == nil { @@ -383,9 +383,9 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se result = res } else { dbres := &dbtypes.SearchAheadExecBlocksResult{} - err = db.ReaderDb.Select(dbres, ` - SELECT slot, root, eth_block_hash, eth_block_number, status - FROM slots + err = db.ReaderDb.SelectContext(ctx, dbres, ` + SELECT slot, root, eth_block_hash, eth_block_number, status + FROM slots WHERE slot < $1 AND eth_block_hash = $2 ORDER BY slot LIMIT 10`, minSlotIdx, blockHash) if err != nil { @@ -422,9 +422,9 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se result = res } else { dbres := &dbtypes.SearchAheadExecBlocksResult{} - err = db.ReaderDb.Select(dbres, ` - SELECT slot, root, eth_block_hash, eth_block_number, status - FROM slots + err = db.ReaderDb.SelectContext(ctx, dbres, ` + SELECT slot, root, eth_block_hash, eth_block_number, status + FROM slots WHERE slot < $1 AND eth_block_number = $2 ORDER BY slot LIMIT 10`, minSlotIdx, blockNumber) if err == nil { @@ -444,7 +444,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } case "graffiti": graffiti := &dbtypes.SearchAheadGraffitiResult{} - err = db.ReaderDb.Select(graffiti, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.SelectContext(ctx, graffiti, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT graffiti, count(*) as count FROM slots @@ -472,7 +472,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } case "valname": names := &dbtypes.SearchAheadValidatorNameResult{} - err = db.ReaderDb.Select(names, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.SelectContext(ctx, names, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT name, count(*) as count FROM validator_names @@ -523,7 +523,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } else if len(search) >= 2 && len(search) <= 96 { // Search by pubkey prefix validators := &dbtypes.SearchAheadValidatorResult{} - err = db.ReaderDb.Select(validators, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.SelectContext(ctx, validators, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT v.validator_index, v.pubkey FROM validators v @@ -573,7 +573,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } else if len(search) >= 2 && len(search) < 40 { // Search by address prefix in DB addresses := &dbtypes.SearchAheadAddressResult{} - err = db.ReaderDb.Select(addresses, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.SelectContext(ctx, addresses, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT address, is_contract FROM el_accounts @@ -626,7 +626,7 @@ func buildSearchAheadResult(ctx context.Context, searchType, search string) (*se } else if len(search) >= 2 && len(search) < 64 { // Search by transaction hash prefix in DB transactions := &dbtypes.SearchAheadTransactionResult{} - err = db.ReaderDb.Select(transactions, db.EngineQuery(map[dbtypes.DBEngineType]string{ + err = db.ReaderDb.SelectContext(ctx, transactions, db.EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` SELECT DISTINCT ON (tx_hash) tx_hash, block_number, reverted FROM el_transactions diff --git a/indexer/beacon/bidcache.go b/indexer/beacon/bidcache.go index 32d23ab41..3cdbe4bc5 100644 --- a/indexer/beacon/bidcache.go +++ b/indexer/beacon/bidcache.go @@ -55,7 +55,7 @@ func (cache *blockBidCache) loadFromDB(currentSlot phase0.Slot) { minSlot = currentSlot - bidCacheRetainSlots } - dbBids := db.GetBidsForSlotRange(uint64(minSlot)) + dbBids := db.GetBidsForSlotRange(cache.indexer.ctx, uint64(minSlot)) for _, bid := range dbBids { key := bidCacheKey{ ParentRoot: phase0.Root(bid.ParentRoot), diff --git a/indexer/beacon/buildercache.go b/indexer/beacon/buildercache.go index 1a8114ae3..14d506aa9 100644 --- a/indexer/beacon/buildercache.go +++ b/indexer/beacon/buildercache.go @@ -480,7 +480,7 @@ func (cache *builderCache) getBuilderByIndexAndRoot(index gloas.BuilderIndex, bl // Fallback to db if builder is not found in cache if builder == nil { - if dbBuilder := db.GetActiveBuilderByIndex(uint64(index)); dbBuilder != nil { + if dbBuilder := db.GetActiveBuilderByIndex(cache.indexer.ctx, uint64(index)); dbBuilder != nil { builder = UnwrapDbBuilder(dbBuilder) } } else { @@ -519,7 +519,7 @@ func (cache *builderCache) prepopulateFromDB() (uint64, error) { cache.cacheMutex.Lock() defer cache.cacheMutex.Unlock() - maxIndex, err := db.GetMaxBuilderIndex() + maxIndex, err := db.GetMaxBuilderIndex(cache.indexer.ctx) if err != nil { return 0, fmt.Errorf("error getting max builder index: %w", err) } @@ -536,7 +536,7 @@ func (cache *builderCache) prepopulateFromDB() (uint64, error) { for start := uint64(0); start <= maxIndex; start += batchSize { end := min(start+batchSize, maxIndex) - builders := db.GetBuilderRange(start, end) + builders := db.GetBuilderRange(cache.indexer.ctx, start, end) for _, dbBuilder := range builders { if dbBuilder.Superseded { continue @@ -713,7 +713,7 @@ func (cache *builderCache) persistBuilderBatch(tx *sqlx.Tx, batch []*dbtypes.Bui } // Fetch existing builders in this batch's range - existingBuilders := db.GetBuilderRange(minIndex, maxIndex) + existingBuilders := db.GetBuilderRange(cache.indexer.ctx, minIndex, maxIndex) existingByIndex := make(map[uint64]*dbtypes.Builder, len(existingBuilders)) for _, b := range existingBuilders { existingByIndex[b.BuilderIndex] = b diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index c4d40922f..c1d85d842 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -520,7 +520,7 @@ func (indexer *Indexer) GetBlockBids(parentBlockRoot phase0.Root) []*dbtypes.Blo } // Fall back to database - return db.GetBidsForBlockRoot(parentBlockRoot[:]) + return db.GetBidsForBlockRoot(indexer.ctx, parentBlockRoot[:]) } // StreamActiveBuilderDataForRoot streams the available builder set data for a given blockRoot. diff --git a/services/chainservice_builder.go b/services/chainservice_builder.go index 6d8dc3a22..49a4c9162 100644 --- a/services/chainservice_builder.go +++ b/services/chainservice_builder.go @@ -2,6 +2,7 @@ package services import ( "bytes" + "context" "slices" "sort" @@ -24,7 +25,7 @@ type BuilderWithIndex struct { } // GetFilteredBuilderSet returns builders matching the filter criteria -func (bs *ChainService) GetFilteredBuilderSet(filter *dbtypes.BuilderFilter, withBalance bool) ([]BuilderWithIndex, uint64) { +func (bs *ChainService) GetFilteredBuilderSet(ctx context.Context, filter *dbtypes.BuilderFilter, withBalance bool) ([]BuilderWithIndex, uint64) { var overrideForkId *beacon.ForkKey canonicalHead := bs.beaconIndexer.GetCanonicalHead(overrideForkId) @@ -81,7 +82,7 @@ func (bs *ChainService) GetFilteredBuilderSet(filter *dbtypes.BuilderFilter, wit }) // Get matching entries from DB - dbIndexes, err := db.GetBuilderIndexesByFilter(*filter, uint64(currentEpoch)) + dbIndexes, err := db.GetBuilderIndexesByFilter(ctx, *filter, uint64(currentEpoch)) if err != nil { bs.logger.Warnf("error getting builder indexes by filter: %v", err) return nil, 0 @@ -171,7 +172,7 @@ func (bs *ChainService) GetFilteredBuilderSet(filter *dbtypes.BuilderFilter, wit resultCount := uint64(0) dbEntryCount := uint64(0) - db.StreamBuildersByIndexes(dbIndexes, func(dbBuilder *dbtypes.Builder) bool { + db.StreamBuildersByIndexes(ctx, dbIndexes, func(dbBuilder *dbtypes.Builder) bool { dbEntryCount++ builderWithIndex := BuilderWithIndex{ Index: gloas.BuilderIndex(dbBuilder.BuilderIndex), From 9edd06f8c3a4ab70798145d102fe6f8a7cb50df9 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 25 Feb 2026 16:06:50 +0100 Subject: [PATCH 48/99] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- services/chainservice_blocks.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 20f3d0a71..155394d18 100644 --- a/go.mod +++ b/go.mod @@ -256,4 +256,4 @@ require ( modernc.org/sqlite v1.46.1 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260225144847-75b86704f554 diff --git a/go.sum b/go.sum index ffb3d0b19..91bb96335 100644 --- a/go.sum +++ b/go.sum @@ -563,8 +563,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.1 h1:84eNMiiOYDiNC2Y1m5A/UtIPs6u/9SsvG4RVSBRGE5U= github.com/pk910/dynamic-ssz v1.2.1/go.mod h1:HXRWLNcgj3DL65Kznrb+RdL3DEKw2JBZ/6crooqGoII= -github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f h1:OjX1YemePnLes4JqeDFIJOZN9YPCc4R6cyvyXFtVA6c= -github.com/pk910/go-eth2-client v0.0.0-20260219114320-6080c2df7e2f/go.mod h1:8fpxrIBBVbOcVG3vcHe5ubOHIeqW3N5t7kS4oU5EeJU= +github.com/pk910/go-eth2-client v0.0.0-20260225144847-75b86704f554 h1:FmusNWzB2XDzRQK1OoLo9XUH/PBatWPkztJOX/Ther4= +github.com/pk910/go-eth2-client v0.0.0-20260225144847-75b86704f554/go.mod h1:8fpxrIBBVbOcVG3vcHe5ubOHIeqW3N5t7kS4oU5EeJU= github.com/pk910/hashtree-bindings v0.0.1 h1:Sw+UlPlrBle4LUg04kqLFybVQcfmamwKL1QsrR3GU0g= github.com/pk910/hashtree-bindings v0.0.1/go.mod h1:eayIpxMFkWzMsydESu/5bV8wglZzSE/c9mq6DQdn204= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 3829b8b33..de3f52549 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -292,7 +292,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl log.Warnf("Error loading block body for slot %v", slot) } - if block.Version >= spec.DataVersionGloas { + if block != nil && block.Version >= spec.DataVersionGloas { payload, err = beacon.LoadExecutionPayload(ctx, client, blockRoot) if payload != nil { break From ffa4f39191af900755253fa2604555bc1a9ca0f4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 4 Mar 2026 15:55:27 +0100 Subject: [PATCH 49/99] fix epbs db schema for pgsql --- .../pgsql/20260108202212_epbs-payload.sql | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/db/schema/pgsql/20260108202212_epbs-payload.sql b/db/schema/pgsql/20260108202212_epbs-payload.sql index fdec29e44..4a9eaf95c 100644 --- a/db/schema/pgsql/20260108202212_epbs-payload.sql +++ b/db/schema/pgsql/20260108202212_epbs-payload.sql @@ -1,18 +1,18 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE public."unfinalized_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" bytea NULL; +ALTER TABLE public."unfinalized_blocks" + ADD COLUMN "payload_ver" int NOT NULL DEFAULT 0, + ADD COLUMN "payload_ssz" bytea NULL; -ALTER TABLE public."orphaned_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" bytea NULL; +ALTER TABLE public."orphaned_blocks" + ADD COLUMN "payload_ver" int NOT NULL DEFAULT 0, + ADD COLUMN "payload_ssz" bytea NULL; -ALTER TABLE public."slots" ADD - "payload_status" smallint NOT NULL DEFAULT 0, - "builder_index" bigint NOT NULL DEFAULT -1, - "eth_block_parent_hash" bytea NULL; +ALTER TABLE public."slots" + ADD COLUMN "payload_status" smallint NOT NULL DEFAULT 0, + ADD COLUMN "builder_index" bigint NOT NULL DEFAULT -1, + ADD COLUMN "eth_block_parent_hash" bytea NULL; CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON public."slots" @@ -26,11 +26,11 @@ CREATE INDEX IF NOT EXISTS "slots_builder_index_idx" ON public."slots" ("builder_index" ASC NULLS LAST); -ALTER TABLE public."epochs" ADD - "payload_count" int NOT NULL DEFAULT 0; +ALTER TABLE public."epochs" + ADD COLUMN "payload_count" int NOT NULL DEFAULT 0; -ALTER TABLE public."unfinalized_epochs" ADD - "payload_count" int NOT NULL DEFAULT 0; +ALTER TABLE public."unfinalized_epochs" + ADD COLUMN "payload_count" int NOT NULL DEFAULT 0; CREATE TABLE IF NOT EXISTS public."block_bids" ( "parent_root" bytea NOT NULL, From 32e0c5bd6d2e203ac5603c566a06275287d1d175 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 4 Mar 2026 17:40:40 +0100 Subject: [PATCH 50/99] fix synchronizing gloas epochs --- indexer/beacon/epochstate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 53cd35f47..db73fdb39 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -10,6 +10,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" ) // epochState represents a beacon state which a epoch status depends on. @@ -146,7 +147,7 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } } - err = s.processState(resState, beaconBlock, executionPayload, cache) + err = s.processState(resState, beaconBlock, executionPayload, cache, client.indexer.consensusPool.GetChainState().GetSpecs()) if err != nil { return nil, err } @@ -164,7 +165,7 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch // processState processes the state and updates the epochState instance. // the function extracts and unifies all relevant information from the beacon state, so the full beacon state can be dropped from memory afterwards. -func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, executionPayload *gloas.SignedExecutionPayloadEnvelope, cache *epochCache) error { +func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, executionPayload *gloas.SignedExecutionPayloadEnvelope, cache *epochCache, specs *consensus.ChainSpec) error { slot, err := state.Slot() if err != nil { return fmt.Errorf("error getting slot from state %v: %v", s.slotRoot.String(), err) @@ -179,7 +180,6 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock return fmt.Errorf("error getting block roots from state %v: %v", s.slotRoot.String(), err) } - specs := cache.indexer.consensusPool.GetChainState().GetSpecs() dependentRoot = blockRoots[slot%phase0.Slot(specs.SlotsPerHistoricalRoot)] } From 45963626a7ec624475ed4b9fd6fa5040463fe1e7 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 4 Mar 2026 17:54:34 +0100 Subject: [PATCH 51/99] fix nil panic in EpochStats pre-computation --- indexer/beacon/epochstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index 234c383dc..8f08fd53a 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -587,7 +587,7 @@ func (es *EpochStats) precomputeFromParentState(indexer *Indexer, parentState *E proposerDuties := []phase0.ValidatorIndex{} specs := chainState.GetSpecs() - if uint64(len(parentState.dependentState.proposerLookahead)) > specs.SlotsPerEpoch { + if parentState.dependentState != nil && uint64(len(parentState.dependentState.proposerLookahead)) > specs.SlotsPerEpoch { proposerDuties = parentState.dependentState.proposerLookahead[specs.SlotsPerEpoch:] } else { for slot := chainState.EpochToSlot(es.epoch); slot < chainState.EpochToSlot(es.epoch+1); slot++ { From a6ab6448babb0d5d982719ec092504d29f5a3120 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 5 Mar 2026 04:16:35 +0100 Subject: [PATCH 52/99] fix SQL error --- db/unfinalized_blocks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/unfinalized_blocks.go b/db/unfinalized_blocks.go index fbb47253e..668797776 100644 --- a/db/unfinalized_blocks.go +++ b/db/unfinalized_blocks.go @@ -189,7 +189,7 @@ func GetUnfinalizedBlock(ctx context.Context, root []byte, withHeader bool, with fmt.Fprint(&sql, `, status, fork_id, recv_delay, min_exec_time, max_exec_time, exec_times, block_uid`) - fmt.Fprint(&sql, `FROM unfinalized_blocks WHERE root = $1`) + fmt.Fprint(&sql, ` FROM unfinalized_blocks WHERE root = $1`) block := dbtypes.UnfinalizedBlock{} err := ReaderDb.GetContext(ctx, &block, sql.String(), root) From 93c78ea1da02efa3093c47ae4debd995abf246c4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 5 Mar 2026 19:08:08 +0100 Subject: [PATCH 53/99] show orphaned payload status on slot details page --- handlers/slot.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/handlers/slot.go b/handlers/slot.go index 755b57bff..4dd400e09 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -1,6 +1,7 @@ package handlers import ( + "bytes" "context" "encoding/hex" "encoding/json" @@ -766,7 +767,30 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock Gloas: blockData.Payload.Message.Payload, } - pageData.PayloadHeader.PayloadStatus = uint16(1) + // Determine payload status by checking if any canonical child + // builds on this block's execution payload. + pageData.PayloadHeader.PayloadStatus = uint16(dbtypes.PayloadStatusCanonical) + childSlots := services.GlobalBeaconService.GetDbBlocksByParentRoot(ctx, blockData.Root) + hasCanonicalChild := false + payloadIncluded := false + + for _, child := range childSlots { + if child.Status != dbtypes.Canonical { + continue + } + + hasCanonicalChild = true + + if bytes.Equal(child.EthBlockParentHash, pageData.PayloadHeader.BlockHash) { + payloadIncluded = true + + break + } + } + + if hasCanonicalChild && !payloadIncluded { + pageData.PayloadHeader.PayloadStatus = uint16(dbtypes.PayloadStatusOrphaned) + } } else { executionPayload, _ = blockData.Block.ExecutionPayload() } From 25ab74adbc12afcf2558b76be41873316c5bc8fc Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 6 Mar 2026 10:56:57 +0100 Subject: [PATCH 54/99] fix: add missing SQL placeholder for orphaned blocks insert The InsertOrphanedBlock SQL had 8 columns but only 7 value placeholders, causing epoch finalization to fail with SQLSTATE 42601 whenever an orphaned block was encountered. Co-Authored-By: Claude Opus 4.6 --- db/orphaned_blocks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/orphaned_blocks.go b/db/orphaned_blocks.go index ae93d4e86..3f6eddd4d 100644 --- a/db/orphaned_blocks.go +++ b/db/orphaned_blocks.go @@ -12,12 +12,12 @@ func InsertOrphanedBlock(ctx context.Context, tx *sqlx.Tx, block *dbtypes.Orphan dbtypes.DBEnginePgsql: ` INSERT INTO orphaned_blocks ( root, header_ver, header_ssz, block_ver, block_ssz, block_uid, payload_ver, payload_ssz - ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (root) DO NOTHING`, dbtypes.DBEngineSqlite: ` INSERT OR IGNORE INTO orphaned_blocks ( root, header_ver, header_ssz, block_ver, block_ssz, block_uid, payload_ver, payload_ssz - ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, }), block.Root, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.BlockUid, block.PayloadVer, block.PayloadSSZ) if err != nil { From 3c0b826b59338b824d57622efb02ced546a63b32 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 6 Mar 2026 11:36:20 +0100 Subject: [PATCH 55/99] fix: use correct dependent root for Fulu+ validator cache tracking For Fulu+ forks, the dependent root passed to updateValidatorSet was calculated from blockRoots[slot % SlotsPerHistoricalRoot], which resolved to a stale block root from ~8192 slots ago not present in the block cache. This caused isCanonicalBlock() to fail for all validator diffs, making latestValidator nil and the validators page empty. Store the epoch stats' original dependent root (parent block root) on the epochState and use it directly instead of the incorrect block_roots lookup. Co-Authored-By: Claude Opus 4.6 --- indexer/beacon/epochcache.go | 1 + indexer/beacon/epochstate.go | 16 ++++++---------- indexer/beacon/synchronizer.go | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index f3a22426a..4b8531c19 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -98,6 +98,7 @@ func (cache *epochCache) ensureEpochDependentState(epochStats *EpochStats, first } epochState = newEpochState(stateRoot) + epochState.dependentRoot = epochStats.dependentRoot cache.stateMap[epochStats.dependentRoot] = epochState cache.indexer.logger.Infof("added epoch state request for epoch %v (%v) to queue", epochStats.epoch, epochStats.dependentRoot.String()) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index db73fdb39..eed389cd3 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -15,8 +15,9 @@ import ( // epochState represents a beacon state which a epoch status depends on. type epochState struct { - slotRoot phase0.Root - stateRoot phase0.Root + slotRoot phase0.Root + stateRoot phase0.Root + dependentRoot phase0.Root loadingCancel context.CancelFunc loadingStatus uint8 @@ -173,14 +174,9 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock s.stateSlot = slot - dependentRoot := s.slotRoot - if state.Version >= spec.DataVersionFulu { - blockRoots, err := getStateBlockRoots(state) - if err != nil { - return fmt.Errorf("error getting block roots from state %v: %v", s.slotRoot.String(), err) - } - - dependentRoot = blockRoots[slot%phase0.Slot(specs.SlotsPerHistoricalRoot)] + dependentRoot := s.dependentRoot + if dependentRoot == (phase0.Root{}) { + dependentRoot = s.slotRoot } validatorList, err := state.Validators() diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 76ca8f687..da6db19bd 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -383,6 +383,7 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry } epochState := newEpochState(dependentRoot) + epochState.dependentRoot = dependentRoot t1 := time.Now() state, err := epochState.loadState(s.syncCtx, client, nil) loadDuration := time.Since(t1) From b6e32ac1f6d277f5f481de50b9421600bf75684a Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 6 Mar 2026 14:19:48 +0100 Subject: [PATCH 56/99] fixes --- indexer/beacon/epochcache.go | 1 - indexer/beacon/epochstate.go | 16 ++++++++++------ indexer/beacon/synchronizer.go | 1 - 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index 4b8531c19..f3a22426a 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -98,7 +98,6 @@ func (cache *epochCache) ensureEpochDependentState(epochStats *EpochStats, first } epochState = newEpochState(stateRoot) - epochState.dependentRoot = epochStats.dependentRoot cache.stateMap[epochStats.dependentRoot] = epochState cache.indexer.logger.Infof("added epoch state request for epoch %v (%v) to queue", epochStats.epoch, epochStats.dependentRoot.String()) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index eed389cd3..07b62de5d 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -15,9 +15,8 @@ import ( // epochState represents a beacon state which a epoch status depends on. type epochState struct { - slotRoot phase0.Root - stateRoot phase0.Root - dependentRoot phase0.Root + slotRoot phase0.Root + stateRoot phase0.Root loadingCancel context.CancelFunc loadingStatus uint8 @@ -174,9 +173,14 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock s.stateSlot = slot - dependentRoot := s.dependentRoot - if dependentRoot == (phase0.Root{}) { - dependentRoot = s.slotRoot + dependentRoot := s.slotRoot + if state.Version >= spec.DataVersionFulu { + blockRoots, err := getStateBlockRoots(state) + if err != nil { + return fmt.Errorf("error getting block roots from state %v: %v", s.slotRoot.String(), err) + } + + dependentRoot = blockRoots[(slot-1)%phase0.Slot(specs.SlotsPerHistoricalRoot)] } validatorList, err := state.Validators() diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index da6db19bd..76ca8f687 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -383,7 +383,6 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry } epochState := newEpochState(dependentRoot) - epochState.dependentRoot = dependentRoot t1 := time.Now() state, err := epochState.loadState(s.syncCtx, client, nil) loadDuration := time.Since(t1) From 5dda8e6a0943a2fd7cd51aba8bcc1ba5a7fb8230 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 6 Mar 2026 15:24:46 +0100 Subject: [PATCH 57/99] fix: use state.latest_block_header.parent_root for Fulu+ dependent root The previous approach using block_roots[slot % N] returned stale values because process_slots never writes block_roots[target_slot % N]. Using state.latest_block_header.parent_root is correct since dora loads state from the first block of the epoch, making the parent guaranteed to be the last block of the previous epoch (the dependent root). Co-Authored-By: Claude Opus 4.6 --- indexer/beacon/block_helper.go | 21 +++++++++++++++++++++ indexer/beacon/epochstate.go | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 979579a0c..64b57c84f 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -766,6 +766,27 @@ func getStateBlockRoots(v *spec.VersionedBeaconState) ([]phase0.Root, error) { } } +// getLatestBlockHeaderParentRoot returns the parent root from the latest block header in the state. +// For Fulu+ states loaded from the first block of an epoch, this is the dependent root (last block of the previous epoch). +func getLatestBlockHeaderParentRoot(v *spec.VersionedBeaconState) (phase0.Root, error) { + switch v.Version { + case spec.DataVersionFulu: + if v.Fulu == nil || v.Fulu.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no fulu state") + } + + return v.Fulu.LatestBlockHeader.ParentRoot, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no gloas state") + } + + return v.Gloas.LatestBlockHeader.ParentRoot, nil + default: + return phase0.Root{}, errors.New("unknown version") + } +} + // getBlockSize returns the block size from a versioned beacon block. func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) (int, error) { switch block.Version { diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 07b62de5d..6ab42317b 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -175,12 +175,12 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock dependentRoot := s.slotRoot if state.Version >= spec.DataVersionFulu { - blockRoots, err := getStateBlockRoots(state) + parentRoot, err := getLatestBlockHeaderParentRoot(state) if err != nil { - return fmt.Errorf("error getting block roots from state %v: %v", s.slotRoot.String(), err) + return fmt.Errorf("error getting latest block header parent root from state %v: %v", s.slotRoot.String(), err) } - dependentRoot = blockRoots[(slot-1)%phase0.Slot(specs.SlotsPerHistoricalRoot)] + dependentRoot = parentRoot } validatorList, err := state.Validators() From 658ddc168dc68f381faa46fba1b3a7e811a4690f Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 6 Mar 2026 15:38:26 +0100 Subject: [PATCH 58/99] clean up block helpers --- indexer/beacon/block_helper.go | 62 +++++++++------------------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 64b57c84f..39deea640 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -709,67 +709,37 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator } } -// getStateProposerLookahead returns the proposer lookahead from a versioned beacon state. -func getStateBlockRoots(v *spec.VersionedBeaconState) ([]phase0.Root, error) { +// getLatestBlockHeaderParentRoot returns the parent root from the latest block header in the state. +func getLatestBlockHeaderParentRoot(v *spec.VersionedBeaconState) (phase0.Root, error) { switch v.Version { - case spec.DataVersionPhase0: - if v.Phase0 == nil || v.Phase0.BlockRoots == nil { - return nil, errors.New("no phase0 block") - } - - return v.Phase0.BlockRoots, nil + return phase0.Root{}, errors.New("no latest block header parent root in phase0 state") case spec.DataVersionAltair: - if v.Altair == nil || v.Altair.BlockRoots == nil { - return nil, errors.New("no altair block") - } - - return v.Altair.BlockRoots, nil + return phase0.Root{}, errors.New("no latest block header parent root in altair state") case spec.DataVersionBellatrix: - if v.Bellatrix == nil || v.Bellatrix.BlockRoots == nil { - return nil, errors.New("no bellatrix block") + if v.Bellatrix == nil || v.Bellatrix.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no bellatrix state") } - return v.Bellatrix.BlockRoots, nil + return v.Bellatrix.LatestBlockHeader.ParentRoot, nil case spec.DataVersionCapella: - if v.Capella == nil || v.Capella.BlockRoots == nil { - return nil, errors.New("no capella block") + if v.Capella == nil || v.Capella.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no capella state") } - return v.Capella.BlockRoots, nil + return v.Capella.LatestBlockHeader.ParentRoot, nil case spec.DataVersionDeneb: - if v.Deneb == nil || v.Deneb.BlockRoots == nil { - return nil, errors.New("no deneb block") + if v.Deneb == nil || v.Deneb.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no deneb state") } - return v.Deneb.BlockRoots, nil + return v.Deneb.LatestBlockHeader.ParentRoot, nil case spec.DataVersionElectra: - if v.Electra == nil || v.Electra.BlockRoots == nil { - return nil, errors.New("no electra block") - } - - return v.Electra.BlockRoots, nil - case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.BlockRoots == nil { - return nil, errors.New("no fulu block") - } - - return v.Fulu.BlockRoots, nil - case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.BlockRoots == nil { - return nil, errors.New("no gloas block") + if v.Electra == nil || v.Electra.LatestBlockHeader == nil { + return phase0.Root{}, errors.New("no electra state") } - return v.Gloas.BlockRoots, nil - default: - return nil, errors.New("unknown version") - } -} - -// getLatestBlockHeaderParentRoot returns the parent root from the latest block header in the state. -// For Fulu+ states loaded from the first block of an epoch, this is the dependent root (last block of the previous epoch). -func getLatestBlockHeaderParentRoot(v *spec.VersionedBeaconState) (phase0.Root, error) { - switch v.Version { + return v.Electra.LatestBlockHeader.ParentRoot, nil case spec.DataVersionFulu: if v.Fulu == nil || v.Fulu.LatestBlockHeader == nil { return phase0.Root{}, errors.New("no fulu state") From d0181ac32d31a42a0fa619d53e2e3f2b68169ed7 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 9 Mar 2026 19:36:54 +0100 Subject: [PATCH 59/99] fixed validator & builder set finalization --- indexer/beacon/buildercache.go | 7 ++++--- indexer/beacon/client.go | 5 +++++ indexer/beacon/finalization.go | 8 +++----- indexer/beacon/validatorcache.go | 10 +++++++--- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/indexer/beacon/buildercache.go b/indexer/beacon/buildercache.go index 14d506aa9..188c5aa6f 100644 --- a/indexer/beacon/buildercache.go +++ b/indexer/beacon/buildercache.go @@ -290,8 +290,9 @@ func (cache *builderCache) getBuilderSetSize() uint64 { return uint64(len(cache.builderSetCache)) } -// setFinalizedEpoch updates the builder cache when a new epoch is finalized -func (cache *builderCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDependentRoot phase0.Root) { +// setFinalizedEpoch updates the builder cache when a new epoch is finalized. +// dependentRoot is the dependent root of the finalized epoch (last block of the parent epoch). +func (cache *builderCache) setFinalizedEpoch(epoch phase0.Epoch, dependentRoot phase0.Root) { cache.cacheMutex.Lock() defer cache.cacheMutex.Unlock() @@ -304,7 +305,7 @@ func (cache *builderCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDepend // Find the finalized builder state for _, diff := range cachedBuilder.builderDiffs { - if diff.dependentRoot == nextEpochDependentRoot { + if diff.dependentRoot == dependentRoot { cachedBuilder.finalBuilder = diff.builder cachedBuilder.finalChecksum = calculateBuilderChecksum(diff.builder) cachedBuilder.statusFlags = GetBuilderStatusFlags(diff.builder) diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 42651be56..2d724c508 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -285,6 +285,11 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { isEpochStart = true } else if parentBlock != nil && chainState.EpochOfSlot(parentBlock.Slot) < currentEpoch { isEpochStart = true + } else if parentBlock == nil && chainState.EpochOfSlot(currentBlock.Slot) == currentEpoch { + // parent block is not in cache, but we're still in currentEpoch. + // this block is the oldest block in cache for this epoch, so treat its + // parent root as the dependent root for epoch boundary detection. + isEpochStart = true } if isEpochStart { diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index dc210993f..95fcc7343 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -602,11 +602,9 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R t1 = time.Now() - // update validator cache - if len(canonicalBlocks) > 0 { - indexer.validatorCache.setFinalizedEpoch(epoch, canonicalBlocks[len(canonicalBlocks)-1].Root) - indexer.builderCache.setFinalizedEpoch(epoch, canonicalBlocks[len(canonicalBlocks)-1].Root) - } + // update validator & builder cache with the epoch's dependent root (last block of parent epoch) + indexer.validatorCache.setFinalizedEpoch(epoch, dependentRoot) + indexer.builderCache.setFinalizedEpoch(epoch, dependentRoot) // clean fork cache indexer.forkCache.setFinalizedEpoch(deleteBeforeSlot, justifiedRoot) diff --git a/indexer/beacon/validatorcache.go b/indexer/beacon/validatorcache.go index 54e3b86cf..43f593136 100644 --- a/indexer/beacon/validatorcache.go +++ b/indexer/beacon/validatorcache.go @@ -345,8 +345,9 @@ func (cache *validatorCache) getValidatorFlags(validatorIndex phase0.ValidatorIn return cache.valsetCache[validatorIndex].statusFlags } -// setFinalizedEpoch sets the last finalized epoch and updates the validator set -func (cache *validatorCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDependentRoot phase0.Root) { +// setFinalizedEpoch sets the last finalized epoch and updates the validator set. +// dependentRoot is the dependent root of the finalized epoch (last block of the parent epoch). +func (cache *validatorCache) setFinalizedEpoch(epoch phase0.Epoch, dependentRoot phase0.Root) { cache.cacheMutex.Lock() defer cache.cacheMutex.Unlock() @@ -361,7 +362,7 @@ func (cache *validatorCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDepe // Find the finalized validator state for _, diff := range cachedValidator.validatorDiffs { - if diff.dependentRoot == nextEpochDependentRoot { + if diff.dependentRoot == dependentRoot { cachedValidator.finalValidator = diff.validator cachedValidator.finalChecksum = calculateValidatorChecksum(diff.validator) cachedValidator.statusFlags = GetValidatorStatusFlags(diff.validator) @@ -398,6 +399,9 @@ func (cache *validatorCache) setFinalizedEpoch(epoch phase0.Epoch, nextEpochDepe cache.lastFinalizedActiveCount = activeCount + cache.indexer.logger.Infof("finalized validator set for epoch %v (dependent root: %v, updated: %v, total: %v)", + epoch, dependentRoot.String(), updatedCount, len(cache.valsetCache)) + if updatedCount > 0 { select { case cache.triggerDbUpdate <- true: From 2e70329b2b4cbd88ec89c553ad4a13d1712a42ef Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 9 Mar 2026 19:41:56 +0100 Subject: [PATCH 60/99] persist execution payloads with unfinalized blocks to survive restarts --- indexer/beacon/block.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 55c185737..b21d6a6d4 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -512,7 +512,7 @@ func (block *Block) buildUnfinalizedBlock(ctx context.Context, compress bool) (* return nil, fmt.Errorf("marshal exec times ssz failed: %v", err) } - return &dbtypes.UnfinalizedBlock{ + unfinalizedBlock := &dbtypes.UnfinalizedBlock{ Root: block.Root[:], Slot: uint64(block.Slot), HeaderVer: 1, @@ -526,7 +526,18 @@ func (block *Block) buildUnfinalizedBlock(ctx context.Context, compress bool) (* MaxExecTime: uint32(block.maxExecutionTime), ExecTimes: execTimesSSZ, BlockUid: block.BlockUID, - }, nil + } + + if block.executionPayload != nil { + payloadVer, payloadSSZ, err := MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) + if err != nil { + return nil, fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + unfinalizedBlock.PayloadVer = payloadVer + unfinalizedBlock.PayloadSSZ = payloadSSZ + } + + return unfinalizedBlock, nil } // buildOrphanedBlock builds an orphaned block from the block data. From d59d03b4d91ae2a9587321cb354655c101c4b640 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 9 Mar 2026 22:06:43 +0100 Subject: [PATCH 61/99] render payload status for gloas+ attestations --- handlers/slot.go | 5 +++++ templates/slot/attestations.html | 19 ++++++++++++++++++- types/models/slot.go | 2 ++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/handlers/slot.go b/handlers/slot.go index 4dd400e09..c7c847084 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -526,6 +526,11 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock includedValidators := []uint64{} attEpochStatsValues := assignmentsMap[attEpoch] + if attVersioned.Version >= spec.DataVersionGloas { + payloadStatus := uint64(attData.Index) + attPageData.PayloadStatus = &payloadStatus + } + if attVersioned.Version >= spec.DataVersionElectra { // EIP-7549 attestation attAssignments = []uint64{} diff --git a/templates/slot/attestations.html b/templates/slot/attestations.html index 57612cf97..783d355f4 100644 --- a/templates/slot/attestations.html +++ b/templates/slot/attestations.html @@ -12,9 +12,25 @@
    Slot:
    -
    +
    +
    + {{ html "" }} + Payload Status: + {{ html "" }} +
    +
    + {{ html "" }} + FULL + {{ html "" }} + {{ html "" }} + EMPTY + {{ html "" }} + {{ html "" }} + UNKNOWN + {{ html "" }} +
    @@ -177,6 +193,7 @@ self.signature = base64ToBytes(data.signature); self.validators = data.validators; self.included_validators = data.included_validators; + self.payload_status = data.payload_status !== undefined ? data.payload_status : null; self.showDetails = ko.observable(false); diff --git a/types/models/slot.go b/types/models/slot.go index d94c6dc4f..3e712db77 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -147,6 +147,8 @@ type SlotPageAttestation struct { Validators []uint64 `json:"validators"` IncludedValidators []uint64 `json:"included_validators"` + PayloadStatus *uint64 `json:"payload_status,omitempty"` + Signature []byte `json:"signature"` BeaconBlockRoot []byte `json:"beaconblockroot"` From 19ed4eb8b0d316a602074707537c38c86ad61906 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 10 Mar 2026 03:03:02 +0100 Subject: [PATCH 62/99] fix orphaned payload status for unfinalized blocks in list views --- db/slots.go | 24 +++++- dbtypes/other.go | 12 ++- indexer/beacon/block.go | 5 ++ indexer/beacon/canonical.go | 4 + indexer/beacon/finalization.go | 47 ++++++----- services/chainservice_blocks.go | 138 +++++++++++++++++++------------- 6 files changed, 152 insertions(+), 78 deletions(-) diff --git a/db/slots.go b/db/slots.go index 33977da9d..4968f0802 100644 --- a/db/slots.go +++ b/db/slots.go @@ -483,10 +483,26 @@ func GetFilteredSlots(ctx context.Context, filter *dbtypes.BlockFilter, firstSlo fmt.Fprintf(&sql, ` AND slots.builder_index = $%v `, argIdx) args = append(args, *filter.BuilderIndex) } - if filter.WithPayloadOrphaned == 0 { - fmt.Fprintf(&sql, ` AND slots.payload_status != 2 `) - } else if filter.WithPayloadOrphaned == 2 { - fmt.Fprintf(&sql, ` AND slots.payload_status = 2 `) + + if filter.WithPayloadMask != dbtypes.PayloadStatusMaskAll { + allowedPayloadStatuses := []dbtypes.PayloadStatus{} + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskMissing != 0 { + allowedPayloadStatuses = append(allowedPayloadStatuses, dbtypes.PayloadStatusMissing) + } + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskCanonical != 0 { + allowedPayloadStatuses = append(allowedPayloadStatuses, dbtypes.PayloadStatusCanonical) + } + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskOrphaned != 0 { + allowedPayloadStatuses = append(allowedPayloadStatuses, dbtypes.PayloadStatusOrphaned) + } + + if len(allowedPayloadStatuses) > 0 { + allowedPayloadStatusesPlaceholders := make([]string, len(allowedPayloadStatuses)) + for i, payloadStatus := range allowedPayloadStatuses { + allowedPayloadStatusesPlaceholders[i] = fmt.Sprintf("%v", payloadStatus) + } + fmt.Fprintf(&sql, ` AND slots.payload_status IN (%s) `, strings.Join(allowedPayloadStatusesPlaceholders, ", ")) + } } if len(filter.EthBlockParentHash) > 0 { argIdx++ diff --git a/dbtypes/other.go b/dbtypes/other.go index 27741606c..260cd6a14 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -43,6 +43,16 @@ type UnfinalizedBlockFilter struct { WithBody bool } +type PayloadStatusMask uint8 + +const ( + PayloadStatusMaskMissing PayloadStatusMask = 0x01 + PayloadStatusMaskCanonical PayloadStatusMask = 0x02 + PayloadStatusMaskOrphaned PayloadStatusMask = 0x04 + + PayloadStatusMaskAll PayloadStatusMask = 0x07 +) + type BlockFilter struct { Graffiti string InvertGraffiti bool @@ -53,7 +63,7 @@ type BlockFilter struct { InvertProposer bool WithOrphaned uint8 WithMissing uint8 - WithPayloadOrphaned uint8 // 0: only canonical payloads, 1: all, 2: only orphaned payloads + WithPayloadMask PayloadStatusMask MinSyncParticipation *float32 MaxSyncParticipation *float32 MinExecTime *uint32 diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index b21d6a6d4..b2d4de18b 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -243,6 +243,11 @@ func (block *Block) AwaitExecutionPayload(ctx context.Context, timeout time.Dura return block.executionPayload } +// HasExecutionPayload returns true if this block has an execution payload. +func (block *Block) HasExecutionPayload() bool { + return block.hasExecutionPayload +} + // GetParentRoot returns the parent root of this block. func (block *Block) GetParentRoot() *phase0.Root { if block.isDisposed { diff --git a/indexer/beacon/canonical.go b/indexer/beacon/canonical.go index c0d51dde0..4467af3ad 100644 --- a/indexer/beacon/canonical.go +++ b/indexer/beacon/canonical.go @@ -61,6 +61,10 @@ func (indexer *Indexer) IsCanonicalBlockByHead(block *Block, headBlock *Block) b return false } + if block == headBlock { + return true + } + if block.forkChecked && headBlock.forkChecked { parentForkIds := indexer.forkCache.getParentForkIds(headBlock.forkId) return slices.Contains(parentForkIds, block.forkId) diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 95fcc7343..194305fd6 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -331,29 +331,38 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R // Determine payload status for canonical blocks (ePBS only) // A payload is orphaned if the next canonical block doesn't build on it allCanonicalBlocks := append(canonicalBlocks, nextEpochCanonicalBlocks...) - for i, block := range canonicalBlocks { - if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { - continue - } + if chainState.IsEip7732Enabled(epoch) { + for i, block := range canonicalBlocks { + blockIndex := block.GetBlockIndex(indexer.ctx) + if blockIndex == nil || blockIndex.ExecutionNumber == 0 { + fmt.Printf("payload status for slot %v: no execution payload\n", block.Slot) + continue // no execution payload + } - blockIndex := block.GetBlockIndex(indexer.ctx) - if blockIndex == nil || blockIndex.ExecutionNumber == 0 { - continue // no execution payload - } + // Find the next canonical block + if i+1 >= len(allCanonicalBlocks) { + fmt.Printf("payload status for slot %v: no next canonical block\n", block.Slot) + continue + } - // Find the next canonical block - var nextBlock *Block - if i+1 < len(allCanonicalBlocks) { - nextBlock = allCanonicalBlocks[i+1] - } + nextBlock := allCanonicalBlocks[i+1] + if nextBlock == nil { + fmt.Printf("payload status for slot %v: no next canonical block\n", block.Slot) + continue + } - if nextBlock != nil { nextBlockIndex := nextBlock.GetBlockIndex(indexer.ctx) - if nextBlockIndex != nil { - // Check if next block builds on this block's payload - if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { - block.isPayloadOrphaned = true - } + if nextBlockIndex == nil { + fmt.Printf("payload status for slot %v: no next canonical block index\n", block.Slot) + continue + } + + // Check if next block builds on this block's payload + if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + fmt.Printf("payload status for slot %v: orphaned\n", block.Slot) + block.isPayloadOrphaned = true + } else { + fmt.Printf("payload status for slot %v: canonical\n", block.Slot) } } } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index de3f52549..aa1f3342a 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -363,6 +363,49 @@ func (bs *ChainService) GetBlobSidecarsByBlockRoot(ctx context.Context, blockroo return client.GetClient().GetRPCClient().GetBlobSidecarsByBlockroot(ctx, blockroot) } +// getPayloadStatus computes the payload status for a given block. +func (bs *ChainService) getPayloadStatus(ctx context.Context, block *beacon.Block, canonicalHead *beacon.Block) dbtypes.PayloadStatus { + chainState := bs.consensusPool.GetChainState() + if !chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + return dbtypes.PayloadStatusCanonical + } + + if !block.HasExecutionPayload() { + return dbtypes.PayloadStatusMissing + } + + blockIndex := block.GetBlockIndex(ctx) + if blockIndex == nil { + return dbtypes.PayloadStatusCanonical + } + + // Get child blocks and check if any canonical child builds on this payload + childBlocks := bs.beaconIndexer.GetBlockByParentRoot(block.Root) + + if len(childBlocks) == 0 { + // no children, so it's canonical for now + return dbtypes.PayloadStatusCanonical + } + + for _, child := range childBlocks { + childIndex := child.GetBlockIndex(ctx) + if childIndex == nil { + continue + } + // Check if child is in the canonical chain (use original head since + // children are at higher slots than the updated lastCanonicalBlock) + if !bs.beaconIndexer.IsCanonicalBlockByHead(child, canonicalHead) { + continue + } + // Check if child builds on this block's execution payload + if bytes.Equal(childIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { + return dbtypes.PayloadStatusCanonical + } + } + + return dbtypes.PayloadStatusOrphaned +} + // GetDbBlocksForSlots retrieves blocks for a range of slots from cache & database. // The firstSlot parameter specifies the starting slot. // The slotLimit parameter limits the number of slots to retrieve. @@ -418,6 +461,7 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 blocks := bs.beaconIndexer.GetBlocksBySlot(slot) for _, block := range blocks { isCanonical := bs.beaconIndexer.IsCanonicalBlockByHead(block, lastCanonicalBlock) + payloadStatus := bs.getPayloadStatus(ctx, block, lastCanonicalBlock) if isCanonical { lastCanonicalBlock = block } @@ -426,6 +470,7 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 } dbBlock := block.GetDbBlock(bs.beaconIndexer, isCanonical) if dbBlock != nil { + dbBlock.PayloadStatus = payloadStatus resBlocks = append(resBlocks, dbBlock) } } @@ -488,6 +533,7 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 } isCanonical := bs.beaconIndexer.IsCanonicalBlockByHead(block, lastCanonicalBlock) + payloadStatus := bs.getPayloadStatus(ctx, block, lastCanonicalBlock) if isCanonical { lastCanonicalBlock = block } @@ -504,9 +550,10 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 blockRoots = append(blockRoots, block.Root[:]) blockRootsIdx = append(blockRootsIdx, len(resBlocks)) resBlocks = append(resBlocks, &dbtypes.Slot{ - Slot: uint64(slot), - Proposer: uint64(blockHeader.Message.ProposerIndex), - Status: blockStatus, + Slot: uint64(slot), + Proposer: uint64(blockHeader.Message.ProposerIndex), + Status: blockStatus, + PayloadStatus: payloadStatus, }) } @@ -549,6 +596,7 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 for idx, blockRoot := range blockRoots { if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { dbBlock.Status = resBlocks[blockRootsIdx[idx]].Status + dbBlock.PayloadStatus = resBlocks[blockRootsIdx[idx]].PayloadStatus resBlocks[blockRootsIdx[idx]] = dbBlock } } @@ -603,10 +651,11 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 } type cachedDbBlock struct { - slot uint64 - proposer uint64 - orphaned bool - block *beacon.Block + slot uint64 + proposer uint64 + orphaned bool + payloadStatus dbtypes.PayloadStatus + block *beacon.Block } // GetDbBlocksByFilter retrieves a filtered range of blocks from cache & database. @@ -742,6 +791,8 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes } isCanonical := bs.beaconIndexer.IsCanonicalBlockByHead(block, lastCanonicalBlock) + payloadStatus := bs.getPayloadStatus(ctx, block, lastCanonicalBlock) + if isCanonical { lastCanonicalBlock = block } @@ -762,6 +813,17 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes continue } + // filter by payload status + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskMissing == 0 && payloadStatus == dbtypes.PayloadStatusMissing { + continue + } + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskCanonical == 0 && payloadStatus == dbtypes.PayloadStatusCanonical { + continue + } + if filter.WithPayloadMask&dbtypes.PayloadStatusMaskOrphaned == 0 && payloadStatus == dbtypes.PayloadStatusOrphaned { + continue + } + // filter by graffiti if filter.Graffiti != "" { blockGraffiti := string(blockIndex.Graffiti[:]) @@ -930,58 +992,23 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes } } - // filter by payload status (runtime computation for unfinalized blocks) - // Only applies to gloas/ePBS blocks where payloads are separate from beacon blocks - blockEpoch := chainState.EpochOfSlot(block.Slot) - if filter.WithPayloadOrphaned != 1 && chainState.IsEip7732Enabled(blockEpoch) { - // Compute payload status by checking if any child block in the canonical chain - // builds on this block's execution payload - payloadIsCanonical := false - if blockIndex.ExecutionNumber > 0 { - // Get child blocks and check if any canonical child builds on this payload - childBlocks := bs.beaconIndexer.GetBlockByParentRoot(block.Root) - for _, child := range childBlocks { - childIndex := child.GetBlockIndex(ctx) - if childIndex == nil { - continue - } - // Check if child is in the canonical chain (use original head since - // children are at higher slots than the updated lastCanonicalBlock) - if !bs.beaconIndexer.IsCanonicalBlockByHead(child, canonicalHead) { - continue - } - // Check if child builds on this block's execution payload - if bytes.Equal(childIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { - payloadIsCanonical = true - break - } - } - } else { - // No execution payload, treat as canonical for filtering purposes - payloadIsCanonical = true - } - - if filter.WithPayloadOrphaned == 0 && !payloadIsCanonical { - // only canonical payloads, skip orphaned - continue - } - if filter.WithPayloadOrphaned == 2 && payloadIsCanonical { - // only orphaned payloads, skip canonical - continue - } - } - cachedMatches = append(cachedMatches, cachedDbBlock{ - slot: uint64(block.Slot), - proposer: uint64(blockHeader.Message.ProposerIndex), - orphaned: !isCanonical, - block: block, + slot: uint64(block.Slot), + proposer: uint64(blockHeader.Message.ProposerIndex), + orphaned: !isCanonical, + payloadStatus: payloadStatus, + block: block, }) } // reconstruct missing blocks from epoch duties // For slot/root filtering, we still need to check if we need missing blocks for that specific slot - shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 && filter.BuilderIndex == nil && filter.WithPayloadOrphaned != 2 && len(filter.EthBlockParentHash) == 0 && filter.MinGasUsed == nil && filter.MaxGasUsed == nil && filter.MinGasLimit == nil && filter.MaxGasLimit == nil && filter.MinBlockSize == nil && filter.MaxBlockSize == nil && filter.WithMevBlock == 0 && filter.ProposerIndex == nil && filter.ProposerName == "" + shouldCheckMissing := filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 && + filter.MinSyncParticipation == nil && filter.MaxSyncParticipation == nil && filter.MinExecTime == nil && filter.MaxExecTime == nil && + filter.MinTxCount == nil && filter.MaxTxCount == nil && filter.MinBlobCount == nil && filter.MaxBlobCount == nil && len(filter.ForkIds) == 0 && + filter.BuilderIndex == nil && filter.WithPayloadMask&dbtypes.PayloadStatusMaskMissing != 0 && len(filter.EthBlockParentHash) == 0 && filter.MinGasUsed == nil && + filter.MaxGasUsed == nil && filter.MinGasLimit == nil && filter.MaxGasLimit == nil && filter.MinBlockSize == nil && filter.MaxBlockSize == nil && + filter.WithMevBlock == 0 && filter.ProposerIndex == nil && filter.ProposerName == "" // If filtering by slot, only check missing for that specific slot if filter.Slot != nil { @@ -1093,6 +1120,7 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes if block.block != nil { if block.slot >= uint64(prunedSlot) { assignedBlock.Block = block.block.GetDbBlock(bs.beaconIndexer, !block.orphaned) + assignedBlock.Block.PayloadStatus = block.payloadStatus } else { blockRoots = append(blockRoots, block.block.Root[:]) blockRootsIdx = append(blockRootsIdx, resIdx) @@ -1110,12 +1138,14 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes if blockMap != nil { for idx, blockRoot := range blockRoots { if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { + cachedMatch := cachedMatches[blockRootsCachedId[idx]] dbBlock.Status = dbtypes.Canonical - if cachedMatches[blockRootsCachedId[idx]].orphaned { + if cachedMatch.orphaned { dbBlock.Status = dbtypes.Orphaned } + dbBlock.PayloadStatus = cachedMatch.payloadStatus resBlocks[blockRootsIdx[idx]].Block = dbBlock } } From 1d0073a283a58ec76bcabad516c73a5ebc7ac362 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 10 Mar 2026 03:20:32 +0100 Subject: [PATCH 63/99] show epoch+1 duties in epoch overview & slot lists (rel #524) --- handlers/epoch.go | 4 ++-- handlers/slots.go | 3 --- handlers/slots_filtered.go | 7 +------ services/chainservice_blocks.go | 23 +++++++++++++++++++++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/handlers/epoch.go b/handlers/epoch.go index c3ac1baf7..b9a1e2e90 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -91,7 +91,7 @@ func buildEpochPageData(ctx context.Context, epoch uint64) (*models.EpochPageDat specs := chainState.GetSpecs() currentSlot := chainState.CurrentSlot() currentEpoch := chainState.EpochOfSlot(currentSlot) - if epoch > uint64(currentEpoch) { + if epoch > uint64(currentEpoch)+1 { return nil, -1 } @@ -105,7 +105,7 @@ func buildEpochPageData(ctx context.Context, epoch uint64) (*models.EpochPageDat } nextEpoch := epoch + 1 - if nextEpoch > uint64(currentEpoch) { + if nextEpoch > uint64(currentEpoch)+1 { nextEpoch = 0 } firstSlot := chainState.EpochToSlot(phase0.Epoch(epoch)) diff --git a/handlers/slots.go b/handlers/slots.go index de22a1905..5168a1c8e 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -158,9 +158,6 @@ func buildSlotsPageData(ctx context.Context, firstSlot uint64, pageSize uint64, currentSlot := chainState.CurrentSlot() currentEpoch := chainState.EpochOfSlot(currentSlot) maxSlot := currentSlot + 8 - if maxSlot >= chainState.EpochToSlot(currentEpoch+1) { - maxSlot = chainState.EpochToSlot(currentEpoch+1) - 1 - } if firstSlot > uint64(maxSlot) { pageData.IsDefaultPage = true firstSlot = uint64(maxSlot) diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 195b0a88f..05739c453 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -436,12 +436,7 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui } } - withScheduledCount := chainState.GetSpecs().SlotsPerEpoch - uint64(chainState.SlotToSlotIndex(currentSlot)) - 1 - if withScheduledCount > 16 { - withScheduledCount = 16 - } - - dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, pageIdx, uint32(pageSize), withScheduledCount) + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, pageIdx, uint32(pageSize), 16) mevBlocksMap := make(map[string]*dbtypes.MevBlock) if pageData.DisplayMevBlock { diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index aa1f3342a..4863d080c 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -431,7 +431,10 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) getCanonicalProposer := func(slot phase0.Slot) phase0.ValidatorIndex { epoch := chainState.EpochOfSlot(slot) - if epoch != proposerAssignmentsEpoch { + if proposerAssignmentsEpoch != phase0.Epoch(math.MaxInt64) && epoch == proposerAssignmentsEpoch+1 && chainState.IsFuluEnabled(epoch) { + // extended proposer lookahead in fulu, use the same proposer assignments as the previous epoch + } else if epoch != proposerAssignmentsEpoch { + assignmentsEpoch := epoch if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} @@ -440,8 +443,20 @@ func (bs *ChainService) GetDbBlocksForSlots(ctx context.Context, firstSlot uint6 proposerAssignments[slot] = proposer } } + } else if epoch > 0 && chainState.IsFuluEnabled(epoch-1) { + if epochStats := bs.beaconIndexer.GetEpochStats(epoch-1, nil); epochStats != nil { + if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { + assignmentsEpoch = epoch - 1 + proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} + for slotIdx, proposer := range epochStatsValues.ProposerDuties { + slot := chainState.EpochToSlot(assignmentsEpoch) + phase0.Slot(slotIdx) + proposerAssignments[slot] = proposer + } + } + } + } - proposerAssignmentsEpoch = epoch + proposerAssignmentsEpoch = assignmentsEpoch } proposer, ok := proposerAssignments[slot] @@ -714,6 +729,10 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes filter.MaxSlot = &maxSlot } + if filter.WithPayloadMask == 0 { + filter.WithPayloadMask = dbtypes.PayloadStatusMaskAll + } + // get blocks from cache // iterate from current slot to finalized slot canonicalHead := bs.beaconIndexer.GetCanonicalHead(nil) From 2f8fe154b7f47d31615eb7655e1d7dc1ec50d9b1 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 10 Mar 2026 03:40:59 +0100 Subject: [PATCH 64/99] add payload status filter to filtered slots page --- handlers/slots_filtered.go | 99 ++++++++++++-------- services/chainservice_blocks.go | 19 +++- templates/slots_filtered/slots_filtered.html | 54 ++++++----- types/models/slots_filtered.go | 44 +++++---- 4 files changed, 134 insertions(+), 82 deletions(-) diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 05739c453..15e197352 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -50,8 +50,8 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { var invertgraffiti bool var invertextradata bool var invertproposer bool - var withOrphaned uint64 - var withMissing uint64 + var statusMask uint64 = 0x07 + var payloadMask uint64 = 0x07 var minSyncAgg string var maxSyncAgg string var minExecTime string @@ -86,11 +86,11 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { if urlArgs.Has("f.pinvert") { invertproposer = urlArgs.Get("f.pinvert") == "on" } - if urlArgs.Has("f.orphaned") { - withOrphaned, _ = strconv.ParseUint(urlArgs.Get("f.orphaned"), 10, 64) + if urlArgs.Has("f.status") { + statusMask, _ = strconv.ParseUint(urlArgs.Get("f.status"), 0, 64) } - if urlArgs.Has("f.missing") { - withMissing, _ = strconv.ParseUint(urlArgs.Get("f.missing"), 10, 64) + if urlArgs.Has("f.pstatus") { + payloadMask, _ = strconv.ParseUint(urlArgs.Get("f.pstatus"), 0, 64) } if urlArgs.Has("f.minsync") { minSyncAgg = urlArgs.Get("f.minsync") @@ -125,14 +125,11 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { if urlArgs.Has("f.maxepoch") { maxEpoch = urlArgs.Get("f.maxepoch") } - } else { - withOrphaned = 1 - withMissing = 1 } var pageError error pageError = services.GlobalCallRateLimiter.CheckCallLimit(r, 2) if pageError == nil { - data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, uint8(withOrphaned), uint8(withMissing), minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, uint8(statusMask), uint8(payloadMask), minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) } if pageError != nil { handlePageError(w, r, pageError) @@ -144,11 +141,11 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { } } -func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, withOrphaned uint8, withMissing uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) (*models.SlotsFilteredPageData, error) { +func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) (*models.SlotsFilteredPageData, error) { pageData := &models.SlotsFilteredPageData{} - pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, withOrphaned, withMissing, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { - return buildFilteredSlotsPageData(pageCall.CallCtx, pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, withOrphaned, withMissing, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + return buildFilteredSlotsPageData(pageCall.CallCtx, pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) }) if pageErr == nil && pageRes != nil { resData, resOk := pageRes.(*models.SlotsFilteredPageData) @@ -160,7 +157,7 @@ func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, return pageData, pageErr } -func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, withOrphaned uint8, withMissing uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) *models.SlotsFilteredPageData { +func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) *models.SlotsFilteredPageData { chainState := services.GlobalBeaconService.GetChainState() filterArgs := url.Values{} if graffiti != "" { @@ -184,11 +181,11 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui if invertproposer { filterArgs.Add("f.pinvert", "on") } - if withOrphaned != 0 { - filterArgs.Add("f.orphaned", fmt.Sprintf("%v", withOrphaned)) + if statusMask != 0x07 { + filterArgs.Add("f.status", fmt.Sprintf("0x%x", statusMask)) } - if withMissing != 0 { - filterArgs.Add("f.missing", fmt.Sprintf("%v", withMissing)) + if payloadMask != 0x07 { + filterArgs.Add("f.pstatus", fmt.Sprintf("0x%x", payloadMask)) } if minSyncAgg != "" { filterArgs.Add("f.minsync", minSyncAgg) @@ -270,27 +267,52 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui filterArgs.Add("d", fmt.Sprintf("0x%x", displayMask)) } + // Map statusMask to WithOrphaned/WithMissing for DB filter + hasMissing := statusMask&0x01 != 0 + hasCanonical := statusMask&0x02 != 0 + hasOrphaned := statusMask&0x04 != 0 + + var withMissing, withOrphaned uint8 + if !hasMissing { + withMissing = 0 + } else if !hasCanonical && !hasOrphaned { + withMissing = 2 + } else { + withMissing = 1 + } + if !hasOrphaned { + withOrphaned = 0 + } else if !hasCanonical && !hasMissing { + withOrphaned = 2 + } else { + withOrphaned = 1 + } + pageData := &models.SlotsFilteredPageData{ - FilterGraffiti: graffiti, - FilterExtraData: extradata, - FilterProposer: proposer, - FilterProposerName: pname, - FilterInvertGraffiti: invertgraffiti, - FilterInvertExtraData: invertextradata, - FilterInvertProposer: invertproposer, - FilterWithOrphaned: withOrphaned, - FilterWithMissing: withMissing, - FilterMinSyncAgg: minSyncAgg, - FilterMaxSyncAgg: maxSyncAgg, - FilterMinExecTime: minExecTime, - FilterMaxExecTime: maxExecTime, - FilterMinTxCount: minTxCount, - FilterMaxTxCount: maxTxCount, - FilterMinBlobCount: minBlobCount, - FilterMaxBlobCount: maxBlobCount, - FilterForkIds: forkIds, - FilterMinEpoch: minEpoch, - FilterMaxEpoch: maxEpoch, + FilterGraffiti: graffiti, + FilterExtraData: extradata, + FilterProposer: proposer, + FilterProposerName: pname, + FilterInvertGraffiti: invertgraffiti, + FilterInvertExtraData: invertextradata, + FilterInvertProposer: invertproposer, + FilterStatusMissing: hasMissing, + FilterStatusCanonical: hasCanonical, + FilterStatusOrphaned: hasOrphaned, + FilterPayloadMissing: payloadMask&0x01 != 0, + FilterPayloadCanonical: payloadMask&0x02 != 0, + FilterPayloadOrphaned: payloadMask&0x04 != 0, + FilterMinSyncAgg: minSyncAgg, + FilterMaxSyncAgg: maxSyncAgg, + FilterMinExecTime: minExecTime, + FilterMaxExecTime: maxExecTime, + FilterMinTxCount: minTxCount, + FilterMaxTxCount: maxTxCount, + FilterMinBlobCount: minBlobCount, + FilterMaxBlobCount: maxBlobCount, + FilterForkIds: forkIds, + FilterMinEpoch: minEpoch, + FilterMaxEpoch: maxEpoch, DisplayEpoch: displayMap[1], DisplaySlot: displayMap[2], @@ -346,6 +368,7 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui InvertProposer: invertproposer, WithOrphaned: withOrphaned, WithMissing: withMissing, + WithPayloadMask: dbtypes.PayloadStatusMask(payloadMask), } if proposer != "" { pidx, _ := strconv.ParseUint(proposer, 10, 64) diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 4863d080c..8364c23c2 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -698,7 +698,10 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) getCanonicalProposer := func(slot phase0.Slot) phase0.ValidatorIndex { epoch := chainState.EpochOfSlot(slot) - if epoch != proposerAssignmentsEpoch { + if proposerAssignmentsEpoch != phase0.Epoch(math.MaxInt64) && epoch == proposerAssignmentsEpoch+1 && chainState.IsFuluEnabled(epoch) { + // extended proposer lookahead in fulu, use the same proposer assignments as the previous epoch + } else if epoch != proposerAssignmentsEpoch { + assignmentsEpoch := epoch if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} @@ -707,8 +710,20 @@ func (bs *ChainService) GetDbBlocksByFilter(ctx context.Context, filter *dbtypes proposerAssignments[slot] = proposer } } + } else if epoch > 0 && chainState.IsFuluEnabled(epoch-1) { + if epochStats := bs.beaconIndexer.GetEpochStats(epoch-1, nil); epochStats != nil { + if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { + assignmentsEpoch = epoch - 1 + proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} + for slotIdx, proposer := range epochStatsValues.ProposerDuties { + slot := chainState.EpochToSlot(assignmentsEpoch) + phase0.Slot(slotIdx) + proposerAssignments[slot] = proposer + } + } + } + } - proposerAssignmentsEpoch = epoch + proposerAssignmentsEpoch = assignmentsEpoch } proposer, ok := proposerAssignments[slot] diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index 5967e6af7..3f9306ee8 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -97,25 +97,27 @@

    Filtered Slots

    - Missing Blocks + Block Status
    -
    - +
    - Orphaned Blocks + Payload Status
    -
    - +
    @@ -463,17 +465,25 @@

    Filtered Slots

    {{ end }} @@ -470,6 +499,12 @@

    padding-right: 10px; } +.entity-field-disabled { + background-color: var(--bs-secondary-bg); + opacity: 0.65; + cursor: pointer; +} + .tx-details-label { min-width: 90px; } diff --git a/templates/voluntary_exits/voluntary_exits.html b/templates/voluntary_exits/voluntary_exits.html index 93b4eca55..e159d187e 100644 --- a/templates/voluntary_exits/voluntary_exits.html +++ b/templates/voluntary_exits/voluntary_exits.html @@ -43,26 +43,36 @@

    - Validator Index + Entity Type +
    +
    + +
    +
    +
    +
    + {{ if eq .FilterEntity "builder" }}Builder Index{{ else }}Validator Index{{ end }}
    - -
    -
    - - +
    +
    -
    - +
    - Validator Name + {{ if eq .FilterEntity "builder" }}Builder Name{{ else }}Validator Name{{ end }}
    - +
    @@ -277,6 +287,24 @@

    $('#pageJumpForm').submit(); } }); + + function updateEntityFields(entity) { + var isAll = (entity === 'all'); + $('.entity-field').each(function() { + $(this).prop('readonly', isAll).toggleClass('entity-field-disabled', isAll); + if (isAll) { $(this).val(''); } + }); + if (!isAll) { + $('.entity-label').each(function() { $(this).text($(this).data(entity)); }); + $('.entity-placeholder').each(function() { $(this).attr('placeholder', $(this).data(entity)); }); + } + } + $('.entity-select').on('change', function() { updateEntityFields($(this).val()); }); + $('.entity-field').on('mousedown', function() { + if ($(this).prop('readonly')) { + $('.entity-select').val('validator').trigger('change'); + } + }); }); {{ end }} @@ -289,5 +317,11 @@

    padding-right: 10px; } +.entity-field-disabled { + background-color: var(--bs-secondary-bg); + opacity: 0.65; + cursor: pointer; +} + {{ end }} \ No newline at end of file diff --git a/templates/withdrawals_list/withdrawals_list.html b/templates/withdrawals_list/withdrawals_list.html index a432b8b32..7ec45b9bb 100644 --- a/templates/withdrawals_list/withdrawals_list.html +++ b/templates/withdrawals_list/withdrawals_list.html @@ -27,10 +27,36 @@

    - Validator + Entity Type
    - + +
    +
    +
    +
    + {{ if eq .FilterEntity "builder" }}Builder Index{{ else }}Validator Index{{ end }} +
    +
    +
    + +
    +
    -
    +
    + +
    +
    +
    +
    +
    + {{ if eq .FilterEntity "builder" }}Builder Name{{ else }}Validator Name{{ end }} +
    +
    +
    @@ -325,6 +351,26 @@

    } }); }); + + // Entity type toggle + function updateEntityFields(entity) { + var isAll = (entity === 'all'); + $('.entity-field').each(function() { + $(this).prop('readonly', isAll).toggleClass('entity-field-disabled', isAll); + if (isAll) { $(this).val(''); } + }); + if (!isAll) { + $('.entity-label').each(function() { $(this).text($(this).data(entity)); }); + $('.entity-placeholder').each(function() { $(this).attr('placeholder', $(this).data(entity)); }); + } + } + $('.entity-select').on('change', function() { updateEntityFields($(this).val()); }); + // Click on readonly/disabled-looking field: switch to Validator + $('.entity-field').on('mousedown', function() { + if ($(this).prop('readonly')) { + $('.entity-select').val('validator').trigger('change'); + } + }); }); {{ end }} @@ -338,6 +384,12 @@

    padding-right: 10px; } +.entity-field-disabled { + background-color: var(--bs-secondary-bg); + opacity: 0.65; + cursor: pointer; +} + .filter-multiselect-container { width: 100%; } diff --git a/types/models/el_withdrawals.go b/types/models/el_withdrawals.go index 04e2090e1..95b6473a0 100644 --- a/types/models/el_withdrawals.go +++ b/types/models/el_withdrawals.go @@ -6,6 +6,7 @@ import ( // ElWithdrawalsPageData is a struct to hold info for the el_withdrawals page type ElWithdrawalsPageData struct { + FilterEntity string `json:"filter_entity"` // "all", "validator", or "builder" FilterMinSlot uint64 `json:"filter_mins"` FilterMaxSlot uint64 `json:"filter_maxs"` FilterAddress string `json:"filter_address"` diff --git a/types/models/voluntary_exits.go b/types/models/voluntary_exits.go index 2dcb7d84e..685dbf825 100644 --- a/types/models/voluntary_exits.go +++ b/types/models/voluntary_exits.go @@ -6,6 +6,7 @@ import ( // VoluntaryExitsPageData is a struct to hold info for the voluntary_exits page type VoluntaryExitsPageData struct { + FilterEntity string `json:"filter_entity"` // "all", "validator", or "builder" FilterMinSlot uint64 `json:"filter_mins"` FilterMaxSlot uint64 `json:"filter_maxs"` FilterMinIndex uint64 `json:"filter_mini"` diff --git a/types/models/withdrawals_list.go b/types/models/withdrawals_list.go index 5374942fa..5f457a099 100644 --- a/types/models/withdrawals_list.go +++ b/types/models/withdrawals_list.go @@ -6,12 +6,15 @@ import ( // WithdrawalsListPageData is a struct to hold info for the withdrawals list page. type WithdrawalsListPageData struct { - FilterValidator string `json:"filter_validator"` - FilterAddress string `json:"filter_address"` - FilterWithType string `json:"filter_type"` - FilterMinAmount string `json:"filter_min_amount"` - FilterMaxAmount string `json:"filter_max_amount"` - FilterWithOrphaned uint8 `json:"filter_orphaned"` + FilterEntity string `json:"filter_entity"` // "all", "validator", or "builder" + FilterMinIndex uint64 `json:"filter_mini"` + FilterMaxIndex uint64 `json:"filter_maxi"` + FilterValidatorName string `json:"filter_vname"` + FilterAddress string `json:"filter_address"` + FilterWithType string `json:"filter_type"` + FilterMinAmount string `json:"filter_min_amount"` + FilterMaxAmount string `json:"filter_max_amount"` + FilterWithOrphaned uint8 `json:"filter_orphaned"` Withdrawals []*WithdrawalsListPageDataWithdrawal `json:"withdrawals"` WithdrawalCount uint64 `json:"withdrawal_count"` From 113cc1835e63d320f815031e97240ba385704c94 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 3 Apr 2026 05:07:40 +0200 Subject: [PATCH 81/99] add builder withdrawals tab --- handlers/builder.go | 36 +++++++++++++ templates/builder/builder.html | 12 +++++ templates/builder/withdrawals.html | 87 ++++++++++++++++++++++++++++++ types/models/builders.go | 19 +++++-- 4 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 templates/builder/withdrawals.html diff --git a/handlers/builder.go b/handlers/builder.go index 13dc4d4b0..d140d4615 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -32,6 +32,7 @@ func BuilderDetail(w http.ResponseWriter, r *http.Request) { "builder/recentBlocks.html", "builder/recentBids.html", "builder/recentDeposits.html", + "builder/withdrawals.html", "_svg/timeline.html", ) var notfoundTemplateFiles = append(layoutTemplateFiles, @@ -253,6 +254,41 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b pageData.RecentBids = buildBuilderRecentBids(ctx, builderIndex, chainState) case "deposits": pageData.RecentDeposits = buildBuilderRecentDeposits(ctx, builder.PublicKey[:], chainState) + case "withdrawals": + builderValidatorIndex := builderIndex | services.BuilderIndexFlag + withdrawalFilter := &dbtypes.WithdrawalFilter{ + MinIndex: builderValidatorIndex, + MaxIndex: builderValidatorIndex, + WithOrphaned: 1, + } + dbWithdrawals, totalRows := services.GlobalBeaconService.GetWithdrawalsByFilter(ctx, withdrawalFilter, 0, 10) + if totalRows > 10 { + pageData.AdditionalWithdrawalCount = totalRows - 10 + } + + for _, w := range dbWithdrawals { + slot := w.BlockUid >> 16 + wd := &models.BuilderPageDataWithdrawal{ + SlotNumber: slot, + Time: chainState.SlotToTime(phase0.Slot(slot)), + Orphaned: w.Orphaned, + Type: w.Type, + Amount: w.Amount, + } + + // Resolve block root + blockFilter := &dbtypes.BlockFilter{ + BlockUids: []uint64{w.BlockUid}, + WithOrphaned: 1, + } + blocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, 0, 1, 0) + if len(blocks) > 0 && blocks[0].Block != nil { + wd.BlockRoot = blocks[0].Block.Root + } + + pageData.Withdrawals = append(pageData.Withdrawals, wd) + } + pageData.WithdrawalCount = uint64(len(pageData.Withdrawals)) } return pageData, 10 * time.Minute diff --git a/templates/builder/builder.html b/templates/builder/builder.html index 3e4e17f50..940adfefa 100644 --- a/templates/builder/builder.html +++ b/templates/builder/builder.html @@ -207,6 +207,11 @@

    Builder {{ form Deposits +
  • + + Withdrawals + +
    • @@ -225,6 +230,11 @@

    Builder {{ form {{ template "recentDeposits" . }} {{ end }}

    +
    + {{ if eq .TabView "withdrawals" }} + {{ template "builderWithdrawals" . }} + {{ end }} +

    @@ -236,6 +246,8 @@

    Builder {{ form {{ template "recentBids" . }} {{ else if eq .TabView "deposits" }} {{ template "recentDeposits" . }} + {{ else if eq .TabView "withdrawals" }} + {{ template "builderWithdrawals" . }} {{ else }} Unknown tab {{ end }} diff --git a/templates/builder/withdrawals.html b/templates/builder/withdrawals.html new file mode 100644 index 000000000..91f3ac901 --- /dev/null +++ b/templates/builder/withdrawals.html @@ -0,0 +1,87 @@ +{{ define "builderWithdrawals" }} +
    +
    +
    + + + + + + + + + + + + {{ if gt .WithdrawalCount 0 }} + {{ range $i, $withdrawal := .Withdrawals }} + + {{ if $withdrawal.Orphaned }} + + {{ else }} + + {{ end }} + + + + + + {{ end }} + + + + {{ else }} + + + + {{ end }} + +
    SlotTimeTypeAmountStatus
    {{ formatAddCommas $withdrawal.SlotNumber }}{{ formatAddCommas $withdrawal.SlotNumber }}{{ formatRecentTimeShort $withdrawal.Time }} + {{ if eq $withdrawal.Type 1 }} + + Full + + {{ else if eq $withdrawal.Type 2 }} + + Sweep + + {{ else if eq $withdrawal.Type 3 }} + + Requested + + {{ else if eq $withdrawal.Type 4 }} + + Builder Withdrawal + + {{ else if eq $withdrawal.Type 5 }} + + Builder Payment + + {{ else if eq $withdrawal.Type 6 }} + + Builder Delayed + + {{ else }} + Unknown + {{ end }} + + {{ formatEthFromGweiP $withdrawal.Amount 6 }} + + {{ if $withdrawal.Orphaned }} + Orphaned + {{ else }} + Canonical + {{ end }} +
    + {{ if gt .AdditionalWithdrawalCount 0 }} + View {{ formatAddCommas .AdditionalWithdrawalCount }} more withdrawals + {{ end }} +
    +
    + {{ template "timeline_svg" }} +
    +
    +
    +
    +
    +{{ end }} diff --git a/types/models/builders.go b/types/models/builders.go index 0d56f47a2..cc512b5e3 100644 --- a/types/models/builders.go +++ b/types/models/builders.go @@ -85,9 +85,12 @@ type BuilderPageData struct { GloasIsActive bool `json:"gloas_is_active"` // Tab data (loaded conditionally) - RecentBlocks []*BuilderPageDataBlock `json:"recent_blocks"` - RecentBids []*BuilderPageDataBid `json:"recent_bids"` - RecentDeposits []*BuilderPageDataDeposit `json:"recent_deposits"` + RecentBlocks []*BuilderPageDataBlock `json:"recent_blocks"` + RecentBids []*BuilderPageDataBid `json:"recent_bids"` + RecentDeposits []*BuilderPageDataDeposit `json:"recent_deposits"` + Withdrawals []*BuilderPageDataWithdrawal `json:"withdrawals"` + WithdrawalCount uint64 `json:"withdrawal_count"` + AdditionalWithdrawalCount uint64 `json:"additional_withdrawal_count"` } // BuilderPageDataBlock represents a block/payload built by this builder @@ -127,6 +130,16 @@ type BuilderPageDataDeposit struct { Orphaned bool `json:"orphaned"` } +// BuilderPageDataWithdrawal represents a withdrawal on the builder detail page +type BuilderPageDataWithdrawal struct { + SlotNumber uint64 `json:"slot"` + BlockRoot []byte `json:"block_root" ssz-size:"32"` + Time time.Time `json:"time"` + Orphaned bool `json:"orphaned"` + Type uint8 `json:"type"` + Amount uint64 `json:"amount"` +} + // BuilderPageDataExitTxDetails contains transaction details for EL-triggered exits type BuilderPageDataExitTxDetails struct { BlockNumber uint64 `json:"block"` From 133ec92c17ee12850bbfdaf12cdc6893a0f9208c Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 3 Apr 2026 05:13:08 +0200 Subject: [PATCH 82/99] fix builder indexes in exit requests tab on exits overview --- handlers/exits.go | 9 +++++++-- templates/exits/exits.html | 6 +++++- types/models/exits.go | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/handlers/exits.go b/handlers/exits.go index 05bd37e26..2cff7d97f 100644 --- a/handlers/exits.go +++ b/handlers/exits.go @@ -248,9 +248,14 @@ func buildExitsPageData(ctx context.Context, firstEpoch uint64, pageSize uint64, } if validatorIndex := exitReq.ValidatorIndex(); validatorIndex != nil { - exitReqData.ValidatorIndex = *validatorIndex - exitReqData.ValidatorName = services.GlobalBeaconService.GetValidatorName(*validatorIndex) exitReqData.ValidatorValid = true + exitReqData.ValidatorName = services.GlobalBeaconService.GetValidatorName(*validatorIndex) + if *validatorIndex&services.BuilderIndexFlag != 0 { + exitReqData.IsBuilder = true + exitReqData.ValidatorIndex = *validatorIndex &^ services.BuilderIndexFlag + } else { + exitReqData.ValidatorIndex = *validatorIndex + } } if request := exitReq.Request; request != nil { diff --git a/templates/exits/exits.html b/templates/exits/exits.html index f48517c3a..90578d2a9 100644 --- a/templates/exits/exits.html +++ b/templates/exits/exits.html @@ -343,7 +343,11 @@
    This table displays the most recent execution layer t {{- if $exitReq.ValidatorValid }} - {{ formatValidatorWithIndex $exitReq.ValidatorIndex $exitReq.ValidatorName }} + {{ if $exitReq.IsBuilder }} + {{ formatBuilderWithIndex $exitReq.ValidatorIndex $exitReq.ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex $exitReq.ValidatorIndex $exitReq.ValidatorName }} + {{ end }} {{- else }}
    0x{{ printf "%x" $exitReq.PublicKey }} diff --git a/types/models/exits.go b/types/models/exits.go index 835ec0c44..34ee0b1ca 100644 --- a/types/models/exits.go +++ b/types/models/exits.go @@ -69,6 +69,7 @@ type ExitsPageDataRecentExitRequest struct { ValidatorValid bool `json:"vvalid"` ValidatorIndex uint64 `json:"vindex"` ValidatorName string `json:"vname"` + IsBuilder bool `json:"is_builder"` PublicKey []byte `json:"pubkey" ssz-size:"48"` LinkedTransaction bool `json:"linked_tx"` TransactionHash []byte `json:"tx_hash" ssz-size:"32"` From 8a26faf5a1d9aac87ace9cdd4b94888baefe92f5 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 3 Apr 2026 18:49:25 +0200 Subject: [PATCH 83/99] trigger CI From 435074c9685cfcd9d777de3d8d775cfb2e3c3da4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 8 Apr 2026 15:35:21 +0200 Subject: [PATCH 84/99] fix withdrawal indexing post gloas --- indexer/beacon/writedb.go | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 2f3c611bc..4c5cd299a 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -843,19 +843,31 @@ func (dbw *dbWriter) persistBlockWithdrawals(tx *sqlx.Tx, block *Block, orphaned // If tx is nil (read path), only existing accounts are looked up. // If sim is non-nil, it's used to determine pending partial withdrawal count for type classification. func (dbw *dbWriter) buildDbWithdrawals(block *Block, orphaned bool, overrideForkId *ForkKey, tx *sqlx.Tx, sim *stateSimulator) []*dbtypes.Withdrawal { - blockBody := block.GetBlock(dbw.indexer.ctx) - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - executionPayload, err := blockBody.ExecutionPayload() - if err != nil || executionPayload == nil { - return nil - } + var executionWithdrawals []*capella.Withdrawal + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + blockPayload := block.GetExecutionPayload(dbw.indexer.ctx) + if blockPayload != nil { + executionWithdrawals = blockPayload.Message.Payload.Withdrawals + } + } else { + blockBody := block.GetBlock(dbw.indexer.ctx) + if blockBody == nil { + return nil + } - withdrawals, err := executionPayload.Withdrawals() - if err != nil || len(withdrawals) == 0 { - return nil + executionPayload, err := blockBody.ExecutionPayload() + if err != nil || executionPayload == nil { + return nil + } + + withdrawals, err := executionPayload.Withdrawals() + if err != nil || len(withdrawals) == 0 { + return nil + } + + executionWithdrawals = withdrawals } forkId := uint64(block.forkId) @@ -868,8 +880,8 @@ func (dbw *dbWriter) buildDbWithdrawals(block *Block, orphaned bool, overrideFor blockEpoch := dbw.indexer.consensusPool.GetChainState().EpochOfSlot(block.Slot) - dbWithdrawals := make([]*dbtypes.Withdrawal, len(withdrawals)) - for idx, withdrawal := range withdrawals { + dbWithdrawals := make([]*dbtypes.Withdrawal, len(executionWithdrawals)) + for idx, withdrawal := range executionWithdrawals { // Classify withdrawal type and resolve reference slot withdrawalType, refSlot := dbw.classifyWithdrawalType(idx, simResult, withdrawal.ValidatorIndex, phase0.Gwei(withdrawal.Amount), blockEpoch) @@ -887,7 +899,7 @@ func (dbw *dbWriter) buildDbWithdrawals(block *Block, orphaned bool, overrideFor } // Resolve account IDs for withdrawal addresses - dbw.resolveWithdrawalAccounts(withdrawals, dbWithdrawals, tx) + dbw.resolveWithdrawalAccounts(executionWithdrawals, dbWithdrawals, tx) return dbWithdrawals } From 0fa5e4e211976cfc8f0910999c293965be796374 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 8 Apr 2026 15:35:46 +0200 Subject: [PATCH 85/99] render builder withdrawals as builder index --- handlers/withdrawals.go | 7 ++++++- handlers/withdrawals_list.go | 7 ++++++- templates/withdrawals/withdrawals.html | 6 +++++- templates/withdrawals_list/withdrawals_list.html | 6 +++++- types/models/withdrawals.go | 1 + types/models/withdrawals_list.go | 1 + 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/handlers/withdrawals.go b/handlers/withdrawals.go index d0e08fad2..4a2651f22 100644 --- a/handlers/withdrawals.go +++ b/handlers/withdrawals.go @@ -237,7 +237,12 @@ func buildWithdrawalsPageData(ctx context.Context, firstEpoch uint64, pageSize u } withdrawalData.HasValidator = true - withdrawalData.ValidatorIndex = withdrawal.Validator + if withdrawal.Validator&services.BuilderIndexFlag != 0 { + withdrawalData.IsBuilder = true + withdrawalData.ValidatorIndex = withdrawal.Validator &^ services.BuilderIndexFlag + } else { + withdrawalData.ValidatorIndex = withdrawal.Validator + } withdrawalData.ValidatorName = services.GlobalBeaconService.GetValidatorName(withdrawal.Validator) if withdrawal.AccountID > 0 { diff --git a/handlers/withdrawals_list.go b/handlers/withdrawals_list.go index e5b58dcad..5fc933087 100644 --- a/handlers/withdrawals_list.go +++ b/handlers/withdrawals_list.go @@ -304,7 +304,12 @@ func buildFilteredWithdrawalsListPageData(ctx context.Context, pageIdx uint64, p } withdrawalData.HasValidator = true - withdrawalData.ValidatorIndex = withdrawal.Validator + if withdrawal.Validator&services.BuilderIndexFlag != 0 { + withdrawalData.IsBuilder = true + withdrawalData.ValidatorIndex = withdrawal.Validator &^ services.BuilderIndexFlag + } else { + withdrawalData.ValidatorIndex = withdrawal.Validator + } withdrawalData.ValidatorName = services.GlobalBeaconService.GetValidatorName(withdrawal.Validator) // Resolve address from account_id diff --git a/templates/withdrawals/withdrawals.html b/templates/withdrawals/withdrawals.html index 23a3ce714..8ef63cad3 100644 --- a/templates/withdrawals/withdrawals.html +++ b/templates/withdrawals/withdrawals.html @@ -403,7 +403,11 @@
    This table displays the most recent beacon chain with {{ if $withdrawal.HasValidator }} - {{ formatValidatorWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- if $withdrawal.IsBuilder }} + {{ formatBuilderWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- else }} + {{ formatValidatorWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- end }} {{ else }} - {{ end }} diff --git a/templates/withdrawals_list/withdrawals_list.html b/templates/withdrawals_list/withdrawals_list.html index 7ec45b9bb..db6807e97 100644 --- a/templates/withdrawals_list/withdrawals_list.html +++ b/templates/withdrawals_list/withdrawals_list.html @@ -218,7 +218,11 @@

    {{ if $withdrawal.HasValidator }} - {{ formatValidatorWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- if $withdrawal.IsBuilder }} + {{ formatBuilderWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- else }} + {{ formatValidatorWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} + {{- end }} {{ else }} - {{ end }} diff --git a/types/models/withdrawals.go b/types/models/withdrawals.go index 08eb55bfc..5b9fe2b60 100644 --- a/types/models/withdrawals.go +++ b/types/models/withdrawals.go @@ -67,6 +67,7 @@ type WithdrawalsPageDataBeaconWithdrawal struct { Orphaned bool `json:"orphaned"` Type uint8 `json:"type"` HasValidator bool `json:"has_validator"` + IsBuilder bool `json:"is_builder"` ValidatorIndex uint64 `json:"vindex"` ValidatorName string `json:"vname"` Address []byte `json:"address" ssz-size:"20"` diff --git a/types/models/withdrawals_list.go b/types/models/withdrawals_list.go index 5f457a099..4682a9319 100644 --- a/types/models/withdrawals_list.go +++ b/types/models/withdrawals_list.go @@ -46,6 +46,7 @@ type WithdrawalsListPageDataWithdrawal struct { Orphaned bool `json:"orphaned"` Type uint8 `json:"type"` HasValidator bool `json:"has_validator"` + IsBuilder bool `json:"is_builder"` ValidatorIndex uint64 `json:"vindex"` ValidatorName string `json:"vname"` Address []byte `json:"address" ssz-size:"20"` From b53daff934b0424893dfd4b590f7763e4550036d Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 9 Apr 2026 04:00:39 +0200 Subject: [PATCH 86/99] implement epoch state transition --- indexer/beacon/statetransition/balance.go | 28 ++ indexer/beacon/statetransition/builder.go | 122 ++++++ indexer/beacon/statetransition/helpers.go | 117 +++++ .../beacon/statetransition/justification.go | 85 ++++ indexer/beacon/statetransition/lookahead.go | 100 +++++ indexer/beacon/statetransition/pending.go | 152 +++++++ indexer/beacon/statetransition/registry.go | 87 ++++ indexer/beacon/statetransition/resets.go | 116 +++++ indexer/beacon/statetransition/rewards.go | 145 +++++++ indexer/beacon/statetransition/slashings.go | 52 +++ indexer/beacon/statetransition/state.go | 408 ++++++++++++++++++ .../beacon/statetransition/statetransition.go | 243 +++++++++++ 12 files changed, 1655 insertions(+) create mode 100644 indexer/beacon/statetransition/balance.go create mode 100644 indexer/beacon/statetransition/builder.go create mode 100644 indexer/beacon/statetransition/helpers.go create mode 100644 indexer/beacon/statetransition/justification.go create mode 100644 indexer/beacon/statetransition/lookahead.go create mode 100644 indexer/beacon/statetransition/pending.go create mode 100644 indexer/beacon/statetransition/registry.go create mode 100644 indexer/beacon/statetransition/resets.go create mode 100644 indexer/beacon/statetransition/rewards.go create mode 100644 indexer/beacon/statetransition/slashings.go create mode 100644 indexer/beacon/statetransition/state.go create mode 100644 indexer/beacon/statetransition/statetransition.go diff --git a/indexer/beacon/statetransition/balance.go b/indexer/beacon/statetransition/balance.go new file mode 100644 index 000000000..31cd98d74 --- /dev/null +++ b/indexer/beacon/statetransition/balance.go @@ -0,0 +1,28 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processEffectiveBalanceUpdates implements the Electra+ version of +// process_effective_balance_updates. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_effective_balance_updates +func processEffectiveBalanceUpdates(s *stateAccessor) { + hysteresisIncrement := s.specs.EffectiveBalanceIncrement / s.specs.HysteresisQuotient + downwardThreshold := hysteresisIncrement * s.specs.HysteresisDownwardMultiplier + upwardThreshold := hysteresisIncrement * s.specs.HysteresisUpwardMultiplier + + for i, v := range s.Validators { + balance := uint64(s.Balances[i]) + maxEB := uint64(s.getMaxEffectiveBalance(v)) + eb := uint64(v.EffectiveBalance) + + if balance+downwardThreshold < eb || eb+upwardThreshold < balance { + newEB := balance - balance%s.specs.EffectiveBalanceIncrement + if newEB > maxEB { + newEB = maxEB + } + v.EffectiveBalance = phase0.Gwei(newEB) + } + } +} diff --git a/indexer/beacon/statetransition/builder.go b/indexer/beacon/statetransition/builder.go new file mode 100644 index 000000000..f8b7eefc3 --- /dev/null +++ b/indexer/beacon/statetransition/builder.go @@ -0,0 +1,122 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// Gloas-specific spec constants for builder payment quorum. +const ( + BuilderPaymentThresholdNumerator = 6 + BuilderPaymentThresholdDenominator = 10 +) + +// processBuilderPendingPayments implements process_builder_pending_payments (Gloas). +// Evaluates the first SLOTS_PER_EPOCH entries of BuilderPendingPayments against +// the quorum threshold. Qualifying payments are promoted to BuilderPendingWithdrawals. +// Then the 2-epoch window shifts forward. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-process_builder_pending_payments +func processBuilderPendingPayments(s *stateAccessor) { + slotsPerEpoch := s.specs.SlotsPerEpoch + quorum := getBuilderPaymentQuorumThreshold(s) + + // Evaluate first half (epoch K-1 payments) + limit := slotsPerEpoch + if limit > uint64(len(s.BuilderPendingPayments)) { + limit = uint64(len(s.BuilderPendingPayments)) + } + + for i := uint64(0); i < limit; i++ { + payment := s.BuilderPendingPayments[i] + if payment == nil || payment.Withdrawal == nil { + continue + } + if uint64(payment.Weight) >= quorum { + s.BuilderPendingWithdrawals = append(s.BuilderPendingWithdrawals, payment.Withdrawal) + } + } + + // Shift window: move second half to first, fill second with empty entries + if uint64(len(s.BuilderPendingPayments)) >= 2*slotsPerEpoch { + copy(s.BuilderPendingPayments[:slotsPerEpoch], s.BuilderPendingPayments[slotsPerEpoch:2*slotsPerEpoch]) + for i := slotsPerEpoch; i < 2*slotsPerEpoch; i++ { + s.BuilderPendingPayments[i] = &gloas.BuilderPendingPayment{} + } + } +} + +// getBuilderPaymentQuorumThreshold computes the quorum threshold for builder payments. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-get_builder_payment_quorum_threshold +func getBuilderPaymentQuorumThreshold(s *stateAccessor) uint64 { + totalActiveBalance := uint64(s.getTotalActiveBalance()) + perSlotBalance := totalActiveBalance / s.specs.SlotsPerEpoch + return perSlotBalance * BuilderPaymentThresholdNumerator / BuilderPaymentThresholdDenominator +} + +// processPtcWindow implements process_ptc_window (Gloas). +// Slides the PTC assignment window and computes new assignments for the lookahead epoch. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-process_ptc_window +func processPtcWindow(s *stateAccessor) { + // PTC window is not available for Fulu states in the accessor + if s.version < 8 { // DataVersionGloas + return + } + + g := s.rawState.Gloas + if g == nil || len(g.PTCWindow) == 0 { + return + } + + slotsPerEpoch := s.specs.SlotsPerEpoch + + // Slide window: remove first SLOTS_PER_EPOCH entries + windowLen := uint64(len(g.PTCWindow)) + if windowLen <= slotsPerEpoch { + return + } + + copy(g.PTCWindow, g.PTCWindow[slotsPerEpoch:]) + + // Compute new PTC assignments for the last SLOTS_PER_EPOCH entries + nextEpoch := s.currentEpoch() + phase0.Epoch(s.specs.MinSeedLookahead) + 1 + startSlot := uint64(nextEpoch) * slotsPerEpoch + + lastStart := windowLen - slotsPerEpoch + for i := uint64(0); i < slotsPerEpoch && lastStart+i < windowLen; i++ { + g.PTCWindow[lastStart+i] = computePtc(s, phase0.Slot(startSlot+i)) + } +} + +// computePtc computes the PTC (Payload Timeliness Committee) for a given slot. +// This selects PTC_SIZE validators from the beacon committees using the PTC seed. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-compute_ptc +func computePtc(s *stateAccessor, slot phase0.Slot) []phase0.ValidatorIndex { + epoch := phase0.Epoch(uint64(slot) / s.specs.SlotsPerEpoch) + activeIndices := s.getActiveValidatorIndices(epoch) + if len(activeIndices) == 0 { + return nil + } + + ptcSize := s.specs.PtcSize + if ptcSize == 0 { + return nil + } + + // Get the seed for PTC computation + seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainPtcAttester)) + + // Select PTC members using shuffling + committeeSize := uint64(len(activeIndices)) + result := make([]phase0.ValidatorIndex, 0, ptcSize) + + slotIndex := uint64(slot) % s.specs.SlotsPerEpoch + startOffset := (committeeSize * slotIndex * ptcSize) / (s.specs.SlotsPerEpoch * ptcSize) + + for i := uint64(0); i < ptcSize && i < committeeSize; i++ { + idx := (startOffset + i) % committeeSize + shuffledIdx := computeShuffledIndex(idx, committeeSize, seed, s.specs) + result = append(result, activeIndices[shuffledIdx]) + } + + return result +} diff --git a/indexer/beacon/statetransition/helpers.go b/indexer/beacon/statetransition/helpers.go new file mode 100644 index 000000000..c597053eb --- /dev/null +++ b/indexer/beacon/statetransition/helpers.go @@ -0,0 +1,117 @@ +package statetransition + +import ( + "crypto/sha256" + "encoding/binary" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" +) + +// hash256 returns the SHA-256 hash of data. +func hash256(data []byte) phase0.Root { + return phase0.Root(sha256.Sum256(data)) +} + +// hashTreeRoot is a placeholder for hash_tree_root of a vector of roots. +// For historical summaries, we need the Merkle root of block_roots and state_roots. +// This is a simplified version — the actual HTR of a fixed-size vector. +func hashTreeRoot(roots []phase0.Root) phase0.Root { + if len(roots) == 0 { + return phase0.Root{} + } + + // Build Merkle tree bottom-up + leaves := make([]phase0.Root, len(roots)) + copy(leaves, roots) + + // Pad to next power of 2 + size := uint64(1) + for size < uint64(len(leaves)) { + size *= 2 + } + for uint64(len(leaves)) < size { + leaves = append(leaves, phase0.Root{}) + } + + for len(leaves) > 1 { + next := make([]phase0.Root, len(leaves)/2) + for i := 0; i < len(next); i++ { + var buf [64]byte + copy(buf[:32], leaves[2*i][:]) + copy(buf[32:], leaves[2*i+1][:]) + next[i] = hash256(buf[:]) + } + leaves = next + } + + return leaves[0] +} + +// getSeed computes the seed for the given epoch and domain type. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_seed +func getSeed(s *stateAccessor, epoch phase0.Epoch, domainType phase0.DomainType) phase0.Root { + mixEpoch := epoch + phase0.Epoch(s.specs.EpochsPerHistoricalVector) - phase0.Epoch(s.specs.MinSeedLookahead) - 1 + mix := s.RANDAOMixes[uint64(mixEpoch)%s.specs.EpochsPerHistoricalVector] + + var buf [4 + 8 + 32]byte + copy(buf[0:4], domainType[:]) + binary.LittleEndian.PutUint64(buf[4:12], uint64(epoch)) + copy(buf[12:44], mix[:]) + + result := hash256(buf[:]) + return result +} + +// computeShuffledIndex implements the swap-or-not shuffle. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_shuffled_index +func computeShuffledIndex(index, indexCount uint64, seed phase0.Root, specs *consensus.ChainSpec) uint64 { + if indexCount == 0 { + return 0 + } + + for currentRound := uint64(0); currentRound < specs.ShuffleRoundCount; currentRound++ { + var buf [33]byte + copy(buf[0:32], seed[:]) + buf[32] = byte(currentRound) + pivotHash := hash256(buf[:]) + pivot := binary.LittleEndian.Uint64(pivotHash[:8]) % indexCount + + flip := (pivot + indexCount - index) % indexCount + position := index + if flip > index { + position = flip + } + + var buf2 [33 + 4]byte + copy(buf2[0:32], seed[:]) + buf2[32] = byte(currentRound) + binary.LittleEndian.PutUint32(buf2[33:37], uint32(position/256)) + source := hash256(buf2[:]) + + byteIdx := (position % 256) / 8 + bitIdx := position % 8 + bit := (source[byteIdx] >> bitIdx) & 1 + + if bit == 1 { + index = flip + } + } + + return index +} + +// getRandomByte extracts a pseudo-random byte from a seed. +func getRandomByte(seed phase0.Root, round, byteIndex uint64) uint8 { + var buf [40]byte + copy(buf[:32], seed[:]) + binary.LittleEndian.PutUint64(buf[32:40], round) + h := hash256(buf[:]) + return h[byteIndex%32] +} + +// computeActivationExitEpoch returns the epoch at which a validator activation/exit takes effect. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_activation_exit_epoch +func computeActivationExitEpoch(epoch phase0.Epoch, specs *consensus.ChainSpec) phase0.Epoch { + return epoch + 1 + phase0.Epoch(specs.MaxSeedLookahead) +} diff --git a/indexer/beacon/statetransition/justification.go b/indexer/beacon/statetransition/justification.go new file mode 100644 index 000000000..350cca1ef --- /dev/null +++ b/indexer/beacon/statetransition/justification.go @@ -0,0 +1,85 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processJustificationAndFinalization implements the Altair+ version of +// process_justification_and_finalization. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#justification-and-finalization +func processJustificationAndFinalization(s *stateAccessor) error { + currentEpoch := s.currentEpoch() + if currentEpoch <= 1 { + return nil + } + + previousEpoch := s.previousEpoch() + oldPreviousJustifiedCheckpoint := s.PreviousJustifiedCheckpoint + oldCurrentJustifiedCheckpoint := s.CurrentJustifiedCheckpoint + + // Process justification + s.PreviousJustifiedCheckpoint = s.CurrentJustifiedCheckpoint + + // Shift justification bits + if len(s.JustificationBits) > 0 { + s.JustificationBits[0] = (s.JustificationBits[0] << 1) & 0x0F + } + + totalActiveBalance := s.getTotalActiveBalance() + + // Previous epoch justification + previousTargetBalance := s.getUnslashedParticipatingBalance(TimelyTargetFlagIndex, previousEpoch) + if previousTargetBalance*3 >= totalActiveBalance*2 { + s.CurrentJustifiedCheckpoint = &phase0.Checkpoint{ + Epoch: previousEpoch, + Root: getBlockRoot(s, previousEpoch), + } + if len(s.JustificationBits) > 0 { + s.JustificationBits[0] |= 0x02 // bit 1 + } + } + + // Current epoch justification + currentTargetBalance := s.getUnslashedParticipatingBalance(TimelyTargetFlagIndex, currentEpoch) + if currentTargetBalance*3 >= totalActiveBalance*2 { + s.CurrentJustifiedCheckpoint = &phase0.Checkpoint{ + Epoch: currentEpoch, + Root: getBlockRoot(s, currentEpoch), + } + if len(s.JustificationBits) > 0 { + s.JustificationBits[0] |= 0x01 // bit 0 + } + } + + bits := byte(0) + if len(s.JustificationBits) > 0 { + bits = s.JustificationBits[0] + } + + // Process finalizations + // The 2/3/4th most recent epochs are justified, the 2nd using the 4th as source + if bits&0x0E == 0x0E && oldPreviousJustifiedCheckpoint.Epoch+3 == currentEpoch { + s.FinalizedCheckpoint = oldPreviousJustifiedCheckpoint + } + // The 2/3rd most recent epochs are justified, the 2nd using the 3rd as source + if bits&0x06 == 0x06 && oldPreviousJustifiedCheckpoint.Epoch+2 == currentEpoch { + s.FinalizedCheckpoint = oldPreviousJustifiedCheckpoint + } + // The 1/2/3rd most recent epochs are justified, the 1st using the 3rd as source + if bits&0x07 == 0x07 && oldCurrentJustifiedCheckpoint.Epoch+2 == currentEpoch { + s.FinalizedCheckpoint = oldCurrentJustifiedCheckpoint + } + // The 1/2nd most recent epochs are justified, the 1st using the 2nd as source + if bits&0x03 == 0x03 && oldCurrentJustifiedCheckpoint.Epoch+1 == currentEpoch { + s.FinalizedCheckpoint = oldCurrentJustifiedCheckpoint + } + + return nil +} + +// getBlockRoot returns the block root at the start of the given epoch. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_block_root +func getBlockRoot(s *stateAccessor, epoch phase0.Epoch) phase0.Root { + startSlot := uint64(epoch) * s.specs.SlotsPerEpoch + return s.BlockRoots[startSlot%s.specs.SlotsPerHistoricalRoot] +} diff --git a/indexer/beacon/statetransition/lookahead.go b/indexer/beacon/statetransition/lookahead.go new file mode 100644 index 000000000..781b45d15 --- /dev/null +++ b/indexer/beacon/statetransition/lookahead.go @@ -0,0 +1,100 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processProposerLookahead implements process_proposer_lookahead (Fulu+/EIP-7917). +// Slides the proposer lookahead window and computes new proposer indices for the +// lookahead epoch. +// New in Fulu: https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#new-process_proposer_lookahead +func processProposerLookahead(s *stateAccessor) { + slotsPerEpoch := s.specs.SlotsPerEpoch + lookaheadLen := uint64(len(s.ProposerLookahead)) + if lookaheadLen == 0 || slotsPerEpoch == 0 { + return + } + + // Slide window: shift left by SLOTS_PER_EPOCH + if lookaheadLen > slotsPerEpoch { + copy(s.ProposerLookahead, s.ProposerLookahead[slotsPerEpoch:]) + } + + // Compute new proposer indices for the last SLOTS_PER_EPOCH entries + nextEpoch := s.currentEpoch() + phase0.Epoch(s.specs.MinSeedLookahead) + 1 + proposers := getBeaconProposerIndices(s, nextEpoch) + + lastStart := lookaheadLen - slotsPerEpoch + for i := uint64(0); i < slotsPerEpoch && i < uint64(len(proposers)); i++ { + if lastStart+i < lookaheadLen { + s.ProposerLookahead[lastStart+i] = proposers[i] + } + } +} + +// getBeaconProposerIndices computes the proposer index for each slot in the given epoch. +// Spec: get_beacon_proposer_index applied to each slot. +func getBeaconProposerIndices(s *stateAccessor, epoch phase0.Epoch) []phase0.ValidatorIndex { + slotsPerEpoch := s.specs.SlotsPerEpoch + startSlot := uint64(epoch) * slotsPerEpoch + indices := make([]phase0.ValidatorIndex, slotsPerEpoch) + + activeIndices := s.getActiveValidatorIndices(epoch) + if len(activeIndices) == 0 { + return indices + } + + for slotOffset := uint64(0); slotOffset < slotsPerEpoch; slotOffset++ { + slot := phase0.Slot(startSlot + slotOffset) + indices[slotOffset] = computeProposerIndex(s, activeIndices, epoch, slot) + } + + return indices +} + +// computeProposerIndex selects the proposer for a specific slot using the +// spec's compute_proposer_index function. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_proposer_index +func computeProposerIndex(s *stateAccessor, activeIndices []phase0.ValidatorIndex, epoch phase0.Epoch, slot phase0.Slot) phase0.ValidatorIndex { + if len(activeIndices) == 0 { + return 0 + } + + seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainBeaconProposer)) + + // Mix in slot + var buf [40]byte + copy(buf[:32], seed[:]) + buf[32] = byte(slot) + buf[33] = byte(slot >> 8) + buf[34] = byte(slot >> 16) + buf[35] = byte(slot >> 24) + buf[36] = byte(slot >> 32) + buf[37] = byte(slot >> 40) + buf[38] = byte(slot >> 48) + buf[39] = byte(slot >> 56) + slotSeed := hash256(buf[:]) + + indexCount := uint64(len(activeIndices)) + maxEB := uint64(s.specs.MaxEffectiveBalanceElectra) + if maxEB == 0 { + maxEB = uint64(s.specs.MaxEffectiveBalance) + } + + i := uint64(0) + for { + candidateIndex := activeIndices[computeShuffledIndex(i%indexCount, indexCount, seed, s.specs)] + randomByte := getRandomByte(slotSeed, i/32, i%32) + + effectiveBalance := uint64(s.Validators[candidateIndex].EffectiveBalance) + if effectiveBalance*255 >= maxEB*uint64(randomByte) { + return candidateIndex + } + i++ + + // Safety: prevent infinite loop + if i > indexCount*100 { + return activeIndices[0] + } + } +} diff --git a/indexer/beacon/statetransition/pending.go b/indexer/beacon/statetransition/pending.go new file mode 100644 index 000000000..ed73172f0 --- /dev/null +++ b/indexer/beacon/statetransition/pending.go @@ -0,0 +1,152 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processPendingDeposits implements the Electra+ version of process_pending_deposits. +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_pending_deposits +func processPendingDeposits(s *stateAccessor) error { + availableForProcessing := s.DepositBalanceToConsume + s.getActivationExitChurnLimit() + + processedCount := 0 + nextDepositIndex := uint64(0) + depositsToPostpone := make([]*electra.PendingDeposit, 0) + isChurnLimitReached := false + + for _, deposit := range s.PendingDeposits { + // Check if deposit is finalized + if deposit.Slot > phase0.Slot(uint64(s.FinalizedCheckpoint.Epoch)*s.specs.SlotsPerEpoch) { + break + } + + processedCount++ + + // Check if validator exists + validatorIndex := findValidatorIndex(s, deposit.Pubkey) + if validatorIndex == nil { + // New validator: apply deposit + if !isChurnLimitReached { + cost := phase0.Gwei(deposit.Amount) + if cost > availableForProcessing { + isChurnLimitReached = true + depositsToPostpone = append(depositsToPostpone, deposit) + continue + } + availableForProcessing -= cost + applyPendingDeposit(s, deposit) + } else { + depositsToPostpone = append(depositsToPostpone, deposit) + } + continue + } + + // Existing validator + v := s.Validators[*validatorIndex] + + // Check for non-applied top-ups + if hasExecutionWithdrawalCredential(v) { + if !isChurnLimitReached { + cost := phase0.Gwei(deposit.Amount) + if cost > availableForProcessing { + isChurnLimitReached = true + depositsToPostpone = append(depositsToPostpone, deposit) + continue + } + availableForProcessing -= cost + } else { + depositsToPostpone = append(depositsToPostpone, deposit) + continue + } + } + + // Apply deposit to existing validator + applyPendingDepositToExisting(s, *validatorIndex, deposit) + + if deposit.Slot > phase0.Slot(0) { + nextDepositIndex = uint64(deposit.Slot) + } + } + + _ = nextDepositIndex // used for deposit index tracking in full spec + + // Remove processed deposits, keep postponed + unprocessed + remaining := make([]*electra.PendingDeposit, 0, len(depositsToPostpone)+len(s.PendingDeposits)-processedCount) + remaining = append(remaining, s.PendingDeposits[processedCount:]...) + remaining = append(remaining, depositsToPostpone...) + s.PendingDeposits = remaining + + // Update deposit balance to consume + if isChurnLimitReached { + s.DepositBalanceToConsume = availableForProcessing + } else { + s.DepositBalanceToConsume = 0 + } + + return nil +} + +// findValidatorIndex finds a validator by pubkey. Returns nil if not found. +func findValidatorIndex(s *stateAccessor, pubkey phase0.BLSPubKey) *phase0.ValidatorIndex { + for i, v := range s.Validators { + if v.PublicKey == pubkey { + idx := phase0.ValidatorIndex(i) + return &idx + } + } + return nil +} + +// applyPendingDeposit processes a deposit for a new validator. +func applyPendingDeposit(s *stateAccessor, deposit *electra.PendingDeposit) { + // Add new validator + v := &phase0.Validator{ + PublicKey: deposit.Pubkey, + WithdrawalCredentials: deposit.WithdrawalCredentials, + EffectiveBalance: 0, + Slashed: false, + ActivationEligibilityEpoch: FarFutureEpoch, + ActivationEpoch: FarFutureEpoch, + ExitEpoch: FarFutureEpoch, + WithdrawableEpoch: FarFutureEpoch, + } + s.Validators = append(s.Validators, v) + s.Balances = append(s.Balances, 0) + s.PreviousEpochParticipation = append(s.PreviousEpochParticipation, 0) + s.CurrentEpochParticipation = append(s.CurrentEpochParticipation, 0) + s.InactivityScores = append(s.InactivityScores, 0) + + idx := phase0.ValidatorIndex(len(s.Validators) - 1) + s.increaseBalance(idx, phase0.Gwei(deposit.Amount)) +} + +// applyPendingDepositToExisting applies a deposit to an existing validator. +func applyPendingDepositToExisting(s *stateAccessor, index phase0.ValidatorIndex, deposit *electra.PendingDeposit) { + s.increaseBalance(index, phase0.Gwei(deposit.Amount)) +} + +// processPendingConsolidations implements the Electra+ version of process_pending_consolidations. +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_pending_consolidations +func processPendingConsolidations(s *stateAccessor) { + nextEpoch := s.currentEpoch() + 1 + processedCount := 0 + + for _, consolidation := range s.PendingConsolidations { + sourceValidator := s.Validators[consolidation.SourceIndex] + if sourceValidator.WithdrawableEpoch > nextEpoch { + break + } + processedCount++ + + // Move balance from source to target + targetValidator := s.Validators[consolidation.TargetIndex] + _ = targetValidator // used for credential check in full spec + + activeBalance := s.Balances[consolidation.SourceIndex] + s.decreaseBalance(consolidation.SourceIndex, activeBalance) + s.increaseBalance(consolidation.TargetIndex, activeBalance) + } + + s.PendingConsolidations = s.PendingConsolidations[processedCount:] +} diff --git a/indexer/beacon/statetransition/registry.go b/indexer/beacon/statetransition/registry.go new file mode 100644 index 000000000..9544bd634 --- /dev/null +++ b/indexer/beacon/statetransition/registry.go @@ -0,0 +1,87 @@ +package statetransition + +import ( + "sort" + + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processRegistryUpdates implements the Electra+ version of process_registry_updates. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_registry_updates +func processRegistryUpdates(s *stateAccessor) error { + currentEpoch := s.currentEpoch() + activationExitChurnLimit := s.getActivationExitChurnLimit() + + // Process ejections + for i, v := range s.Validators { + if isActiveValidator(v, currentEpoch) && + v.EffectiveBalance <= phase0.Gwei(s.specs.EjectionBalance) { + initiateValidatorExit(s, phase0.ValidatorIndex(i)) + } + } + + // Set activation eligibility + for _, v := range s.Validators { + if isEligibleForActivationQueue(v, s.specs) { + v.ActivationEligibilityEpoch = currentEpoch + 1 + } + } + + // Dequeue validators for activation (Electra+: balance-based churn) + activationQueue := make([]phase0.ValidatorIndex, 0) + for i, v := range s.Validators { + if isEligibleForActivation(v, s.FinalizedCheckpoint.Epoch) { + activationQueue = append(activationQueue, phase0.ValidatorIndex(i)) + } + } + + // Sort by activation eligibility epoch, then by index + sort.Slice(activationQueue, func(i, j int) bool { + vi := s.Validators[activationQueue[i]] + vj := s.Validators[activationQueue[j]] + if vi.ActivationEligibilityEpoch != vj.ActivationEligibilityEpoch { + return vi.ActivationEligibilityEpoch < vj.ActivationEligibilityEpoch + } + return activationQueue[i] < activationQueue[j] + }) + + // Activate validators up to the churn limit + activatedBalance := phase0.Gwei(0) + for _, idx := range activationQueue { + v := s.Validators[idx] + if activatedBalance+s.getMaxEffectiveBalance(v) > activationExitChurnLimit { + break + } + activatedBalance += s.getMaxEffectiveBalance(v) + v.ActivationEpoch = computeActivationExitEpoch(currentEpoch, s.specs) + } + + return nil +} + +// initiateValidatorExit queues a validator for exit. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-initiate_validator_exit +func initiateValidatorExit(s *stateAccessor, index phase0.ValidatorIndex) { + v := s.Validators[index] + if v.ExitEpoch != FarFutureEpoch { + return // already exiting + } + + exitQueueEpoch := computeActivationExitEpoch(s.currentEpoch(), s.specs) + if s.EarliestExitEpoch > exitQueueEpoch { + exitQueueEpoch = s.EarliestExitEpoch + } + + // Consume exit churn + exitBalance := s.getMaxEffectiveBalance(v) + if s.ExitBalanceToConsume < exitBalance { + // Not enough churn left, push to next epoch + s.ExitBalanceToConsume += s.getActivationExitChurnLimit() + exitQueueEpoch++ + } + s.ExitBalanceToConsume -= exitBalance + s.EarliestExitEpoch = exitQueueEpoch + + v.ExitEpoch = exitQueueEpoch + v.WithdrawableEpoch = exitQueueEpoch + phase0.Epoch(s.specs.MinValidatorWithdrawbilityDelay) +} diff --git a/indexer/beacon/statetransition/resets.go b/indexer/beacon/statetransition/resets.go new file mode 100644 index 000000000..74578497b --- /dev/null +++ b/indexer/beacon/statetransition/resets.go @@ -0,0 +1,116 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processEth1DataReset resets the ETH1 data votes at the start of a new voting period. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#eth1-data-votes-updates +func processEth1DataReset(s *stateAccessor) { + nextEpoch := s.currentEpoch() + 1 + if uint64(nextEpoch)%s.specs.EpochsPerEth1VotingPeriod == 0 { + s.ETH1DataVotes = s.ETH1DataVotes[:0] + } +} + +// processSlashingsReset rotates the slashings vector. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#slashings-balances-updates +func processSlashingsReset(s *stateAccessor) { + nextEpoch := s.currentEpoch() + 1 + idx := uint64(nextEpoch) % s.specs.EpochsPerSlashingVector + s.Slashings[idx] = 0 +} + +// processRandaoMixesReset copies the current epoch mix to the next epoch slot. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#randao-mixes-updates +func processRandaoMixesReset(s *stateAccessor) { + currentEpoch := s.currentEpoch() + nextEpoch := currentEpoch + 1 + srcIdx := uint64(currentEpoch) % s.specs.EpochsPerHistoricalVector + dstIdx := uint64(nextEpoch) % s.specs.EpochsPerHistoricalVector + s.RANDAOMixes[dstIdx] = s.RANDAOMixes[srcIdx] +} + +// processParticipationFlagUpdates rotates epoch participation. +// New in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#participation-flags-updates +func processParticipationFlagUpdates(s *stateAccessor) { + s.PreviousEpochParticipation = s.CurrentEpochParticipation + s.CurrentEpochParticipation = make([]altair.ParticipationFlags, len(s.Validators)) +} + +// processHistoricalSummariesUpdate appends a new historical summary at period boundaries. +// Modified in Capella: https://github.com/ethereum/consensus-specs/blob/master/specs/capella/beacon-chain.md#modified-process_historical_summaries_update +func processHistoricalSummariesUpdate(s *stateAccessor) { + nextEpoch := s.currentEpoch() + 1 + epochsPerPeriod := s.specs.SlotsPerHistoricalRoot / s.specs.SlotsPerEpoch + if epochsPerPeriod == 0 { + return + } + if uint64(nextEpoch)%epochsPerPeriod != 0 { + return + } + + blockSummary := hashTreeRoot(s.BlockRoots) + stateSummary := hashTreeRoot(s.StateRoots) + + s.HistoricalSummaries = append(s.HistoricalSummaries, &capella.HistoricalSummary{ + BlockSummaryRoot: blockSummary, + StateSummaryRoot: stateSummary, + }) +} + +// processSyncCommitteeUpdates rotates the sync committee at period boundaries. +// New in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#sync-committee-updates +func processSyncCommitteeUpdates(s *stateAccessor) { + nextEpoch := s.currentEpoch() + 1 + if s.specs.EpochsPerSyncCommitteePeriod == 0 { + return + } + if uint64(nextEpoch)%s.specs.EpochsPerSyncCommitteePeriod != 0 { + return + } + + s.CurrentSyncCommittee = s.NextSyncCommittee + s.NextSyncCommittee = computeNextSyncCommittee(s) +} + +// computeNextSyncCommittee computes the next sync committee. +// https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#get_next_sync_committee +func computeNextSyncCommittee(s *stateAccessor) *altair.SyncCommittee { + indices := s.getActiveValidatorIndices(s.currentEpoch() + 1) + if len(indices) == 0 { + return s.NextSyncCommittee // fallback: keep current + } + + epoch := s.currentEpoch() + 1 + seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainSyncCommittee)) + + syncCommitteeSize := s.specs.SyncCommitteeSize + committee := make([]phase0.ValidatorIndex, 0, syncCommitteeSize) + pubkeys := make([]phase0.BLSPubKey, 0, syncCommitteeSize) + + i := uint64(0) + for uint64(len(committee)) < syncCommitteeSize { + shuffledIndex := computeShuffledIndex(i%uint64(len(indices)), uint64(len(indices)), seed, s.specs) + candidateIndex := indices[shuffledIndex] + + randomByte := getRandomByte(seed, i/32, i%32) + effectiveBalance := s.Validators[candidateIndex].EffectiveBalance + maxEB := s.getMaxEffectiveBalance(s.Validators[candidateIndex]) + + if effectiveBalance*255 >= maxEB*phase0.Gwei(randomByte) { + committee = append(committee, candidateIndex) + pubkeys = append(pubkeys, s.Validators[candidateIndex].PublicKey) + } + i++ + } + + // Compute aggregate pubkey placeholder (we don't actually need it for the explorer, + // but the struct requires it). Use empty value. + return &altair.SyncCommittee{ + Pubkeys: pubkeys, + AggregatePubkey: phase0.BLSPubKey{}, + } +} diff --git a/indexer/beacon/statetransition/rewards.go b/indexer/beacon/statetransition/rewards.go new file mode 100644 index 000000000..ff4dfc67e --- /dev/null +++ b/indexer/beacon/statetransition/rewards.go @@ -0,0 +1,145 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processInactivityUpdates implements process_inactivity_updates (Altair+). +// https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#inactivity-scores +func processInactivityUpdates(s *stateAccessor) error { + currentEpoch := s.currentEpoch() + if currentEpoch <= 1 { + return nil + } + + previousEpoch := s.previousEpoch() + isInactivityLeak := isInInactivityLeak(s) + + // Build set of timely target participants for previous epoch + targetParticipants := make(map[phase0.ValidatorIndex]bool) + for _, idx := range s.getUnslashedParticipatingIndices(TimelyTargetFlagIndex, previousEpoch) { + targetParticipants[idx] = true + } + + for i, v := range s.Validators { + if !isActiveValidator(v, previousEpoch) || v.Slashed { + continue + } + + idx := phase0.ValidatorIndex(i) + if targetParticipants[idx] { + // Decrease inactivity score + if s.InactivityScores[i] > 0 { + decrease := s.specs.InactivityScoreRecoveryRate + if s.InactivityScores[i] < decrease { + s.InactivityScores[i] = 0 + } else { + s.InactivityScores[i] -= decrease + } + } + } else { + // Increase inactivity score + s.InactivityScores[i] += s.specs.InactivityScoreBias + } + + if !isInactivityLeak { + // Not in inactivity leak: decrease score faster + if s.InactivityScores[i] > 0 { + decrease := uint64(1) + if s.InactivityScores[i] < decrease { + s.InactivityScores[i] = 0 + } else { + s.InactivityScores[i] -= decrease + } + } + } + } + + return nil +} + +// processRewardsAndPenalties implements the Altair+ version of process_rewards_and_penalties. +// Modified in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#modified-get_flag_index_deltas +func processRewardsAndPenalties(s *stateAccessor) error { + currentEpoch := s.currentEpoch() + if currentEpoch <= 1 { + return nil + } + + previousEpoch := s.previousEpoch() + totalActiveBalance := s.getTotalActiveBalance() + isInactivityLeak := isInInactivityLeak(s) + + // Precompute participating balances and sets for each flag + type flagData struct { + participatingBalance phase0.Gwei + participants map[phase0.ValidatorIndex]bool + } + + flags := make([]flagData, ParticipationFlagCount) + for fi := 0; fi < ParticipationFlagCount; fi++ { + indices := s.getUnslashedParticipatingIndices(fi, previousEpoch) + balance := phase0.Gwei(0) + pMap := make(map[phase0.ValidatorIndex]bool, len(indices)) + for _, idx := range indices { + pMap[idx] = true + balance += s.Validators[idx].EffectiveBalance + } + if balance < phase0.Gwei(s.specs.EffectiveBalanceIncrement) { + balance = phase0.Gwei(s.specs.EffectiveBalanceIncrement) + } + flags[fi] = flagData{participatingBalance: balance, participants: pMap} + } + + for i, v := range s.Validators { + if !isActiveValidator(v, previousEpoch) && !(v.Slashed && previousEpoch+1 < v.WithdrawableEpoch) { + continue + } + + idx := phase0.ValidatorIndex(i) + baseReward := s.getBaseReward(idx) + + for fi := 0; fi < ParticipationFlagCount; fi++ { + weight := ParticipationFlagWeights[fi] + + if flags[fi].participants[idx] && !v.Slashed { + if !isInactivityLeak { + // Reward + rewardNumerator := baseReward * phase0.Gwei(weight) * flags[fi].participatingBalance + reward := rewardNumerator / (totalActiveBalance * WeightDenominator) + s.increaseBalance(idx, reward) + } + } else { + // Penalty + penalty := baseReward * phase0.Gwei(weight) / WeightDenominator + s.decreaseBalance(idx, penalty) + } + } + + // Inactivity penalty (additional penalty for validators not participating in target) + if !flags[TimelyTargetFlagIndex].participants[idx] || v.Slashed { + penaltyNumerator := v.EffectiveBalance * phase0.Gwei(s.InactivityScores[i]) + penaltyDenominator := phase0.Gwei(s.specs.InactivityScoreRecoveryRate * s.getInactivityPenaltyQuotient()) + if penaltyDenominator > 0 { + penalty := penaltyNumerator / penaltyDenominator + s.decreaseBalance(idx, penalty) + } + } + } + + return nil +} + +// isInInactivityLeak checks if the chain is in an inactivity leak. +func isInInactivityLeak(s *stateAccessor) bool { + return s.previousEpoch()-s.FinalizedCheckpoint.Epoch > phase0.Epoch(s.specs.MinEpochsToInactivityPenalty) +} + +// getInactivityPenaltyQuotient returns the inactivity penalty quotient for the current fork. +func (s *stateAccessor) getInactivityPenaltyQuotient() uint64 { + // Electra/Fulu/Gloas use Bellatrix quotient + if s.specs.InactivityPenaltyQuotientBellatrix > 0 { + return s.specs.InactivityPenaltyQuotientBellatrix + } + return s.specs.InactivityPenaltyQuotient +} diff --git a/indexer/beacon/statetransition/slashings.go b/indexer/beacon/statetransition/slashings.go new file mode 100644 index 000000000..b60d6d8c9 --- /dev/null +++ b/indexer/beacon/statetransition/slashings.go @@ -0,0 +1,52 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processSlashings implements the Electra+ version of process_slashings. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#slashings +// Modified in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#modified-process_slashings +func processSlashings(s *stateAccessor) error { + currentEpoch := s.currentEpoch() + totalBalance := s.getTotalActiveBalance() + + totalSlashings := phase0.Gwei(0) + for _, slashing := range s.Slashings { + totalSlashings += slashing + } + + // Electra+: PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3 + proportionalSlashingMultiplier := s.specs.ProportionalSlashingMultiplierBellatrix + if proportionalSlashingMultiplier == 0 { + proportionalSlashingMultiplier = s.specs.ProportionalSlashingMultiplier + } + + adjustedTotalSlashingBalance := totalSlashings * phase0.Gwei(proportionalSlashingMultiplier) + if adjustedTotalSlashingBalance > totalBalance { + adjustedTotalSlashingBalance = totalBalance + } + + for i, v := range s.Validators { + if !v.Slashed { + continue + } + + withdrawableEpoch := v.WithdrawableEpoch + halfSlashingsVector := phase0.Epoch(s.specs.EpochsPerSlashingVector / 2) + + if currentEpoch+halfSlashingsVector != withdrawableEpoch { + continue + } + + // Electra+: use per-validator max effective balance + effectiveBalance := v.EffectiveBalance + increment := phase0.Gwei(s.specs.EffectiveBalanceIncrement) + penaltyNumerator := effectiveBalance / increment * adjustedTotalSlashingBalance + penalty := penaltyNumerator / totalBalance * increment + + s.decreaseBalance(phase0.ValidatorIndex(i), penalty) + } + + return nil +} diff --git a/indexer/beacon/statetransition/state.go b/indexer/beacon/statetransition/state.go new file mode 100644 index 000000000..bdb432abd --- /dev/null +++ b/indexer/beacon/statetransition/state.go @@ -0,0 +1,408 @@ +package statetransition + +import ( + "fmt" + "math" + + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" +) + +// stateAccessor provides a unified interface to access and mutate beacon state fields +// across Fulu and Gloas versions. All fields are pointers/slices into the underlying +// VersionedBeaconState, so mutations are applied in-place. +type stateAccessor struct { + version spec.DataVersion + specs *consensus.ChainSpec + + // Common fields shared by Fulu and Gloas (pointers into the underlying state). + Slot phase0.Slot + Validators []*phase0.Validator + Balances []phase0.Gwei + RANDAOMixes []phase0.Root + Slashings []phase0.Gwei + PreviousEpochParticipation []altair.ParticipationFlags + CurrentEpochParticipation []altair.ParticipationFlags + JustificationBits []byte // bitfield.Bitvector4 is []byte + PreviousJustifiedCheckpoint *phase0.Checkpoint + CurrentJustifiedCheckpoint *phase0.Checkpoint + FinalizedCheckpoint *phase0.Checkpoint + InactivityScores []uint64 + CurrentSyncCommittee *altair.SyncCommittee + NextSyncCommittee *altair.SyncCommittee + ETH1DataVotes []*phase0.ETH1Data + BlockRoots []phase0.Root + StateRoots []phase0.Root + HistoricalSummaries []*capella.HistoricalSummary + DepositRequestsStartIndex uint64 + DepositBalanceToConsume phase0.Gwei + ExitBalanceToConsume phase0.Gwei + EarliestExitEpoch phase0.Epoch + ConsolidationBalanceToConsume phase0.Gwei + EarliestConsolidationEpoch phase0.Epoch + PendingDeposits []*electra.PendingDeposit + PendingPartialWithdrawals []*electra.PendingPartialWithdrawal + PendingConsolidations []*electra.PendingConsolidation + ProposerLookahead []phase0.ValidatorIndex + + // Gloas-only fields (nil for Fulu) + BuilderPendingPayments []*gloas.BuilderPendingPayment + BuilderPendingWithdrawals []*gloas.BuilderPendingWithdrawal + + // Back-references for writing mutated slices/values back to the underlying state. + rawState *spec.VersionedBeaconState +} + +func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSpec) (*stateAccessor, error) { + s := &stateAccessor{ + version: state.Version, + specs: specs, + rawState: state, + } + + switch state.Version { + case spec.DataVersionFulu: + if state.Fulu == nil { + return nil, fmt.Errorf("nil fulu state") + } + f := state.Fulu + s.Slot = f.Slot + s.Validators = f.Validators + s.Balances = f.Balances + s.RANDAOMixes = f.RANDAOMixes + s.Slashings = f.Slashings + s.PreviousEpochParticipation = f.PreviousEpochParticipation + s.CurrentEpochParticipation = f.CurrentEpochParticipation + s.JustificationBits = f.JustificationBits + s.PreviousJustifiedCheckpoint = f.PreviousJustifiedCheckpoint + s.CurrentJustifiedCheckpoint = f.CurrentJustifiedCheckpoint + s.FinalizedCheckpoint = f.FinalizedCheckpoint + s.InactivityScores = f.InactivityScores + s.CurrentSyncCommittee = f.CurrentSyncCommittee + s.NextSyncCommittee = f.NextSyncCommittee + s.ETH1DataVotes = f.ETH1DataVotes + s.BlockRoots = f.BlockRoots + s.StateRoots = f.StateRoots + s.HistoricalSummaries = f.HistoricalSummaries + s.DepositRequestsStartIndex = f.DepositRequestsStartIndex + s.DepositBalanceToConsume = f.DepositBalanceToConsume + s.ExitBalanceToConsume = f.ExitBalanceToConsume + s.EarliestExitEpoch = f.EarliestExitEpoch + s.ConsolidationBalanceToConsume = f.ConsolidationBalanceToConsume + s.EarliestConsolidationEpoch = f.EarliestConsolidationEpoch + s.PendingDeposits = f.PendingDeposits + s.PendingPartialWithdrawals = f.PendingPartialWithdrawals + s.PendingConsolidations = f.PendingConsolidations + s.ProposerLookahead = f.ProposerLookahead + case spec.DataVersionGloas: + if state.Gloas == nil { + return nil, fmt.Errorf("nil gloas state") + } + g := state.Gloas + s.Slot = g.Slot + s.Validators = g.Validators + s.Balances = g.Balances + s.RANDAOMixes = g.RANDAOMixes + s.Slashings = g.Slashings + s.PreviousEpochParticipation = g.PreviousEpochParticipation + s.CurrentEpochParticipation = g.CurrentEpochParticipation + s.JustificationBits = g.JustificationBits + s.PreviousJustifiedCheckpoint = g.PreviousJustifiedCheckpoint + s.CurrentJustifiedCheckpoint = g.CurrentJustifiedCheckpoint + s.FinalizedCheckpoint = g.FinalizedCheckpoint + s.InactivityScores = g.InactivityScores + s.CurrentSyncCommittee = g.CurrentSyncCommittee + s.NextSyncCommittee = g.NextSyncCommittee + s.ETH1DataVotes = g.ETH1DataVotes + s.BlockRoots = g.BlockRoots + s.StateRoots = g.StateRoots + s.HistoricalSummaries = g.HistoricalSummaries + s.DepositRequestsStartIndex = g.DepositRequestsStartIndex + s.DepositBalanceToConsume = g.DepositBalanceToConsume + s.ExitBalanceToConsume = g.ExitBalanceToConsume + s.EarliestExitEpoch = g.EarliestExitEpoch + s.ConsolidationBalanceToConsume = g.ConsolidationBalanceToConsume + s.EarliestConsolidationEpoch = g.EarliestConsolidationEpoch + s.PendingDeposits = g.PendingDeposits + s.PendingPartialWithdrawals = g.PendingPartialWithdrawals + s.PendingConsolidations = g.PendingConsolidations + s.ProposerLookahead = g.ProposerLookahead + s.BuilderPendingPayments = g.BuilderPendingPayments + s.BuilderPendingWithdrawals = g.BuilderPendingWithdrawals + default: + return nil, fmt.Errorf("unsupported state version: %v", state.Version) + } + + return s, nil +} + +// writeBack writes mutated slice headers and scalar fields back to the underlying +// VersionedBeaconState. This is needed because Go slice reassignment (e.g. +// s.Balances = newSlice) doesn't update the original struct field. +// Call this after all epoch processing is complete. +func (s *stateAccessor) writeBack() { + switch s.version { + case spec.DataVersionFulu: + f := s.rawState.Fulu + f.Slot = s.Slot + f.Validators = s.Validators + f.Balances = s.Balances + f.RANDAOMixes = s.RANDAOMixes + f.Slashings = s.Slashings + f.PreviousEpochParticipation = s.PreviousEpochParticipation + f.CurrentEpochParticipation = s.CurrentEpochParticipation + f.JustificationBits = s.JustificationBits + f.InactivityScores = s.InactivityScores + f.ETH1DataVotes = s.ETH1DataVotes + f.HistoricalSummaries = s.HistoricalSummaries + f.DepositBalanceToConsume = s.DepositBalanceToConsume + f.ExitBalanceToConsume = s.ExitBalanceToConsume + f.EarliestExitEpoch = s.EarliestExitEpoch + f.ConsolidationBalanceToConsume = s.ConsolidationBalanceToConsume + f.EarliestConsolidationEpoch = s.EarliestConsolidationEpoch + f.PendingDeposits = s.PendingDeposits + f.PendingPartialWithdrawals = s.PendingPartialWithdrawals + f.PendingConsolidations = s.PendingConsolidations + f.ProposerLookahead = s.ProposerLookahead + case spec.DataVersionGloas: + g := s.rawState.Gloas + g.Slot = s.Slot + g.Validators = s.Validators + g.Balances = s.Balances + g.RANDAOMixes = s.RANDAOMixes + g.Slashings = s.Slashings + g.PreviousEpochParticipation = s.PreviousEpochParticipation + g.CurrentEpochParticipation = s.CurrentEpochParticipation + g.JustificationBits = s.JustificationBits + g.InactivityScores = s.InactivityScores + g.ETH1DataVotes = s.ETH1DataVotes + g.HistoricalSummaries = s.HistoricalSummaries + g.DepositBalanceToConsume = s.DepositBalanceToConsume + g.ExitBalanceToConsume = s.ExitBalanceToConsume + g.EarliestExitEpoch = s.EarliestExitEpoch + g.ConsolidationBalanceToConsume = s.ConsolidationBalanceToConsume + g.EarliestConsolidationEpoch = s.EarliestConsolidationEpoch + g.PendingDeposits = s.PendingDeposits + g.PendingPartialWithdrawals = s.PendingPartialWithdrawals + g.PendingConsolidations = s.PendingConsolidations + g.ProposerLookahead = s.ProposerLookahead + g.BuilderPendingPayments = s.BuilderPendingPayments + g.BuilderPendingWithdrawals = s.BuilderPendingWithdrawals + } +} + +// currentEpoch returns the current epoch derived from the slot. +func (s *stateAccessor) currentEpoch() phase0.Epoch { + return phase0.Epoch(uint64(s.Slot) / s.specs.SlotsPerEpoch) +} + +// previousEpoch returns the previous epoch (minimum 0). +func (s *stateAccessor) previousEpoch() phase0.Epoch { + epoch := s.currentEpoch() + if epoch == 0 { + return 0 + } + return epoch - 1 +} + +// isActiveValidator checks if a validator is active at the given epoch. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#is_active_validator +func isActiveValidator(v *phase0.Validator, epoch phase0.Epoch) bool { + return v.ActivationEpoch <= epoch && epoch < v.ExitEpoch +} + +// getActiveValidatorIndices returns all active validator indices for the given epoch. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_active_validator_indices +func (s *stateAccessor) getActiveValidatorIndices(epoch phase0.Epoch) []phase0.ValidatorIndex { + indices := make([]phase0.ValidatorIndex, 0, len(s.Validators)) + for i, v := range s.Validators { + if isActiveValidator(v, epoch) { + indices = append(indices, phase0.ValidatorIndex(i)) + } + } + return indices +} + +// getTotalActiveBalance returns the sum of effective balances for all active validators. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_total_active_balance +func (s *stateAccessor) getTotalActiveBalance() phase0.Gwei { + total := phase0.Gwei(0) + epoch := s.currentEpoch() + for _, v := range s.Validators { + if isActiveValidator(v, epoch) { + total += v.EffectiveBalance + } + } + if total < phase0.Gwei(s.specs.EffectiveBalanceIncrement) { + return phase0.Gwei(s.specs.EffectiveBalanceIncrement) + } + return total +} + +// getMaxEffectiveBalance returns the max effective balance for a validator (Electra+). +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-get_max_effective_balance +func (s *stateAccessor) getMaxEffectiveBalance(v *phase0.Validator) phase0.Gwei { + if hasCompoundingWithdrawalCredential(v) { + return phase0.Gwei(s.specs.MaxEffectiveBalanceElectra) + } + return phase0.Gwei(s.specs.MinActivationBalance) +} + +// hasCompoundingWithdrawalCredential checks for 0x02 withdrawal credential prefix. +func hasCompoundingWithdrawalCredential(v *phase0.Validator) bool { + return len(v.WithdrawalCredentials) > 0 && v.WithdrawalCredentials[0] == 0x02 +} + +// hasExecutionWithdrawalCredential checks for 0x01 or 0x02 withdrawal credential prefix. +func hasExecutionWithdrawalCredential(v *phase0.Validator) bool { + if len(v.WithdrawalCredentials) == 0 { + return false + } + return v.WithdrawalCredentials[0] == 0x01 || v.WithdrawalCredentials[0] == 0x02 +} + +// isEligibleForActivationQueue checks if a validator is eligible to be added to activation queue. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-is_eligible_for_activation_queue +func isEligibleForActivationQueue(v *phase0.Validator, specs *consensus.ChainSpec) bool { + return v.ActivationEligibilityEpoch == FarFutureEpoch && + v.EffectiveBalance >= phase0.Gwei(specs.MinActivationBalance) +} + +// isEligibleForActivation checks if a validator is eligible for activation. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#is_eligible_for_activation +func isEligibleForActivation(v *phase0.Validator, finalizedEpoch phase0.Epoch) bool { + return v.ActivationEligibilityEpoch <= finalizedEpoch && + v.ActivationEpoch == FarFutureEpoch +} + +// intSqrt returns the integer square root of n. +func intSqrt(n uint64) uint64 { + if n == 0 { + return 0 + } + x := n + y := (x + 1) / 2 + for y < x { + x = y + y = (x + n/x) / 2 + } + return x +} + +// FarFutureEpoch is the sentinel value for unset epochs. +const FarFutureEpoch = phase0.Epoch(math.MaxUint64) + +// Altair constants. +const ( + TimelySourceFlagIndex = 0 + TimelyTargetFlagIndex = 1 + TimelyHeadFlagIndex = 2 + + TimelySourceWeight = 14 + TimelyTargetWeight = 26 + TimelyHeadWeight = 14 + SyncRewardWeight = 2 + ProposerWeight = 8 + WeightDenominator = 64 + + ParticipationFlagCount = 3 + BaseRewardsPerEpoch = 4 +) + +var ParticipationFlagWeights = [ParticipationFlagCount]uint64{ + TimelySourceWeight, + TimelyTargetWeight, + TimelyHeadWeight, +} + +// hasFlag checks if a participation flags byte has the given flag set. +func hasFlag(flags altair.ParticipationFlags, flagIndex int) bool { + return flags&altair.ParticipationFlags(1<= delta { + s.Balances[index] -= delta + } else { + s.Balances[index] = 0 + } +} + +// getBalanceChurnLimit returns the balance churn limit (Electra+). +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-get_balance_churn_limit +func (s *stateAccessor) getBalanceChurnLimit() phase0.Gwei { + churn := uint64(s.getTotalActiveBalance()) / s.specs.ChurnLimitQuotient + if s.specs.MinPerEpochChurnLimitElectra > churn { + churn = s.specs.MinPerEpochChurnLimitElectra + } + return phase0.Gwei(churn - churn%s.specs.EffectiveBalanceIncrement) +} + +// getActivationExitChurnLimit returns the activation/exit churn limit. +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-get_activation_exit_churn_limit +func (s *stateAccessor) getActivationExitChurnLimit() phase0.Gwei { + churn := s.getBalanceChurnLimit() + if phase0.Gwei(s.specs.MaxPerEpochActivationExitChurnLimit) < churn { + return phase0.Gwei(s.specs.MaxPerEpochActivationExitChurnLimit) + } + return churn +} diff --git a/indexer/beacon/statetransition/statetransition.go b/indexer/beacon/statetransition/statetransition.go new file mode 100644 index 000000000..d8f461825 --- /dev/null +++ b/indexer/beacon/statetransition/statetransition.go @@ -0,0 +1,243 @@ +// Package statetransition implements consensus-spec state transition functions +// for Fulu+ beacon states. +// +// The primary entry point is PrepareEpochPreState, which takes a post-block state +// (typically the last block of a parent epoch) and advances it to the pre-state +// of a target epoch by applying payload processing (Gloas+) and epoch transitions. +// +// This produces the normally inaccessible pre-slot-1, post-epoch-transition state +// that the beacon API cannot serve directly. +// +// Only needed for Fulu+ states. Pre-Fulu states already provide the correct +// epoch boundary values from the post-state of the parent epoch's last block. +package statetransition + +import ( + "fmt" + + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" +) + +// PrepareEpochPreState takes a post-block state and mutates it into the pre-state +// of the target epoch. This is the main entry point for epoch state preparation. +// +// The input state is typically the post-state of the last block of a parent epoch. +// The function: +// 1. For Gloas+: if the state is pre-payload and a payload envelope is provided, +// applies the execution payload transition first. +// 2. Advances the state to the first slot of the target epoch, applying epoch +// transitions at every epoch boundary crossed (handles skipped slots and epochs). +// +// After this call, the state represents the pre-block state at the first slot of +// the target epoch, with all epoch transitions applied — including builder payment +// conversions, balance updates, proposer lookahead, etc. +func PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec) error { + if state.Version < spec.DataVersionFulu { + return nil + } + + // Step 1: For Gloas+ pre-payload states, apply the execution payload transition. + if payload != nil && isPrePayloadState(state) { + if err := processExecutionPayload(state, payload, specs); err != nil { + return fmt.Errorf("process_execution_payload: %w", err) + } + } + + // Step 2: Advance to the first slot of the target epoch. + targetSlot := phase0.Slot(uint64(epoch) * specs.SlotsPerEpoch) + if err := processSlots(state, targetSlot, specs); err != nil { + return fmt.Errorf("process_slots to epoch %d (slot %d): %w", epoch, targetSlot, err) + } + + return nil +} + +// processSlots advances the state from its current slot to targetSlot, applying +// epoch transitions at every epoch boundary crossed. +// +// Skips per-slot state/block root caching (process_slot) since we cannot compute +// hash_tree_root efficiently and the cached roots don't affect the epoch transition +// outputs we need. Jumps directly to each epoch boundary. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots +func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec) error { + currentSlot, err := state.Slot() + if err != nil { + return fmt.Errorf("failed to get state slot: %w", err) + } + + if currentSlot >= targetSlot { + return nil + } + + s, err := newStateAccessor(state, specs) + if err != nil { + return fmt.Errorf("failed to create state accessor: %w", err) + } + + slotsPerEpoch := specs.SlotsPerEpoch + + // Jump to each epoch boundary and apply the epoch transition. + // The spec applies process_epoch when (state.slot + 1) % SLOTS_PER_EPOCH == 0, + // i.e. at the last slot of each epoch, then increments the slot. + for { + nextBoundary := phase0.Slot(((uint64(s.Slot)/slotsPerEpoch)+1)*slotsPerEpoch - 1) + + if nextBoundary >= targetSlot { + break + } + + s.Slot = nextBoundary + if err := processEpochInternal(s); err != nil { + return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) + } + + s.Slot++ // cross into next epoch + } + + s.Slot = targetSlot + s.writeBack() + + return nil +} + +// isPrePayloadState checks whether a Gloas state is pre-payload +// (the execution payload for the latest block has NOT been processed yet). +func isPrePayloadState(state *spec.VersionedBeaconState) bool { + if state.Version < spec.DataVersionGloas || state.Gloas == nil { + return false + } + + slot, err := state.Slot() + if err != nil { + return false + } + + bitfieldLen := uint64(len(state.Gloas.ExecutionPayloadAvailability)) * 8 + if bitfieldLen == 0 { + return true + } + + idx := uint64(slot) % bitfieldLen + return state.Gloas.ExecutionPayloadAvailability[idx/8]&(1<<(idx%8)) == 0 +} + +// processExecutionPayload applies the Gloas execution payload state transition +// on a pre-payload state. Processes execution requests and records the builder +// payment, transitioning the state from pre-payload to post-payload. +// +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_execution_payload +func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec) error { + if state.Version < spec.DataVersionGloas || state.Gloas == nil || envelope == nil { + return nil + } + + g := state.Gloas + slotsPerEpoch := specs.SlotsPerEpoch + + // Process deposit requests → convert to pending deposits. + // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_deposit_request + if envelope.ExecutionRequests != nil { + for _, deposit := range envelope.ExecutionRequests.Deposits { + g.PendingDeposits = append(g.PendingDeposits, &electra.PendingDeposit{ + Pubkey: deposit.Pubkey, + WithdrawalCredentials: deposit.WithdrawalCredentials, + Amount: deposit.Amount, + Signature: deposit.Signature, + Slot: g.Slot, + }) + } + + // Note: withdrawal requests and consolidation requests are NOT processed here + // since they require full validator lookup and exit queue logic. + // For our purpose (advancing to epoch boundary), the pending deposits + // are the critical part. Withdrawal/consolidation requests affect the + // state sim which replays them separately. + } + + // Queue the builder payment (direct withdrawal for delivered payload). + // The bid was recorded in builder_pending_payments during process_execution_payload_bid. + // Now that the payload is delivered, we move the payment to builder_pending_withdrawals + // and clear the pending payment entry. + paymentIdx := slotsPerEpoch + uint64(g.Slot)%slotsPerEpoch + if paymentIdx < uint64(len(g.BuilderPendingPayments)) { + payment := g.BuilderPendingPayments[paymentIdx] + if payment != nil && payment.Withdrawal != nil && payment.Withdrawal.Amount > 0 { + g.BuilderPendingWithdrawals = append(g.BuilderPendingWithdrawals, payment.Withdrawal) + } + g.BuilderPendingPayments[paymentIdx] = &gloas.BuilderPendingPayment{} + } + + // Set execution payload availability bit. + bitfieldLen := uint64(len(g.ExecutionPayloadAvailability)) * 8 + if bitfieldLen > 0 { + idx := uint64(g.Slot) % bitfieldLen + g.ExecutionPayloadAvailability[idx/8] |= 1 << (idx % 8) + } + + // Cache the execution payload block hash. + if envelope.Payload != nil { + copy(g.LatestBlockHash[:], envelope.Payload.BlockHash[:]) + } + + return nil +} + +// processEpochInternal runs the epoch transition on the accessor without writeBack. +// Used by processSlots at each epoch boundary. +// +// Fulu: https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#modified-process_epoch +// Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_epoch +func processEpochInternal(s *stateAccessor) error { + if err := processJustificationAndFinalization(s); err != nil { + return fmt.Errorf("process_justification_and_finalization: %w", err) + } + + if err := processInactivityUpdates(s); err != nil { + return fmt.Errorf("process_inactivity_updates: %w", err) + } + + if err := processRewardsAndPenalties(s); err != nil { + return fmt.Errorf("process_rewards_and_penalties: %w", err) + } + + if err := processRegistryUpdates(s); err != nil { + return fmt.Errorf("process_registry_updates: %w", err) + } + + if err := processSlashings(s); err != nil { + return fmt.Errorf("process_slashings: %w", err) + } + + processEth1DataReset(s) + + if err := processPendingDeposits(s); err != nil { + return fmt.Errorf("process_pending_deposits: %w", err) + } + + processPendingConsolidations(s) + + // Gloas-only: process builder pending payments + if s.version >= spec.DataVersionGloas { + processBuilderPendingPayments(s) + } + + processEffectiveBalanceUpdates(s) + processSlashingsReset(s) + processRandaoMixesReset(s) + processHistoricalSummariesUpdate(s) + processParticipationFlagUpdates(s) + processSyncCommitteeUpdates(s) + processProposerLookahead(s) + + // Gloas-only: process PTC window + if s.version >= spec.DataVersionGloas { + processPtcWindow(s) + } + + return nil +} From 4fca0e67cef6cafb28a9a3ee09a99f21882a7f16 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 9 Apr 2026 04:55:01 +0200 Subject: [PATCH 87/99] fix epochState cache key --- indexer/beacon/epochcache.go | 132 ++++++++++++++++++++++++--------- indexer/beacon/epochstate.go | 83 ++++++++------------- indexer/beacon/synchronizer.go | 2 +- 3 files changed, 131 insertions(+), 86 deletions(-) diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index a43b9a2f9..ebba995c4 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -14,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/dora/indexer/beacon/statetransition" ) // epochStatsKey is the primary key for EpochStats entries in cache. @@ -30,12 +31,18 @@ func getEpochStatsKey(epoch phase0.Epoch, dependentRoot phase0.Root) epochStatsK return key } +// epochStateKey is the key for epochState entries in cache. +// Includes targetEpoch because the same dependentRoot may serve multiple epochs +// (when all slots in intermediate epochs are missed), and each epoch needs its +// own post-epoch-transition state values. +type epochStateKey = epochStatsKey + // epochCache is the cache for EpochStats (epoch status) and epochState (beacon state) structures. type epochCache struct { indexer *Indexer cacheMutex sync.RWMutex // mutex to protect statsMap & stateMap for concurrent read/write statsMap map[epochStatsKey]*EpochStats // epoch status cache by epochStatsKey - stateMap map[phase0.Root]*epochState // beacon state cache by dependentRoot + stateMap map[epochStateKey]*epochState // beacon state cache by (dependentRoot, targetEpoch) loadingChan chan bool // limits concurrent state calls by channel capacity syncMutex sync.Mutex // mutex to protect syncCache for concurrent access syncCache []phase0.ValidatorIndex // global sync committee cache for reuse if matching @@ -52,7 +59,7 @@ func newEpochCache(indexer *Indexer) *epochCache { cache := &epochCache{ indexer: indexer, statsMap: map[epochStatsKey]*EpochStats{}, - stateMap: map[phase0.Root]*epochState{}, + stateMap: map[epochStateKey]*epochState{}, loadingChan: make(chan bool, indexer.maxParallelStateCalls), votesCache: lru.NewCache[epochVotesKey, *EpochVotes](500), @@ -88,17 +95,14 @@ func (cache *epochCache) ensureEpochDependentState(epochStats *EpochStats, first return } - // get or create beacon state which the epoch status depends on (dependentRoot beacon state) - epochState := cache.stateMap[epochStats.dependentRoot] + // get or create beacon state for this (dependentRoot, epoch) combination. + // Always loads the post-state of the dependent root (last block of parent epoch). + // For Fulu+, the epoch transition is applied after loading via statetransition.PrepareEpochPreState. + stateKey := getEpochStatsKey(epochStats.epoch, epochStats.dependentRoot) + epochState := cache.stateMap[stateKey] if epochState == nil && !epochStats.ready { - stateRoot := epochStats.dependentRoot - chainState := cache.indexer.consensusPool.GetChainState() - if chainState.IsFuluEnabled(epochStats.epoch) { - stateRoot = firstBlockRoot - } - - epochState = newEpochState(stateRoot) - cache.stateMap[epochStats.dependentRoot] = epochState + epochState = newEpochState(epochStats.dependentRoot, epochStats.epoch) + cache.stateMap[stateKey] = epochState cache.indexer.logger.Infof("added epoch state request for epoch %v (%v) to queue", epochStats.epoch, epochStats.dependentRoot.String()) } @@ -123,13 +127,27 @@ func (cache *epochCache) getEpochStats(epoch phase0.Epoch, dependentRoot phase0. } // getPendingEpochStats gets all EpochStats with unloaded epochStates. +// Skips stats where another epochState with the same dependentRoot is already loading, +// since those will be processed via the dedup path in loadEpochStats after the +// first load completes. func (cache *epochCache) getPendingEpochStats() []*EpochStats { cache.cacheMutex.Lock() defer cache.cacheMutex.Unlock() + // Collect roots that are currently being loaded. + loadingRoots := make(map[phase0.Root]bool) + for _, state := range cache.stateMap { + if state.loadingStatus == 1 { + loadingRoots[state.slotRoot] = true + } + } + pendingStats := make([]*EpochStats, 0) for _, stats := range cache.statsMap { if stats.dependentState != nil && stats.dependentState.loadingStatus == 0 { + if loadingRoots[stats.dependentState.slotRoot] { + continue // another epochState with same root is already loading + } pendingStats = append(pendingStats, stats) } } @@ -221,19 +239,9 @@ func (cache *epochCache) removeEpochStats(epochStats *EpochStats) { delete(cache.statsMap, statsKey) if epochStats.dependentState != nil { - foundOtherStats := false - for _, stats := range cache.statsMap { - if bytes.Equal(stats.dependentRoot[:], epochStats.dependentRoot[:]) { - foundOtherStats = true - break - } - } - - if !foundOtherStats { - // no other epoch status depends on this beacon state - epochStats.dependentState.dispose() - delete(cache.stateMap, epochStats.dependentRoot) - } + stateKey := getEpochStatsKey(epochStats.epoch, epochStats.dependentRoot) + epochStats.dependentState.dispose() + delete(cache.stateMap, stateKey) } } @@ -487,17 +495,73 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { } } - dependentStats := []*EpochStats{} - cache.cacheMutex.Lock() - for _, stats := range cache.statsMap { - if stats.dependentState == epochStats.dependentState { - dependentStats = append(dependentStats, stats) + // Process the triggering EpochStats + go epochStats.processState(cache.indexer, validatorSet, loadDuration) + + // Find other pending epochStates with the same dependentRoot (different target epochs). + // These share the same raw state but need their own epoch transition applied. + // We advance the state sequentially through each epoch in order. + if state != nil { + type pendingEntry struct { + epochState *epochState + stats *EpochStats } - } - cache.cacheMutex.Unlock() - for _, stats := range dependentStats { - go stats.processState(cache.indexer, validatorSet, loadDuration) + var pendingOthers []pendingEntry + cache.cacheMutex.RLock() + for _, stats := range cache.statsMap { + if stats == epochStats { + continue + } + if stats.dependentState == nil || stats.dependentState.loadingStatus != 0 { + continue + } + if stats.dependentState.slotRoot != epochStats.dependentState.slotRoot { + continue + } + pendingOthers = append(pendingOthers, pendingEntry{ + epochState: stats.dependentState, + stats: stats, + }) + } + cache.cacheMutex.RUnlock() + + if len(pendingOthers) > 0 { + specs := client.indexer.consensusPool.GetChainState().GetSpecs() + + // Sort by target epoch so we advance the state forward incrementally. + sort.Slice(pendingOthers, func(i, j int) bool { + return pendingOthers[i].epochState.targetEpoch < pendingOthers[j].epochState.targetEpoch + }) + + for _, entry := range pendingOthers { + // Advance the already-loaded state to the next target epoch. + // Payload is nil since it was already applied on the first PrepareEpochPreState call. + if err := statetransition.PrepareEpochPreState(state, entry.epochState.targetEpoch, nil, specs); err != nil { + cache.indexer.logger.Errorf("error advancing state to epoch %v: %v", entry.epochState.targetEpoch, err) + continue + } + + // Extract values from the advanced state. + if err := entry.epochState.processState(state, cache, specs); err != nil { + cache.indexer.logger.Errorf("error processing state for epoch %v: %v", entry.epochState.targetEpoch, err) + continue + } + + entry.epochState.loadingStatus = 2 + + // Signal ready. + entry.epochState.readyChanMutex.Lock() + if entry.epochState.readyChan != nil { + close(entry.epochState.readyChan) + entry.epochState.readyChan = nil + } + entry.epochState.readyChanMutex.Unlock() + + // Trigger EpochStats processing. + go entry.stats.processState(cache.indexer, validatorSet, loadDuration) + } + } } return true diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 1f7d8e6b2..c718054ae 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -11,12 +11,14 @@ import ( "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/dora/indexer/beacon/statetransition" ) // epochState represents a beacon state which a epoch status depends on. type epochState struct { - slotRoot phase0.Root - stateRoot phase0.Root + slotRoot phase0.Root + stateRoot phase0.Root + targetEpoch phase0.Epoch // the epoch this state is being prepared for loadingCancel context.CancelFunc loadingStatus uint8 @@ -40,9 +42,10 @@ type epochState struct { } // newEpochState creates a new epochState instance with the root of the state to be loaded. -func newEpochState(slotRoot phase0.Root) *epochState { +func newEpochState(slotRoot phase0.Root, targetEpoch phase0.Epoch) *epochState { return &epochState{ - slotRoot: slotRoot, + slotRoot: slotRoot, + targetEpoch: targetEpoch, } } @@ -138,17 +141,33 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch return nil, err } - var executionPayload *gloas.SignedExecutionPayloadEnvelope - if beaconBlock != nil && beaconBlock.Version >= spec.DataVersionGloas { - if block != nil { - executionPayload = block.GetExecutionPayload(ctx) + specs := client.indexer.consensusPool.GetChainState().GetSpecs() + + // For Fulu+: apply epoch transition to advance the state from the post-block state + // of the parent epoch's last block to the pre-state of the target epoch. + // This includes payload processing (Gloas) and epoch transitions across boundaries. + // Skip for genesis (epoch 0) — the genesis state is already the correct pre-state. + if resState.Version >= spec.DataVersionFulu && s.targetEpoch > 0 { + var payloadEnvelope *gloas.ExecutionPayloadEnvelope + if resState.Version >= spec.DataVersionGloas { + var executionPayload *gloas.SignedExecutionPayloadEnvelope + if block != nil { + executionPayload = block.GetExecutionPayload(ctx) + } + if executionPayload == nil { + executionPayload, _ = LoadExecutionPayload(ctx, client, s.slotRoot) + } + if executionPayload != nil { + payloadEnvelope = executionPayload.Message + } } - if executionPayload == nil { - executionPayload, _ = LoadExecutionPayload(ctx, client, s.slotRoot) + + if err := statetransition.PrepareEpochPreState(resState, s.targetEpoch, payloadEnvelope, specs); err != nil { + return nil, fmt.Errorf("error applying epoch transition for epoch %v: %w", s.targetEpoch, err) } } - err = s.processState(resState, beaconBlock, executionPayload, cache, client.indexer.consensusPool.GetChainState().GetSpecs()) + err = s.processState(resState, cache, specs) if err != nil { return nil, err } @@ -166,23 +185,14 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch // processState processes the state and updates the epochState instance. // the function extracts and unifies all relevant information from the beacon state, so the full beacon state can be dropped from memory afterwards. -func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock *spec.VersionedSignedBeaconBlock, executionPayload *gloas.SignedExecutionPayloadEnvelope, cache *epochCache, specs *consensus.ChainSpec) error { +func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epochCache, specs *consensus.ChainSpec) error { slot, err := state.Slot() if err != nil { return fmt.Errorf("error getting slot from state %v: %v", s.slotRoot.String(), err) } s.stateSlot = slot - dependentRoot := s.slotRoot - if state.Version >= spec.DataVersionFulu { - parentRoot, err := getLatestBlockHeaderParentRoot(state) - if err != nil { - return fmt.Errorf("error getting latest block header parent root from state %v: %v", s.slotRoot.String(), err) - } - - dependentRoot = parentRoot - } validatorList, err := state.Validators() if err != nil { @@ -199,7 +209,6 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock cache.indexer.builderCache.updateBuilderSet(slot, dependentRoot, state.Gloas.Builders) } - // Extract builder balances builderBalances := make([]phase0.Gwei, len(state.Gloas.Builders)) for i, builder := range state.Gloas.Builders { builderBalances[i] = builder.Balance @@ -225,33 +234,7 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock } s.randaoMixes = randaoMixes - - if state.Version >= spec.DataVersionFulu { - if state.Version >= spec.DataVersionGloas { - isPostPayload := isGloasPostPayloadState(state, slot) - if isPostPayload && executionPayload != nil && - executionPayload.Message != nil && - executionPayload.Message.ExecutionRequests != nil && - len(executionPayload.Message.ExecutionRequests.Deposits) > 0 { - s.depositIndex = executionPayload.Message.ExecutionRequests.Deposits[0].Index - } else { - s.depositIndex = getStateDepositIndex(state) - } - } else { - blockRequests, err := beaconBlock.ExecutionRequests() - if err != nil { - return fmt.Errorf("error getting execution requests from block %v: %v", - s.slotRoot.String(), err) - } - if len(blockRequests.Deposits) > 0 { - s.depositIndex = blockRequests.Deposits[0].Index - } else { - s.depositIndex = getStateDepositIndex(state) - } - } - } else { - s.depositIndex = getStateDepositIndex(state) - } + s.depositIndex = getStateDepositIndex(state) if state.Version >= spec.DataVersionAltair { currentSyncCommittee, err := getStateCurrentSyncCommittee(state) @@ -299,8 +282,6 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, beaconBlock if err != nil { return fmt.Errorf("error getting pending consolidation indices from state %v: %v", s.slotRoot.String(), err) } - - // apply epoch transition to get remaining pending consolidations s.pendingConsolidations = pendingConsolidations } diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 76ca8f687..68a12a434 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -382,7 +382,7 @@ func (s *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry dependentRoot = phase0.Root(depRoot) } - epochState := newEpochState(dependentRoot) + epochState := newEpochState(dependentRoot, syncEpoch) t1 := time.Now() state, err := epochState.loadState(s.syncCtx, client, nil) loadDuration := time.Since(t1) From 0420b1be8a69d50aecf31750e5a49d33463f4f41 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 9 Apr 2026 05:52:22 +0200 Subject: [PATCH 88/99] simplify state simulator & fix delayed payment detection --- blockdb/types/execdata_sections_ssz.go | 243 +++++++-------- indexer/beacon/epochcache.go | 4 +- indexer/beacon/epochstate.go | 36 ++- indexer/beacon/epochstats.go | 117 ++++---- indexer/beacon/epochstats_ssz.go | 278 ++++++++++-------- indexer/beacon/finalization.go | 7 - indexer/beacon/state_sim.go | 212 ++++--------- indexer/beacon/statetransition/builder.go | 8 +- .../beacon/statetransition/statetransition.go | 28 +- 9 files changed, 439 insertions(+), 494 deletions(-) diff --git a/blockdb/types/execdata_sections_ssz.go b/blockdb/types/execdata_sections_ssz.go index b679cac9e..0517875ae 100644 --- a/blockdb/types/execdata_sections_ssz.go +++ b/blockdb/types/execdata_sections_ssz.go @@ -1,6 +1,6 @@ // Code generated by dynamic-ssz. DO NOT EDIT. // Hash: d3acce19a9e9c0afc0038f9dfd949d804d4b82beafef97cc4d618d03c202d4af -// Version: v1.2.1 (https://github.com/pk910/dynamic-ssz) +// Version: v1.2.2 (https://github.com/pk910/dynamic-ssz) package types import ( @@ -21,43 +21,41 @@ func (t *ReceiptMetaData) MarshalSSZTo(buf []byte) (dst []byte, err error) { if t == nil { t = new(ReceiptMetaData) } - { // Field #0 'Version' + { // Static Field #0 'Version' dst = binary.LittleEndian.AppendUint16(dst, t.Version) } - { // Field #1 'Status' + { // Static Field #1 'Status' dst = append(dst, byte(t.Status)) } - { // Field #2 'TxType' + { // Static Field #2 'TxType' dst = append(dst, byte(t.TxType)) } - { // Field #3 'CumulativeGasUsed' + { // Static Field #3 'CumulativeGasUsed' dst = binary.LittleEndian.AppendUint64(dst, t.CumulativeGasUsed) } - { // Field #4 'GasUsed' + { // Static Field #4 'GasUsed' dst = binary.LittleEndian.AppendUint64(dst, t.GasUsed) } - { // Field #5 'EffectiveGasPrice' + { // Static Field #5 'EffectiveGasPrice' t := &t.EffectiveGasPrice - for i := range 4 { - dst = binary.LittleEndian.AppendUint64(dst, t[i]) - } + dst = sszutils.MarshalUint64Slice(dst, t[:4]) } - { // Field #6 'BlobGasUsed' + { // Static Field #6 'BlobGasUsed' dst = binary.LittleEndian.AppendUint64(dst, t.BlobGasUsed) } - { // Field #7 'LogsBloom' + { // Static Field #7 'LogsBloom' dst = append(dst, t.LogsBloom[:256]...) } - { // Field #8 'From' + { // Static Field #8 'From' dst = append(dst, t.From[:20]...) } - { // Field #9 'To' + { // Static Field #9 'To' dst = append(dst, t.To[:20]...) } - { // Field #10 'ContractAddress' + { // Static Field #10 'ContractAddress' dst = append(dst, t.ContractAddress[:20]...) } - { // Field #11 'HasContractAddr' + { // Static Field #11 'HasContractAddr' dst = sszutils.MarshalBool(dst, t.HasContractAddr) } return dst, nil @@ -66,7 +64,7 @@ func (t *ReceiptMetaData) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *ReceiptMetaData) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) if buflen < 377 { - return sszutils.ErrUnexpectedEOF + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 377) } { // Field #0 'Version' (static) buf := buf[0:2] @@ -91,10 +89,7 @@ func (t *ReceiptMetaData) UnmarshalSSZ(buf []byte) (err error) { { // Field #5 'EffectiveGasPrice' (static) buf := buf[20:52] val1 := t.EffectiveGasPrice - for i := range 4 { - buf := buf[8*i : 8*(i+1)] - val1[i] = binary.LittleEndian.Uint64(buf) - } + sszutils.UnmarshalUint64Slice(val1[:4], buf) t.EffectiveGasPrice = val1 } { // Field #6 'BlobGasUsed' (static) @@ -120,7 +115,7 @@ func (t *ReceiptMetaData) UnmarshalSSZ(buf []byte) (err error) { { // Field #11 'HasContractAddr' (static) buf := buf[376:377] if buf[0] != 1 && buf[0] != 0 { - return sszutils.ErrInvalidValueRange + return sszutils.NewSszError(sszutils.ErrInvalidValueRange, "bool value must be 0 or 1") } t.HasContractAddr = buf[0] == 1 } @@ -164,7 +159,7 @@ func (t *ReceiptMetaData) HashTreeRootWith(hh sszutils.HashWalker) error { { // Field #5 'EffectiveGasPrice' t := &t.EffectiveGasPrice if root, err := t.HashTreeRoot(); err != nil { - return err + return sszutils.ErrorWithPath(err, "EffectiveGasPrice") } else { hh.AppendBytes32(root[:]) } @@ -199,10 +194,10 @@ func (t *BlockReceiptMeta) MarshalSSZTo(buf []byte) (dst []byte, err error) { if t == nil { t = new(BlockReceiptMeta) } - { // Field #0 'Version' + { // Static Field #0 'Version' dst = binary.LittleEndian.AppendUint16(dst, t.Version) } - { // Field #1 'BlobGasPrice' + { // Static Field #1 'BlobGasPrice' dst = binary.LittleEndian.AppendUint64(dst, t.BlobGasPrice) } return dst, nil @@ -211,7 +206,7 @@ func (t *BlockReceiptMeta) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *BlockReceiptMeta) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) if buflen < 10 { - return sszutils.ErrUnexpectedEOF + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 10) } { // Field #0 'Version' (static) buf := buf[0:2] @@ -258,64 +253,56 @@ func (t *StateChangeAccount) MarshalSSZ() ([]byte, error) { } func (t *StateChangeAccount) MarshalSSZTo(buf []byte) (dst []byte, err error) { dst = buf + zeroBytes := sszutils.ZeroBytes() if t == nil { t = new(StateChangeAccount) } dstlen := len(dst) - { // Field #0 'Address' + { // Static Field #0 'Address' dst = append(dst, t.Address[:20]...) } - { // Field #1 'Flags' + { // Static Field #1 'Flags' dst = append(dst, byte(t.Flags)) } - { // Field #2 'PreBalance' + { // Static Field #2 'PreBalance' t := &t.PreBalance - for i := range 4 { - dst = binary.LittleEndian.AppendUint64(dst, t[i]) - } + dst = sszutils.MarshalUint64Slice(dst, t[:4]) } - { // Field #3 'PostBalance' + { // Static Field #3 'PostBalance' t := &t.PostBalance - for i := range 4 { - dst = binary.LittleEndian.AppendUint64(dst, t[i]) - } + dst = sszutils.MarshalUint64Slice(dst, t[:4]) } - { // Field #4 'PreNonce' + { // Static Field #4 'PreNonce' dst = binary.LittleEndian.AppendUint64(dst, t.PreNonce) } - { // Field #5 'PostNonce' + { // Static Field #5 'PostNonce' dst = binary.LittleEndian.AppendUint64(dst, t.PostNonce) } - // Offset #6 'PreCode' - offset6 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #7 'PostCode' - offset7 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #8 'Slots' - offset8 := len(dst) - dst = append(dst, 0, 0, 0, 0) + // Offset Field #6 'PreCode' + // Offset Field #7 'PostCode' + // Offset Field #8 'Slots' + dst = append(dst, zeroBytes[:12]...) { // Dynamic Field #6 'PreCode' - binary.LittleEndian.PutUint32(dst[offset6:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+101:], uint32(len(dst)-dstlen)) dst = append(dst, t.PreCode[:]...) } { // Dynamic Field #7 'PostCode' - binary.LittleEndian.PutUint32(dst[offset7:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+105:], uint32(len(dst)-dstlen)) dst = append(dst, t.PostCode[:]...) } { // Dynamic Field #8 'Slots' - binary.LittleEndian.PutUint32(dst[offset8:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+109:], uint32(len(dst)-dstlen)) t := t.Slots vlen := len(t) - for i := range vlen { - t := &t[i] - { // Field #0 'Slot' + for idx1 := range vlen { + t := &t[idx1] + { // Static Field #0 'Slot' dst = append(dst, t.Slot[:32]...) } - { // Field #1 'PreValue' + { // Static Field #1 'PreValue' dst = append(dst, t.PreValue[:32]...) } - { // Field #2 'PostValue' + { // Static Field #2 'PostValue' dst = append(dst, t.PostValue[:32]...) } } @@ -326,7 +313,7 @@ func (t *StateChangeAccount) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *StateChangeAccount) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) if buflen < 113 { - return sszutils.ErrUnexpectedEOF + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 113) } { // Field #0 'Address' (static) buf := buf[0:20] @@ -339,19 +326,13 @@ func (t *StateChangeAccount) UnmarshalSSZ(buf []byte) (err error) { { // Field #2 'PreBalance' (static) buf := buf[21:53] val1 := t.PreBalance - for i := range 4 { - buf := buf[8*i : 8*(i+1)] - val1[i] = binary.LittleEndian.Uint64(buf) - } + sszutils.UnmarshalUint64Slice(val1[:4], buf) t.PreBalance = val1 } { // Field #3 'PostBalance' (static) buf := buf[53:85] val2 := t.PostBalance - for i := range 4 { - buf := buf[8*i : 8*(i+1)] - val2[i] = binary.LittleEndian.Uint64(buf) - } + sszutils.UnmarshalUint64Slice(val2[:4], buf) t.PostBalance = val2 } { // Field #4 'PreNonce' (static) @@ -365,17 +346,17 @@ func (t *StateChangeAccount) UnmarshalSSZ(buf []byte) (err error) { // Field #6 'PreCode' (offset) offset6 := int(binary.LittleEndian.Uint32(buf[101:105])) if offset6 != 113 { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "first offset does not match (offset: %d, static len: %d)", offset6, 113), "PreCode:o") } // Field #7 'PostCode' (offset) offset7 := int(binary.LittleEndian.Uint32(buf[105:109])) if offset7 < offset6 || offset7 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset7, offset6, buflen), "PostCode:o") } // Field #8 'Slots' (offset) offset8 := int(binary.LittleEndian.Uint32(buf[109:113])) if offset8 < offset7 || offset8 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset8, offset7, buflen), "Slots:o") } { // Field #6 'PreCode' (dynamic) buf := buf[offset6:offset7] @@ -392,15 +373,15 @@ func (t *StateChangeAccount) UnmarshalSSZ(buf []byte) (err error) { val3 := t.Slots itemCount := len(buf) / 96 if len(buf)%96 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 96), "Slots") } val3 = sszutils.ExpandSlice(val3, itemCount) - for i := range itemCount { - val4 := val3[i] - buf := buf[96*i : 96*(i+1)] + for idx1 := range itemCount { + val4 := val3[idx1] + buf := buf[96*idx1 : 96*(idx1+1)] buflen := len(buf) if buflen < 96 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPathf(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 96), "Slots[%d]", idx1) } { // Field #0 'Slot' (static) buf := buf[0:32] @@ -414,7 +395,7 @@ func (t *StateChangeAccount) UnmarshalSSZ(buf []byte) (err error) { buf := buf[64:96] copy(val4.PostValue[:], buf) } - val3[i] = val4 + val3[idx1] = val4 } t.Slots = val3 } @@ -471,7 +452,7 @@ func (t *StateChangeAccount) HashTreeRootWith(hh sszutils.HashWalker) error { { // Field #2 'PreBalance' t := &t.PreBalance if root, err := t.HashTreeRoot(); err != nil { - return err + return sszutils.ErrorWithPath(err, "PreBalance") } else { hh.AppendBytes32(root[:]) } @@ -479,7 +460,7 @@ func (t *StateChangeAccount) HashTreeRootWith(hh sszutils.HashWalker) error { { // Field #3 'PostBalance' t := &t.PostBalance if root, err := t.HashTreeRoot(); err != nil { - return err + return sszutils.ErrorWithPath(err, "PostBalance") } else { hh.AppendBytes32(root[:]) } @@ -504,8 +485,8 @@ func (t *StateChangeAccount) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.Slots idx := hh.Index() vlen := uint64(len(t)) - for i := range int(vlen) { - t := &t[i] + for idx1 := range int(vlen) { + t := &t[idx1] idx := hh.Index() { // Field #0 'Slot' hh.PutBytes(t.Slot[:32]) @@ -529,56 +510,50 @@ func (t *FlatCallFrame) MarshalSSZ() ([]byte, error) { } func (t *FlatCallFrame) MarshalSSZTo(buf []byte) (dst []byte, err error) { dst = buf + zeroBytes := sszutils.ZeroBytes() if t == nil { t = new(FlatCallFrame) } dstlen := len(dst) - { // Field #0 'Depth' + { // Static Field #0 'Depth' dst = binary.LittleEndian.AppendUint16(dst, t.Depth) } - { // Field #1 'Type' + { // Static Field #1 'Type' dst = append(dst, byte(t.Type)) } - { // Field #2 'From' + { // Static Field #2 'From' dst = append(dst, t.From[:20]...) } - { // Field #3 'To' + { // Static Field #3 'To' dst = append(dst, t.To[:20]...) } - { // Field #4 'Value' + { // Static Field #4 'Value' t := &t.Value - for i := range 4 { - dst = binary.LittleEndian.AppendUint64(dst, t[i]) - } + dst = sszutils.MarshalUint64Slice(dst, t[:4]) } - { // Field #5 'Gas' + { // Static Field #5 'Gas' dst = binary.LittleEndian.AppendUint64(dst, t.Gas) } - { // Field #6 'GasUsed' + { // Static Field #6 'GasUsed' dst = binary.LittleEndian.AppendUint64(dst, t.GasUsed) } - { // Field #7 'Status' + { // Static Field #7 'Status' dst = append(dst, byte(t.Status)) } - // Offset #8 'Input' - offset8 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #9 'Output' - offset9 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #10 'Error' - offset10 := len(dst) - dst = append(dst, 0, 0, 0, 0) + // Offset Field #8 'Input' + // Offset Field #9 'Output' + // Offset Field #10 'Error' + dst = append(dst, zeroBytes[:12]...) { // Dynamic Field #8 'Input' - binary.LittleEndian.PutUint32(dst[offset8:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+92:], uint32(len(dst)-dstlen)) dst = append(dst, t.Input[:]...) } { // Dynamic Field #9 'Output' - binary.LittleEndian.PutUint32(dst[offset9:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+96:], uint32(len(dst)-dstlen)) dst = append(dst, t.Output[:]...) } { // Dynamic Field #10 'Error' - binary.LittleEndian.PutUint32(dst[offset10:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+100:], uint32(len(dst)-dstlen)) dst = append(dst, t.Error[:]...) } return dst, nil @@ -587,7 +562,7 @@ func (t *FlatCallFrame) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *FlatCallFrame) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) if buflen < 104 { - return sszutils.ErrUnexpectedEOF + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 104) } { // Field #0 'Depth' (static) buf := buf[0:2] @@ -608,10 +583,7 @@ func (t *FlatCallFrame) UnmarshalSSZ(buf []byte) (err error) { { // Field #4 'Value' (static) buf := buf[43:75] val1 := t.Value - for i := range 4 { - buf := buf[8*i : 8*(i+1)] - val1[i] = binary.LittleEndian.Uint64(buf) - } + sszutils.UnmarshalUint64Slice(val1[:4], buf) t.Value = val1 } { // Field #5 'Gas' (static) @@ -629,17 +601,17 @@ func (t *FlatCallFrame) UnmarshalSSZ(buf []byte) (err error) { // Field #8 'Input' (offset) offset8 := int(binary.LittleEndian.Uint32(buf[92:96])) if offset8 != 104 { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "first offset does not match (offset: %d, static len: %d)", offset8, 104), "Input:o") } // Field #9 'Output' (offset) offset9 := int(binary.LittleEndian.Uint32(buf[96:100])) if offset9 < offset8 || offset9 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset9, offset8, buflen), "Output:o") } // Field #10 'Error' (offset) offset10 := int(binary.LittleEndian.Uint32(buf[100:104])) if offset10 < offset9 || offset10 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset10, offset9, buflen), "Error:o") } { // Field #8 'Input' (dynamic) buf := buf[offset8:offset9] @@ -716,7 +688,7 @@ func (t *FlatCallFrame) HashTreeRootWith(hh sszutils.HashWalker) error { { // Field #4 'Value' t := &t.Value if root, err := t.HashTreeRoot(); err != nil { - return err + return sszutils.ErrorWithPath(err, "Value") } else { hh.AppendBytes32(root[:]) } @@ -758,38 +730,35 @@ func (t *EventData) MarshalSSZTo(buf []byte) (dst []byte, err error) { t = new(EventData) } dstlen := len(dst) - { // Field #0 'EventIndex' + { // Static Field #0 'EventIndex' dst = binary.LittleEndian.AppendUint32(dst, t.EventIndex) } - { // Field #1 'Source' + { // Static Field #1 'Source' dst = append(dst, t.Source[:20]...) } - // Offset #2 'Topics' - offset2 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #3 'Data' - offset3 := len(dst) - dst = append(dst, 0, 0, 0, 0) + // Offset Field #2 'Topics' + // Offset Field #3 'Data' + dst = append(dst, 0, 0, 0, 0, 0, 0, 0, 0) { // Dynamic Field #2 'Topics' - binary.LittleEndian.PutUint32(dst[offset2:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+24:], uint32(len(dst)-dstlen)) t := t.Topics vlen := len(t) if vlen > 5 { - return nil, sszutils.ErrListTooBig + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 5), "Topics") } - for i := range vlen { - vlen := len(t[i]) + for idx1 := range vlen { + vlen := len(t[idx1]) if vlen > 32 { - return nil, sszutils.ErrVectorLength + return nil, sszutils.ErrorWithPathf(sszutils.NewSszErrorf(sszutils.ErrVectorLength, "vector length %d exceeds limit %d", vlen, 32), "Topics[%d]", idx1) } - dst = append(dst, t[i][:vlen]...) + dst = append(dst, t[idx1][:vlen]...) if vlen < 32 { dst = sszutils.AppendZeroPadding(dst, (32-vlen)*1) } } } { // Dynamic Field #3 'Data' - binary.LittleEndian.PutUint32(dst[offset3:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+28:], uint32(len(dst)-dstlen)) dst = append(dst, t.Data[:]...) } return dst, nil @@ -798,7 +767,7 @@ func (t *EventData) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *EventData) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) if buflen < 32 { - return sszutils.ErrUnexpectedEOF + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 32) } { // Field #0 'EventIndex' (static) buf := buf[0:4] @@ -811,25 +780,25 @@ func (t *EventData) UnmarshalSSZ(buf []byte) (err error) { // Field #2 'Topics' (offset) offset2 := int(binary.LittleEndian.Uint32(buf[24:28])) if offset2 != 32 { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "first offset does not match (offset: %d, static len: %d)", offset2, 32), "Topics:o") } // Field #3 'Data' (offset) offset3 := int(binary.LittleEndian.Uint32(buf[28:32])) if offset3 < offset2 || offset3 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset3, offset2, buflen), "Data:o") } { // Field #2 'Topics' (dynamic) buf := buf[offset2:offset3] val1 := t.Topics itemCount := len(buf) / 32 if len(buf)%32 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 32), "Topics") } val1 = sszutils.ExpandSlice(val1, itemCount) - for i := range itemCount { - buf := buf[32*i : 32*(i+1)] - val1[i] = sszutils.ExpandSlice(val1[i], 32) - copy(val1[i][:], buf) + for idx1 := range itemCount { + buf := buf[32*idx1 : 32*(idx1+1)] + val1[idx1] = sszutils.ExpandSlice(val1[idx1], 32) + copy(val1[idx1][:], buf) } t.Topics = val1 } @@ -884,15 +853,15 @@ func (t *EventData) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.Topics vlen := uint64(len(t)) if vlen > 5 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 5), "Topics") } idx := hh.Index() - for i := range int(vlen) { - vlen := len(t[i]) + for idx1 := range int(vlen) { + vlen := len(t[idx1]) if vlen > 32 { - return sszutils.ErrVectorLength + return sszutils.ErrorWithPathf(sszutils.NewSszErrorf(sszutils.ErrVectorLength, "vector length %d exceeds limit %d", vlen, 32), "Topics[%d]", idx1) } - val := t[i][:] + val := t[idx1][:] if vlen < 32 { val = sszutils.AppendZeroPadding(val, (32-vlen)*1) } diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index ebba995c4..6d5030ecc 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -537,10 +537,12 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { for _, entry := range pendingOthers { // Advance the already-loaded state to the next target epoch. // Payload is nil since it was already applied on the first PrepareEpochPreState call. - if err := statetransition.PrepareEpochPreState(state, entry.epochState.targetEpoch, nil, specs); err != nil { + var transitionInfo statetransition.TransitionInfo + if err := statetransition.PrepareEpochPreState(state, entry.epochState.targetEpoch, nil, specs, &transitionInfo); err != nil { cache.indexer.logger.Errorf("error advancing state to epoch %v: %v", entry.epochState.targetEpoch, err) continue } + entry.epochState.delayedBuilderPaymentCount = transitionInfo.DelayedBuilderPayments // Extract values from the advanced state. if err := entry.epochState.processState(state, cache, specs); err != nil { diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index c718054ae..40f9f0904 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -27,18 +27,20 @@ type epochState struct { readyChan chan bool highPriority bool - stateSlot phase0.Slot - validatorBalances []phase0.Gwei - builderBalances []phase0.Gwei - randaoMixes []phase0.Root - depositIndex uint64 - syncCommittee []phase0.ValidatorIndex - depositBalanceToConsume phase0.Gwei - pendingDeposits []*electra.PendingDeposit - pendingPartialWithdrawals []*electra.PendingPartialWithdrawal - builderPendingWithdrawals []*gloas.BuilderPendingWithdrawal - pendingConsolidations []*electra.PendingConsolidation - proposerLookahead []phase0.ValidatorIndex + stateSlot phase0.Slot + sourceBlockSlot phase0.Slot // slot of the source block (before epoch transition) + validatorBalances []phase0.Gwei + builderBalances []phase0.Gwei + randaoMixes []phase0.Root + depositIndex uint64 + syncCommittee []phase0.ValidatorIndex + depositBalanceToConsume phase0.Gwei + pendingDeposits []*electra.PendingDeposit + pendingPartialWithdrawals []*electra.PendingPartialWithdrawal + builderPendingWithdrawals []*gloas.BuilderPendingWithdrawal + delayedBuilderPaymentCount uint32 // number of delayed payments at the tail of builderPendingWithdrawals + pendingConsolidations []*electra.PendingConsolidation + proposerLookahead []phase0.ValidatorIndex } // newEpochState creates a new epochState instance with the root of the state to be loaded. @@ -143,6 +145,12 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch specs := client.indexer.consensusPool.GetChainState().GetSpecs() + // Save the source block slot before epoch transition (needed for ref slot of + // direct builder payments from the parent epoch's last block). + if sourceSlot, err := resState.Slot(); err == nil { + s.sourceBlockSlot = sourceSlot + } + // For Fulu+: apply epoch transition to advance the state from the post-block state // of the parent epoch's last block to the pre-state of the target epoch. // This includes payload processing (Gloas) and epoch transitions across boundaries. @@ -162,9 +170,11 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } } - if err := statetransition.PrepareEpochPreState(resState, s.targetEpoch, payloadEnvelope, specs); err != nil { + var transitionInfo statetransition.TransitionInfo + if err := statetransition.PrepareEpochPreState(resState, s.targetEpoch, payloadEnvelope, specs, &transitionInfo); err != nil { return nil, fmt.Errorf("error applying epoch transition for epoch %v: %w", s.targetEpoch, err) } + s.delayedBuilderPaymentCount = transitionInfo.DelayedBuilderPayments } err = s.processState(resState, cache, specs) diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index 56a981221..d88650257 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -45,23 +45,25 @@ type EpochStats struct { // EpochStatsValues holds the values for the epoch-specific information. type EpochStatsValues struct { - RandaoMix phase0.Hash32 - NextRandaoMix phase0.Hash32 - ActiveIndices []phase0.ValidatorIndex - EffectiveBalances []uint32 // effective balance in full ETH of last epoch for pre-fulu stats, effective balance in full ETH of current epoch for fulu+ stats - ProposerDuties []phase0.ValidatorIndex - AttesterDuties [][][]duties.ActiveIndiceIndex - SyncCommitteeDuties []phase0.ValidatorIndex - PtcDuties [][]duties.ActiveIndiceIndex // [slot_index][ptc_member_index] - PTC duties for Gloas+ epochs - ActiveValidators uint64 - TotalBalance phase0.Gwei - ActiveBalance phase0.Gwei - EffectiveBalance phase0.Gwei - FirstDepositIndex uint64 - PendingWithdrawals []electra.PendingPartialWithdrawal - BuilderPendingWithdrawals []gloas.BuilderPendingWithdrawal - PendingConsolidations []electra.PendingConsolidation - ConsolidatingBalance phase0.Gwei + RandaoMix phase0.Hash32 + NextRandaoMix phase0.Hash32 + ActiveIndices []phase0.ValidatorIndex + EffectiveBalances []uint32 // effective balance in full ETH of last epoch for pre-fulu stats, effective balance in full ETH of current epoch for fulu+ stats + ProposerDuties []phase0.ValidatorIndex + AttesterDuties [][][]duties.ActiveIndiceIndex + SyncCommitteeDuties []phase0.ValidatorIndex + PtcDuties [][]duties.ActiveIndiceIndex // [slot_index][ptc_member_index] - PTC duties for Gloas+ epochs + ActiveValidators uint64 + TotalBalance phase0.Gwei + ActiveBalance phase0.Gwei + EffectiveBalance phase0.Gwei + FirstDepositIndex uint64 + PendingWithdrawals []electra.PendingPartialWithdrawal + BuilderPendingWithdrawals []gloas.BuilderPendingWithdrawal + DelayedBuilderPaymentCount uint32 // number of delayed payments at the tail of BuilderPendingWithdrawals + SourceBlockSlot phase0.Slot // slot of the source block (last block of parent epoch, before epoch transition) + PendingConsolidations []electra.PendingConsolidation + ConsolidatingBalance phase0.Gwei } // EpochStatsPacked holds the packed values for the epoch-specific information. @@ -69,17 +71,20 @@ type EpochStatsValues struct { // generate ssz: (this is really ugly, needs path patching and post-fixing to work) // sszgen --suffix ssz --path . --include $GOPATH/pkg/mod/github.com/attestantio/go-eth2-client\@v0.26.0/spec/phase0,$GOPATH/pkg/mod/github.com/attestantio/go-eth2-client\@v0.26.0/spec/electra --objs EpochStatsPacked type EpochStatsPacked struct { - ActiveValidators []EpochStatsPackedValidator `ssz-max:"10000000"` - ProposerDuties []phase0.ValidatorIndex `ssz-max:"100"` - SyncCommitteeDuties []phase0.ValidatorIndex `ssz-max:"10000"` - RandaoMix phase0.Hash32 `ssz-size:"32"` - NextRandaoMix phase0.Hash32 `ssz-size:"32"` - TotalBalance phase0.Gwei - ActiveBalance phase0.Gwei - FirstDepositIndex uint64 - PendingWithdrawals []electra.PendingPartialWithdrawal `ssz-max:"10000000"` - PendingConsolidations []electra.PendingConsolidation `ssz-max:"10000000"` - ConsolidatingBalance phase0.Gwei + ActiveValidators []EpochStatsPackedValidator `ssz-max:"10000000"` + ProposerDuties []phase0.ValidatorIndex `ssz-max:"100"` + SyncCommitteeDuties []phase0.ValidatorIndex `ssz-max:"10000"` + RandaoMix phase0.Hash32 `ssz-size:"32"` + NextRandaoMix phase0.Hash32 `ssz-size:"32"` + TotalBalance phase0.Gwei + ActiveBalance phase0.Gwei + FirstDepositIndex uint64 + PendingWithdrawals []electra.PendingPartialWithdrawal `ssz-max:"10000000"` + BuilderPendingWithdrawals []gloas.BuilderPendingWithdrawal `ssz-max:"10000000"` + DelayedBuilderPaymentCount uint32 + SourceBlockSlot phase0.Slot + PendingConsolidations []electra.PendingConsolidation `ssz-max:"10000000"` + ConsolidatingBalance phase0.Gwei } // EpochStatsPackedValidator holds the packed values for an active validator. @@ -167,17 +172,20 @@ func (es *EpochStats) buildPackedSSZ() ([]byte, error) { } packedValues := &EpochStatsPacked{ - ActiveValidators: make([]EpochStatsPackedValidator, es.values.ActiveValidators), - ProposerDuties: es.values.ProposerDuties, - SyncCommitteeDuties: es.values.SyncCommitteeDuties, - RandaoMix: es.values.RandaoMix, - NextRandaoMix: es.values.NextRandaoMix, - TotalBalance: es.values.TotalBalance, - ActiveBalance: es.values.ActiveBalance, - FirstDepositIndex: es.values.FirstDepositIndex, - PendingWithdrawals: es.values.PendingWithdrawals, - PendingConsolidations: es.values.PendingConsolidations, - ConsolidatingBalance: es.values.ConsolidatingBalance, + ActiveValidators: make([]EpochStatsPackedValidator, es.values.ActiveValidators), + ProposerDuties: es.values.ProposerDuties, + SyncCommitteeDuties: es.values.SyncCommitteeDuties, + RandaoMix: es.values.RandaoMix, + NextRandaoMix: es.values.NextRandaoMix, + TotalBalance: es.values.TotalBalance, + ActiveBalance: es.values.ActiveBalance, + FirstDepositIndex: es.values.FirstDepositIndex, + PendingWithdrawals: es.values.PendingWithdrawals, + PendingConsolidations: es.values.PendingConsolidations, + BuilderPendingWithdrawals: es.values.BuilderPendingWithdrawals, + DelayedBuilderPaymentCount: es.values.DelayedBuilderPaymentCount, + SourceBlockSlot: es.values.SourceBlockSlot, + ConsolidatingBalance: es.values.ConsolidatingBalance, } lastValidatorIndex := phase0.ValidatorIndex(0) @@ -218,19 +226,22 @@ func (es *EpochStats) parsePackedSSZ(chainState *consensus.ChainState, ssz []byt } values := &EpochStatsValues{ - RandaoMix: packedValues.RandaoMix, - NextRandaoMix: packedValues.NextRandaoMix, - ActiveIndices: make([]phase0.ValidatorIndex, len(packedValues.ActiveValidators)), - EffectiveBalances: make([]uint32, len(packedValues.ActiveValidators)), - ProposerDuties: packedValues.ProposerDuties, - SyncCommitteeDuties: packedValues.SyncCommitteeDuties, - TotalBalance: packedValues.TotalBalance, - ActiveBalance: packedValues.ActiveBalance, - EffectiveBalance: 0, - FirstDepositIndex: packedValues.FirstDepositIndex, - PendingWithdrawals: packedValues.PendingWithdrawals, - PendingConsolidations: packedValues.PendingConsolidations, - ConsolidatingBalance: packedValues.ConsolidatingBalance, + RandaoMix: packedValues.RandaoMix, + NextRandaoMix: packedValues.NextRandaoMix, + ActiveIndices: make([]phase0.ValidatorIndex, len(packedValues.ActiveValidators)), + EffectiveBalances: make([]uint32, len(packedValues.ActiveValidators)), + ProposerDuties: packedValues.ProposerDuties, + SyncCommitteeDuties: packedValues.SyncCommitteeDuties, + TotalBalance: packedValues.TotalBalance, + ActiveBalance: packedValues.ActiveBalance, + EffectiveBalance: 0, + FirstDepositIndex: packedValues.FirstDepositIndex, + PendingWithdrawals: packedValues.PendingWithdrawals, + PendingConsolidations: packedValues.PendingConsolidations, + BuilderPendingWithdrawals: packedValues.BuilderPendingWithdrawals, + DelayedBuilderPaymentCount: packedValues.DelayedBuilderPaymentCount, + SourceBlockSlot: packedValues.SourceBlockSlot, + ConsolidatingBalance: packedValues.ConsolidatingBalance, } lastValidatorIndex := phase0.ValidatorIndex(0) @@ -386,6 +397,8 @@ func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Vali values.BuilderPendingWithdrawals[i] = *bpw } } + values.DelayedBuilderPaymentCount = dependentState.delayedBuilderPaymentCount + values.SourceBlockSlot = dependentState.sourceBlockSlot for i, pendingConsolidation := range dependentState.pendingConsolidations { srcIndicee := pendingConsolidation.SourceIndex diff --git a/indexer/beacon/epochstats_ssz.go b/indexer/beacon/epochstats_ssz.go index a28e4b984..93d393e0a 100644 --- a/indexer/beacon/epochstats_ssz.go +++ b/indexer/beacon/epochstats_ssz.go @@ -1,6 +1,6 @@ // Code generated by dynamic-ssz. DO NOT EDIT. -// Hash: a6f4f377206be4aef1242896821b07617571dd21a37f76322c8872bc7c3cf3f3 -// Version: v1.2.1 (https://github.com/pk910/dynamic-ssz) +// Hash: 23cc3d81625bca9f5609a71ab4097c8e2e896f2fb49ddcc1567f532b4e30ec00 +// Version: v1.2.2 (https://github.com/pk910/dynamic-ssz) package beacon import ( @@ -19,107 +19,115 @@ func (t *EpochStatsPacked) MarshalSSZ() ([]byte, error) { } func (t *EpochStatsPacked) MarshalSSZTo(buf []byte) (dst []byte, err error) { dst = buf + zeroBytes := sszutils.ZeroBytes() if t == nil { t = new(EpochStatsPacked) } dstlen := len(dst) - // Offset #0 'ActiveValidators' - offset0 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #1 'ProposerDuties' - offset1 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #2 'SyncCommitteeDuties' - offset2 := len(dst) - dst = append(dst, 0, 0, 0, 0) - { // Field #3 'RandaoMix' + // Offset Field #0 'ActiveValidators' + // Offset Field #1 'ProposerDuties' + // Offset Field #2 'SyncCommitteeDuties' + dst = append(dst, zeroBytes[:12]...) + { // Static Field #3 'RandaoMix' dst = append(dst, t.RandaoMix[:32]...) } - { // Field #4 'NextRandaoMix' + { // Static Field #4 'NextRandaoMix' dst = append(dst, t.NextRandaoMix[:32]...) } - { // Field #5 'TotalBalance' + { // Static Field #5 'TotalBalance' dst = binary.LittleEndian.AppendUint64(dst, uint64(t.TotalBalance)) } - { // Field #6 'ActiveBalance' + { // Static Field #6 'ActiveBalance' dst = binary.LittleEndian.AppendUint64(dst, uint64(t.ActiveBalance)) } - { // Field #7 'FirstDepositIndex' + { // Static Field #7 'FirstDepositIndex' dst = binary.LittleEndian.AppendUint64(dst, t.FirstDepositIndex) } - // Offset #8 'PendingWithdrawals' - offset8 := len(dst) - dst = append(dst, 0, 0, 0, 0) - // Offset #9 'PendingConsolidations' - offset9 := len(dst) + // Offset Field #8 'PendingWithdrawals' + // Offset Field #9 'BuilderPendingWithdrawals' + dst = append(dst, 0, 0, 0, 0, 0, 0, 0, 0) + { // Static Field #10 'DelayedBuilderPaymentCount' + dst = binary.LittleEndian.AppendUint32(dst, t.DelayedBuilderPaymentCount) + } + // Offset Field #11 'PendingConsolidations' dst = append(dst, 0, 0, 0, 0) - { // Field #10 'ConsolidatingBalance' + { // Static Field #12 'ConsolidatingBalance' dst = binary.LittleEndian.AppendUint64(dst, uint64(t.ConsolidatingBalance)) } { // Dynamic Field #0 'ActiveValidators' - binary.LittleEndian.PutUint32(dst[offset0:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen:], uint32(len(dst)-dstlen)) t := t.ActiveValidators vlen := len(t) if vlen > 10000000 { - return nil, sszutils.ErrListTooBig + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "ActiveValidators") } - for i := range vlen { - t := &t[i] - { // Field #0 'ValidatorIndexOffset' + for idx1 := range vlen { + t := &t[idx1] + { // Static Field #0 'ValidatorIndexOffset' dst = binary.LittleEndian.AppendUint32(dst, t.ValidatorIndexOffset) } - { // Field #1 'EffectiveBalanceEth' + { // Static Field #1 'EffectiveBalanceEth' dst = binary.LittleEndian.AppendUint32(dst, t.EffectiveBalanceEth) } } } { // Dynamic Field #1 'ProposerDuties' - binary.LittleEndian.PutUint32(dst[offset1:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+4:], uint32(len(dst)-dstlen)) t := t.ProposerDuties vlen := len(t) if vlen > 100 { - return nil, sszutils.ErrListTooBig - } - for i := range vlen { - dst = binary.LittleEndian.AppendUint64(dst, uint64(t[i])) + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 100), "ProposerDuties") } + dst = sszutils.MarshalUint64Slice(dst, t[:vlen]) } { // Dynamic Field #2 'SyncCommitteeDuties' - binary.LittleEndian.PutUint32(dst[offset2:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+8:], uint32(len(dst)-dstlen)) t := t.SyncCommitteeDuties vlen := len(t) if vlen > 10000 { - return nil, sszutils.ErrListTooBig - } - for i := range vlen { - dst = binary.LittleEndian.AppendUint64(dst, uint64(t[i])) + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000), "SyncCommitteeDuties") } + dst = sszutils.MarshalUint64Slice(dst, t[:vlen]) } { // Dynamic Field #8 'PendingWithdrawals' - binary.LittleEndian.PutUint32(dst[offset8:], uint32(len(dst)-dstlen)) + binary.LittleEndian.PutUint32(dst[dstlen+100:], uint32(len(dst)-dstlen)) t := t.PendingWithdrawals vlen := len(t) if vlen > 10000000 { - return nil, sszutils.ErrListTooBig + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "PendingWithdrawals") } - for i := range vlen { - t := &t[i] + for idx1 := range vlen { + t := &t[idx1] if dst, err = t.MarshalSSZTo(dst); err != nil { - return nil, err + return nil, sszutils.ErrorWithPathf(err, "PendingWithdrawals[%d]", idx1) } } } - { // Dynamic Field #9 'PendingConsolidations' - binary.LittleEndian.PutUint32(dst[offset9:], uint32(len(dst)-dstlen)) + { // Dynamic Field #9 'BuilderPendingWithdrawals' + binary.LittleEndian.PutUint32(dst[dstlen+104:], uint32(len(dst)-dstlen)) + t := t.BuilderPendingWithdrawals + vlen := len(t) + if vlen > 10000000 { + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "BuilderPendingWithdrawals") + } + for idx1 := range vlen { + t := &t[idx1] + if dst, err = t.MarshalSSZTo(dst); err != nil { + return nil, sszutils.ErrorWithPathf(err, "BuilderPendingWithdrawals[%d]", idx1) + } + } + } + { // Dynamic Field #11 'PendingConsolidations' + binary.LittleEndian.PutUint32(dst[dstlen+112:], uint32(len(dst)-dstlen)) t := t.PendingConsolidations vlen := len(t) if vlen > 10000000 { - return nil, sszutils.ErrListTooBig + return nil, sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "PendingConsolidations") } - for i := range vlen { - t := &t[i] + for idx1 := range vlen { + t := &t[idx1] if dst, err = t.MarshalSSZTo(dst); err != nil { - return nil, err + return nil, sszutils.ErrorWithPathf(err, "PendingConsolidations[%d]", idx1) } } } @@ -128,23 +136,23 @@ func (t *EpochStatsPacked) MarshalSSZTo(buf []byte) (dst []byte, err error) { func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { buflen := len(buf) - if buflen < 116 { - return sszutils.ErrUnexpectedEOF + if buflen < 124 { + return sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 124) } // Field #0 'ActiveValidators' (offset) offset0 := int(binary.LittleEndian.Uint32(buf[0:4])) - if offset0 != 116 { - return sszutils.ErrOffset + if offset0 != 124 { + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "first offset does not match (offset: %d, static len: %d)", offset0, 124), "ActiveValidators:o") } // Field #1 'ProposerDuties' (offset) offset1 := int(binary.LittleEndian.Uint32(buf[4:8])) if offset1 < offset0 || offset1 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset1, offset0, buflen), "ProposerDuties:o") } // Field #2 'SyncCommitteeDuties' (offset) offset2 := int(binary.LittleEndian.Uint32(buf[8:12])) if offset2 < offset1 || offset2 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset2, offset1, buflen), "SyncCommitteeDuties:o") } { // Field #3 'RandaoMix' (static) buf := buf[12:44] @@ -169,15 +177,24 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { // Field #8 'PendingWithdrawals' (offset) offset8 := int(binary.LittleEndian.Uint32(buf[100:104])) if offset8 < offset2 || offset8 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset8, offset2, buflen), "PendingWithdrawals:o") } - // Field #9 'PendingConsolidations' (offset) + // Field #9 'BuilderPendingWithdrawals' (offset) offset9 := int(binary.LittleEndian.Uint32(buf[104:108])) if offset9 < offset8 || offset9 > buflen { - return sszutils.ErrOffset + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset9, offset8, buflen), "BuilderPendingWithdrawals:o") + } + { // Field #10 'DelayedBuilderPaymentCount' (static) + buf := buf[108:112] + t.DelayedBuilderPaymentCount = binary.LittleEndian.Uint32(buf) } - { // Field #10 'ConsolidatingBalance' (static) - buf := buf[108:116] + // Field #11 'PendingConsolidations' (offset) + offset11 := int(binary.LittleEndian.Uint32(buf[112:116])) + if offset11 < offset9 || offset11 > buflen { + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrOffset, "offset out of range (offset=%d, prev=%d, buflen=%d)", offset11, offset9, buflen), "PendingConsolidations:o") + } + { // Field #12 'ConsolidatingBalance' (static) + buf := buf[116:124] t.ConsolidatingBalance = phase0.Gwei(binary.LittleEndian.Uint64(buf)) } { // Field #0 'ActiveValidators' (dynamic) @@ -185,15 +202,15 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { val1 := t.ActiveValidators itemCount := len(buf) / 8 if len(buf)%8 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 8), "ActiveValidators") } val1 = sszutils.ExpandSlice(val1, itemCount) - for i := range itemCount { - val2 := val1[i] - buf := buf[8*i : 8*(i+1)] + for idx1 := range itemCount { + val2 := val1[idx1] + buf := buf[8*idx1 : 8*(idx1+1)] buflen := len(buf) if buflen < 8 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPathf(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "not enough data for fixed fields (have %d, needed %d)", buflen, 8), "ActiveValidators[%d]", idx1) } { // Field #0 'ValidatorIndexOffset' (static) buf := buf[0:4] @@ -203,7 +220,7 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { buf := buf[4:8] val2.EffectiveBalanceEth = binary.LittleEndian.Uint32(buf) } - val1[i] = val2 + val1[idx1] = val2 } t.ActiveValidators = val1 } @@ -212,13 +229,10 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { val3 := t.ProposerDuties itemCount := len(buf) / 8 if len(buf)%8 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszError(sszutils.ErrUnexpectedEOF, "uint64 list length not a multiple of 8"), "ProposerDuties") } val3 = sszutils.ExpandSlice(val3, itemCount) - for i := range itemCount { - buf := buf[8*i : 8*(i+1)] - val3[i] = phase0.ValidatorIndex(binary.LittleEndian.Uint64(buf)) - } + sszutils.UnmarshalUint64Slice(val3, buf) t.ProposerDuties = val3 } { // Field #2 'SyncCommitteeDuties' (dynamic) @@ -226,13 +240,10 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { val4 := t.SyncCommitteeDuties itemCount := len(buf) / 8 if len(buf)%8 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszError(sszutils.ErrUnexpectedEOF, "uint64 list length not a multiple of 8"), "SyncCommitteeDuties") } val4 = sszutils.ExpandSlice(val4, itemCount) - for i := range itemCount { - buf := buf[8*i : 8*(i+1)] - val4[i] = phase0.ValidatorIndex(binary.LittleEndian.Uint64(buf)) - } + sszutils.UnmarshalUint64Slice(val4, buf) t.SyncCommitteeDuties = val4 } { // Field #8 'PendingWithdrawals' (dynamic) @@ -240,32 +251,48 @@ func (t *EpochStatsPacked) UnmarshalSSZ(buf []byte) (err error) { val5 := t.PendingWithdrawals itemCount := len(buf) / 24 if len(buf)%24 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 24), "PendingWithdrawals") } val5 = sszutils.ExpandSlice(val5, itemCount) - for i := range itemCount { - buf := buf[24*i : 24*(i+1)] - if err = val5[i].UnmarshalSSZ(buf); err != nil { - return err + for idx1 := range itemCount { + buf := buf[24*idx1 : 24*(idx1+1)] + if err = val5[idx1].UnmarshalSSZ(buf); err != nil { + return sszutils.ErrorWithPathf(err, "PendingWithdrawals[%d]", idx1) } } t.PendingWithdrawals = val5 } - { // Field #9 'PendingConsolidations' (dynamic) - buf := buf[offset9:] - val6 := t.PendingConsolidations + { // Field #9 'BuilderPendingWithdrawals' (dynamic) + buf := buf[offset9:offset11] + val6 := t.BuilderPendingWithdrawals + itemCount := len(buf) / 36 + if len(buf)%36 != 0 { + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 36), "BuilderPendingWithdrawals") + } + val6 = sszutils.ExpandSlice(val6, itemCount) + for idx1 := range itemCount { + buf := buf[36*idx1 : 36*(idx1+1)] + if err = val6[idx1].UnmarshalSSZ(buf); err != nil { + return sszutils.ErrorWithPathf(err, "BuilderPendingWithdrawals[%d]", idx1) + } + } + t.BuilderPendingWithdrawals = val6 + } + { // Field #11 'PendingConsolidations' (dynamic) + buf := buf[offset11:] + val7 := t.PendingConsolidations itemCount := len(buf) / 16 if len(buf)%16 != 0 { - return sszutils.ErrUnexpectedEOF + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrUnexpectedEOF, "list length %d not a multiple of element size %d", len(buf), 16), "PendingConsolidations") } - val6 = sszutils.ExpandSlice(val6, itemCount) - for i := range itemCount { - buf := buf[16*i : 16*(i+1)] - if err = val6[i].UnmarshalSSZ(buf); err != nil { - return err + val7 = sszutils.ExpandSlice(val7, itemCount) + for idx1 := range itemCount { + buf := buf[16*idx1 : 16*(idx1+1)] + if err = val7[idx1].UnmarshalSSZ(buf); err != nil { + return sszutils.ErrorWithPathf(err, "PendingConsolidations[%d]", idx1) } } - t.PendingConsolidations = val6 + t.PendingConsolidations = val7 } return nil } @@ -283,9 +310,11 @@ func (t *EpochStatsPacked) SizeSSZ() (size int) { // Field #6 'ActiveBalance' static (8 bytes) // Field #7 'FirstDepositIndex' static (8 bytes) // Field #8 'PendingWithdrawals' offset (4 bytes) - // Field #9 'PendingConsolidations' offset (4 bytes) - // Field #10 'ConsolidatingBalance' static (8 bytes) - size += 116 + // Field #9 'BuilderPendingWithdrawals' offset (4 bytes) + // Field #10 'DelayedBuilderPaymentCount' static (4 bytes) + // Field #11 'PendingConsolidations' offset (4 bytes) + // Field #12 'ConsolidatingBalance' static (8 bytes) + size += 124 { // Dynamic field #0 'ActiveValidators' size += len(t.ActiveValidators) * 8 } @@ -298,7 +327,10 @@ func (t *EpochStatsPacked) SizeSSZ() (size int) { { // Dynamic field #8 'PendingWithdrawals' size += len(t.PendingWithdrawals) * 24 } - { // Dynamic field #9 'PendingConsolidations' + { // Dynamic field #9 'BuilderPendingWithdrawals' + size += len(t.BuilderPendingWithdrawals) * 36 + } + { // Dynamic field #11 'PendingConsolidations' size += len(t.PendingConsolidations) * 16 } return size @@ -323,11 +355,11 @@ func (t *EpochStatsPacked) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.ActiveValidators vlen := uint64(len(t)) if vlen > 10000000 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "ActiveValidators") } idx := hh.Index() - for i := range int(vlen) { - t := &t[i] + for idx1 := range int(vlen) { + t := &t[idx1] idx := hh.Index() { // Field #0 'ValidatorIndexOffset' hh.PutUint32(t.ValidatorIndexOffset) @@ -343,12 +375,10 @@ func (t *EpochStatsPacked) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.ProposerDuties vlen := uint64(len(t)) if vlen > 100 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 100), "ProposerDuties") } idx := hh.Index() - for i := range int(vlen) { - hh.AppendUint64(uint64(t[i])) - } + sszutils.HashUint64Slice(hh, t) hh.FillUpTo32() hh.MerkleizeWithMixin(idx, vlen, sszutils.CalculateLimit(100, vlen, 8)) } @@ -356,12 +386,10 @@ func (t *EpochStatsPacked) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.SyncCommitteeDuties vlen := uint64(len(t)) if vlen > 10000 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000), "SyncCommitteeDuties") } idx := hh.Index() - for i := range int(vlen) { - hh.AppendUint64(uint64(t[i])) - } + sszutils.HashUint64Slice(hh, t) hh.FillUpTo32() hh.MerkleizeWithMixin(idx, vlen, sszutils.CalculateLimit(10000, vlen, 8)) } @@ -384,33 +412,51 @@ func (t *EpochStatsPacked) HashTreeRootWith(hh sszutils.HashWalker) error { t := t.PendingWithdrawals vlen := uint64(len(t)) if vlen > 10000000 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "PendingWithdrawals") } idx := hh.Index() - for i := range int(vlen) { - t := &t[i] + for idx1 := range int(vlen) { + t := &t[idx1] if err := t.HashTreeRootWith(hh); err != nil { - return err + return sszutils.ErrorWithPathf(err, "PendingWithdrawals[%d]", idx1) } } hh.MerkleizeWithMixin(idx, vlen, sszutils.CalculateLimit(10000000, vlen, 32)) } - { // Field #9 'PendingConsolidations' + { // Field #9 'BuilderPendingWithdrawals' + t := t.BuilderPendingWithdrawals + vlen := uint64(len(t)) + if vlen > 10000000 { + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "BuilderPendingWithdrawals") + } + idx := hh.Index() + for idx1 := range int(vlen) { + t := &t[idx1] + if err := t.HashTreeRootWith(hh); err != nil { + return sszutils.ErrorWithPathf(err, "BuilderPendingWithdrawals[%d]", idx1) + } + } + hh.MerkleizeWithMixin(idx, vlen, sszutils.CalculateLimit(10000000, vlen, 32)) + } + { // Field #10 'DelayedBuilderPaymentCount' + hh.PutUint32(t.DelayedBuilderPaymentCount) + } + { // Field #11 'PendingConsolidations' t := t.PendingConsolidations vlen := uint64(len(t)) if vlen > 10000000 { - return sszutils.ErrListTooBig + return sszutils.ErrorWithPath(sszutils.NewSszErrorf(sszutils.ErrListTooBig, "list length %d exceeds maximum %d", vlen, 10000000), "PendingConsolidations") } idx := hh.Index() - for i := range int(vlen) { - t := &t[i] + for idx1 := range int(vlen) { + t := &t[idx1] if err := t.HashTreeRootWith(hh); err != nil { - return err + return sszutils.ErrorWithPathf(err, "PendingConsolidations[%d]", idx1) } } hh.MerkleizeWithMixin(idx, vlen, sszutils.CalculateLimit(10000000, vlen, 32)) } - { // Field #10 'ConsolidatingBalance' + { // Field #12 'ConsolidatingBalance' hh.PutUint64(uint64(t.ConsolidatingBalance)) } hh.Merkleize(idx) diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 194305fd6..4cb65ddfb 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -335,34 +335,27 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R for i, block := range canonicalBlocks { blockIndex := block.GetBlockIndex(indexer.ctx) if blockIndex == nil || blockIndex.ExecutionNumber == 0 { - fmt.Printf("payload status for slot %v: no execution payload\n", block.Slot) continue // no execution payload } // Find the next canonical block if i+1 >= len(allCanonicalBlocks) { - fmt.Printf("payload status for slot %v: no next canonical block\n", block.Slot) continue } nextBlock := allCanonicalBlocks[i+1] if nextBlock == nil { - fmt.Printf("payload status for slot %v: no next canonical block\n", block.Slot) continue } nextBlockIndex := nextBlock.GetBlockIndex(indexer.ctx) if nextBlockIndex == nil { - fmt.Printf("payload status for slot %v: no next canonical block index\n", block.Slot) continue } // Check if next block builds on this block's payload if !bytes.Equal(nextBlockIndex.ExecutionParentHash[:], blockIndex.ExecutionHash[:]) { - fmt.Printf("payload status for slot %v: orphaned\n", block.Slot) block.isPayloadOrphaned = true - } else { - fmt.Printf("payload status for slot %v: canonical\n", block.Slot) } } } diff --git a/indexer/beacon/state_sim.go b/indexer/beacon/state_sim.go index 7eaaa6710..40a1e7805 100644 --- a/indexer/beacon/state_sim.go +++ b/indexer/beacon/state_sim.go @@ -26,7 +26,7 @@ type stateSimulatorState struct { block *Block pendingWithdrawals []electra.PendingPartialWithdrawal builderPendingWithdrawals []gloas.BuilderPendingWithdrawal - builderDelayedCount int // how many entries in builderPendingWithdrawals are delayed/quorum payments + builderDelayedCount uint32 // how many entries in builderPendingWithdrawals are delayed/quorum payments additionalWithdrawals []phase0.ValidatorIndex pendingConsolidationCount uint64 validatorMap map[phase0.ValidatorIndex]*phase0.Validator @@ -49,24 +49,15 @@ func newStateSimulator(indexer *Indexer, epochStats *EpochStats) *stateSimulator } // getParentBlocks returns blocks that need to be replayed before the target block. -// Uses the sim's epoch as the boundary. -// -// In Fulu+, the epoch state is the post-state of the first block of the epoch, -// so the first block must be excluded (its effects are already in the state). -// The boundary is set to firstSlot+1 to exclude it. -// -// In pre-Fulu, the epoch state is from the last block of the previous epoch, -// so all blocks in the current epoch need to be replayed (boundary = firstSlot). +// Uses the sim's epoch as the boundary — all blocks from the epoch start up to +// (but not including) the target block are returned. +// The epoch state is always the pre-state of the first slot (post-epoch-transition), +// so all blocks in the epoch need to be replayed. func (sim *stateSimulator) getParentBlocks(block *Block) []*Block { chainState := sim.indexer.consensusPool.GetChainState() simEpoch := sim.epochStats.epoch minSlot := chainState.EpochToSlot(simEpoch) - // In Fulu+, skip the first block of the sim's epoch (state is its post-state) - if chainState.IsFuluEnabled(simEpoch) { - minSlot++ - } - parentBlocks := []*Block{} for { @@ -93,41 +84,6 @@ func (sim *stateSimulator) getParentBlocks(block *Block) []*Block { return parentBlocks } -// getSimForBlock returns the correct stateSimulator for a given block. -// In Fulu+, the first block of an epoch needs the previous epoch's sim, -// since the current epoch state is the post-state of that first block. -// For all other blocks, the current sim is returned unchanged. -func (sim *stateSimulator) getSimForBlock(block *Block) *stateSimulator { - chainState := sim.indexer.consensusPool.GetChainState() - blockEpoch := chainState.EpochOfSlot(block.Slot) - - if !chainState.IsFuluEnabled(blockEpoch) || blockEpoch == 0 { - return sim - } - - epochFirstSlot := chainState.EpochToSlot(blockEpoch) - if block.Slot != epochFirstSlot { - return sim - } - - // First block of a Fulu+ epoch: need previous epoch's sim - prevEpoch := blockEpoch - 1 - prevEpochStats := sim.indexer.epochCache.getEpochStatsByEpochAndRoot(prevEpoch, block.Root) - if prevEpochStats == nil { - if parentRoot := block.GetParentRoot(); parentRoot != nil { - prevEpochStats = sim.indexer.epochCache.getEpochStatsByEpochAndRoot(prevEpoch, *parentRoot) - } - } - if prevEpochStats != nil { - prevSim := newStateSimulator(sim.indexer, prevEpochStats) - if prevSim != nil { - return prevSim - } - } - - return sim // fallback: can't get prev epoch, use current (may be imprecise) -} - func (sim *stateSimulator) resetState(block *Block) *stateSimulatorState { pendingWithdrawals := sim.epochStatsValues.PendingWithdrawals if pendingWithdrawals == nil { @@ -154,7 +110,7 @@ func (sim *stateSimulator) resetState(block *Block) *stateSimulatorState { epochRoot: epochRoot, pendingWithdrawals: pendingWithdrawals, builderPendingWithdrawals: builderPendingWithdrawals, - builderDelayedCount: len(builderPendingWithdrawals), // all entries from epoch state are delayed/quorum payments + builderDelayedCount: sim.epochStatsValues.DelayedBuilderPaymentCount, // delayed payments from epoch transition are at the tail pendingConsolidationCount: 0, additionalWithdrawals: []phase0.ValidatorIndex{}, validatorMap: map[phase0.ValidatorIndex]*phase0.Validator{}, @@ -403,10 +359,18 @@ func (sim *stateSimulator) applyBlock(block *Block) [][]uint8 { // process builder pending withdrawals (come first in the spec) chainState := sim.indexer.consensusPool.GetChainState() chainSpec := chainState.GetSpecs() - processedBuilderWithdrawals := len(sim.prevState.builderPendingWithdrawals) + processedBuilderWithdrawals := uint32(len(sim.prevState.builderPendingWithdrawals)) if processedBuilderWithdrawals > 0 { - // Track how many delayed entries were consumed - delayedConsumed := processedBuilderWithdrawals + // Delayed entries are at the tail. When consuming N entries from the front, + // the delayed count decreases by however many delayed entries were in that batch. + // directCount = total - delayed; consumed from front = min(total, processed). + // If we consume all: delayed consumed = delayed count. + // If we consume partial: delayed consumed = max(0, processed - (total - delayed)). + directCount := uint32(0) + if processedBuilderWithdrawals > sim.prevState.builderDelayedCount { + directCount = processedBuilderWithdrawals - sim.prevState.builderDelayedCount + } + delayedConsumed := processedBuilderWithdrawals - directCount if delayedConsumed > sim.prevState.builderDelayedCount { delayedConsumed = sim.prevState.builderDelayedCount } @@ -418,9 +382,6 @@ func (sim *stateSimulator) applyBlock(block *Block) [][]uint8 { if block.HasExecutionPayload() && !block.isPayloadOrphaned { blockIndex := block.GetBlockIndex(sim.indexer.ctx) if blockIndex != nil && blockIndex.BuilderIndex != math.MaxUint64 { - // Block has a delivered payload from a builder — direct payment will be appended - // We don't know the exact BuilderPendingWithdrawal details here, but we track - // that a non-delayed entry was added to the queue sim.prevState.builderPendingWithdrawals = append(sim.prevState.builderPendingWithdrawals, gloas.BuilderPendingWithdrawal{ BuilderIndex: gloas.BuilderIndex(blockIndex.BuilderIndex), }) @@ -556,9 +517,6 @@ func (sim *stateSimulator) applyBlock(block *Block) [][]uint8 { } func (sim *stateSimulator) replayBlockResults(block *Block) [][]uint8 { - // Use the correct sim for this block (handles Fulu first-block-of-epoch case) - sim = sim.getSimForBlock(block) - chainState := sim.indexer.consensusPool.GetChainState() chainSpec := chainState.GetSpecs() if chainSpec.ElectraForkEpoch == nil || sim.epochStats.epoch < phase0.Epoch(*chainSpec.ElectraForkEpoch) { @@ -627,11 +585,8 @@ type withdrawalSimResult struct { // replayWithdrawalState simulates the pending withdrawal queue for the given block // and returns classification info for builder payments and counts of each category. -// This handles the Fulu epoch state nuances: -// - The Fulu state (post-block-1) has only remaining delayed entries (no direct from block 1) -// - process_execution_payload adds direct entries to the BACK (not in state_root) -// - For block 1: queue = [carry_over_direct?, delayed_0, ..., delayed_D] -// - For block 2+: queue = [remaining_delayed..., direct_from_prev_block?] +// The epoch state is always the pre-state of the first slot (post-epoch-transition), +// so all blocks from the epoch start are replayed uniformly — no first-slot special casing. func (sim *stateSimulator) replayWithdrawalState(block *Block) *withdrawalSimResult { result := &withdrawalSimResult{} @@ -641,9 +596,6 @@ func (sim *stateSimulator) replayWithdrawalState(block *Block) *withdrawalSimRes return result } - // Use the correct sim for this block (handles Fulu first-block-of-epoch case) - sim = sim.getSimForBlock(block) - // Replay state up to (but not including) target block parentBlocks := sim.getParentBlocks(block) state := sim.resetState(block) @@ -684,111 +636,49 @@ func (sim *stateSimulator) replayWithdrawalState(block *Block) *withdrawalSimRes } // classifyBuilderPayments determines the type and reference slot for each builder payment -// in the pending queue. Handles carry-over detection, delayed vs direct classification, -// and the fallback guard for unreliable detection. +// in the pending queue. The epoch pre-state queue layout is: +// +// [direct_from_prev_payloads..., delayed_0, ..., delayed_N] +// +// Direct entries (from delivered payloads) are at the front — the payload transition +// (process_execution_payload) runs before the epoch transition. Delayed entries +// (from process_builder_pending_payments during the epoch transition) are at the tail. +// The number of delayed entries is known from DelayedBuilderPaymentCount. +// +// During block replay, each block consumes all entries from the front and may append +// a new direct entry at the back if it has a delivered payload. func (sim *stateSimulator) classifyBuilderPayments(block *Block, builderCount int) []builderPaymentClassification { - chainState := sim.indexer.consensusPool.GetChainState() - chainSpec := chainState.GetSpecs() - blockEpoch := chainState.EpochOfSlot(block.Slot) - epochFirstSlot := chainState.EpochToSlot(blockEpoch) - isFirstBlock := block.Slot == epochFirstSlot - delayedCount := sim.prevState.builderDelayedCount - // Validate detection reliability for Fulu - hasCarryOver := false - fallback := false - var lastPrevEpochPayloadSlot phase0.Slot - - if chainState.IsFuluEnabled(blockEpoch) && blockEpoch > 0 { - prevEpoch := blockEpoch - 1 - prevEpochFirstSlot := chainState.EpochToSlot(prevEpoch) - currentEpochFirstSlot := chainState.EpochToSlot(blockEpoch) - - // Count payloads in previous epoch and find the last one - payloadCount := 0 - for slot := prevEpochFirstSlot; slot < currentEpochFirstSlot; slot++ { - blocks := sim.indexer.GetBlocksBySlot(slot) - for _, b := range blocks { - if b.HasExecutionPayload() && !b.isPayloadOrphaned { - payloadCount++ - lastPrevEpochPayloadSlot = slot - } - } - } - - // Guard: if delayed count exceeds prev epoch drain capacity, detection is unreliable. - // Each block can process up to MAX_WITHDRAWALS_PER_PAYLOAD - 1 builder withdrawals, - // but a delivered payload adds 1 direct payment back. Net drain = (max - 1) - 1 per payload. - netDrainPerPayload := int(chainSpec.MaxWithdrawalsPerPayload) - 2 - if netDrainPerPayload < 1 { - netDrainPerPayload = 1 - } - if delayedCount > payloadCount*netDrainPerPayload { - fallback = true - } - - // For block 1: check carry-over direct from last payload of prev epoch - if isFirstBlock && lastPrevEpochPayloadSlot > 0 { - blocks := sim.indexer.GetBlocksBySlot(lastPrevEpochPayloadSlot) - for _, b := range blocks { - blockIndex := b.GetBlockIndex(sim.indexer.ctx) - if blockIndex != nil && blockIndex.BuilderIndex != math.MaxUint64 { - hasCarryOver = true - } - } - } - } - - // Build classifications payments := make([]builderPaymentClassification, builderCount) - if fallback { - for i := range payments { - payments[i] = builderPaymentClassification{Type: dbtypes.WithdrawalTypeBuilderPayment} - } - return payments - } + // Delayed entries are at the tail: positions [builderCount - delayedCount, builderCount) + delayedStart := builderCount - int(delayedCount) - // Get previous block slot for direct payment ref slot - var prevBlockSlot phase0.Slot + // For direct payment ref slots: use the last replayed block's slot if available, + // otherwise fall back to the source block slot (last block of parent epoch). + // At epoch start (no blocks replayed yet), prevState.block is nil, so we use + // SourceBlockSlot for the direct payment that carried over from the parent epoch. + var directRefSlot phase0.Slot if sim.prevState.block != nil { - prevBlockSlot = sim.prevState.block.Slot + directRefSlot = sim.prevState.block.Slot + } else { + directRefSlot = sim.epochStatsValues.SourceBlockSlot } for i := range payments { - if isFirstBlock { - // Block 1: [carry_over_direct?, delayed_0, ..., delayed_D] - if hasCarryOver && i == 0 { - // First entry = carry-over direct from prev epoch's last payload - payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment - if lastPrevEpochPayloadSlot > 0 { - slot := uint64(lastPrevEpochPayloadSlot) - payments[i].RefSlot = &slot - } - } else { - // Remaining = delayed - delayedIdx := i - if hasCarryOver { - delayedIdx = i - 1 - } - payments[i].Type = dbtypes.WithdrawalTypeBuilderDelayedPayment - payments[i].RefSlot = sim.resolveDelayedPaymentRefSlot(delayedIdx, block) - } + if i >= delayedStart && delayedStart >= 0 { + // Delayed entry (tail of queue, from epoch transition) + payments[i].Type = dbtypes.WithdrawalTypeBuilderDelayedPayment + payments[i].RefSlot = sim.resolveDelayedPaymentRefSlot(i, block) + } else if directRefSlot > 0 { + // Direct entry with known ref slot (from a previous block's delivered payload) + payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment + slot := uint64(directRefSlot) + payments[i].RefSlot = &slot } else { - // Block 2+: [remaining_delayed..., direct_from_prev_block?] - if i < delayedCount { - // Delayed entry (front of queue) - payments[i].Type = dbtypes.WithdrawalTypeBuilderDelayedPayment - payments[i].RefSlot = sim.resolveDelayedPaymentRefSlot(i, block) - } else { - // Direct entry (back of queue) - payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment - if prevBlockSlot > 0 { - slot := uint64(prevBlockSlot) - payments[i].RefSlot = &slot - } - } + // Direct entry with unknown ref slot (rare leftover from earlier epochs) + payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment } } diff --git a/indexer/beacon/statetransition/builder.go b/indexer/beacon/statetransition/builder.go index f8b7eefc3..e88794743 100644 --- a/indexer/beacon/statetransition/builder.go +++ b/indexer/beacon/statetransition/builder.go @@ -16,7 +16,9 @@ const ( // the quorum threshold. Qualifying payments are promoted to BuilderPendingWithdrawals. // Then the 2-epoch window shifts forward. // New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-process_builder_pending_payments -func processBuilderPendingPayments(s *stateAccessor) { +// processBuilderPendingPayments returns the number of delayed payments appended to +// BuilderPendingWithdrawals. +func processBuilderPendingPayments(s *stateAccessor) uint32 { slotsPerEpoch := s.specs.SlotsPerEpoch quorum := getBuilderPaymentQuorumThreshold(s) @@ -26,6 +28,7 @@ func processBuilderPendingPayments(s *stateAccessor) { limit = uint64(len(s.BuilderPendingPayments)) } + count := uint32(0) for i := uint64(0); i < limit; i++ { payment := s.BuilderPendingPayments[i] if payment == nil || payment.Withdrawal == nil { @@ -33,6 +36,7 @@ func processBuilderPendingPayments(s *stateAccessor) { } if uint64(payment.Weight) >= quorum { s.BuilderPendingWithdrawals = append(s.BuilderPendingWithdrawals, payment.Withdrawal) + count++ } } @@ -43,6 +47,8 @@ func processBuilderPendingPayments(s *stateAccessor) { s.BuilderPendingPayments[i] = &gloas.BuilderPendingPayment{} } } + + return count } // getBuilderPaymentQuorumThreshold computes the quorum threshold for builder payments. diff --git a/indexer/beacon/statetransition/statetransition.go b/indexer/beacon/statetransition/statetransition.go index d8f461825..346c6f95c 100644 --- a/indexer/beacon/statetransition/statetransition.go +++ b/indexer/beacon/statetransition/statetransition.go @@ -22,6 +22,17 @@ import ( "github.com/ethpandaops/dora/clients/consensus" ) +// TransitionInfo collects metadata from the state transition that callers may +// need for downstream processing. Pass a non-nil pointer to PrepareEpochPreState +// to receive this information; pass nil if not needed. +type TransitionInfo struct { + // DelayedBuilderPayments is the number of delayed builder payments appended + // to BuilderPendingWithdrawals by the last epoch transition's + // process_builder_pending_payments. This tells the state simulator how many + // entries at the tail of the queue are delayed (vs direct payments from block payloads). + DelayedBuilderPayments uint32 +} + // PrepareEpochPreState takes a post-block state and mutates it into the pre-state // of the target epoch. This is the main entry point for epoch state preparation. // @@ -35,7 +46,9 @@ import ( // After this call, the state represents the pre-block state at the first slot of // the target epoch, with all epoch transitions applied — including builder payment // conversions, balance updates, proposer lookahead, etc. -func PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec) error { +// +// If info is non-nil, it is populated with metadata from the transition. +func PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec, info *TransitionInfo) error { if state.Version < spec.DataVersionFulu { return nil } @@ -49,7 +62,7 @@ func PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, // Step 2: Advance to the first slot of the target epoch. targetSlot := phase0.Slot(uint64(epoch) * specs.SlotsPerEpoch) - if err := processSlots(state, targetSlot, specs); err != nil { + if err := processSlots(state, targetSlot, specs, info); err != nil { return fmt.Errorf("process_slots to epoch %d (slot %d): %w", epoch, targetSlot, err) } @@ -64,7 +77,7 @@ func PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, // outputs we need. Jumps directly to each epoch boundary. // // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots -func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec) error { +func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec, info *TransitionInfo) error { currentSlot, err := state.Slot() if err != nil { return fmt.Errorf("failed to get state slot: %w", err) @@ -92,7 +105,7 @@ func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, spec } s.Slot = nextBoundary - if err := processEpochInternal(s); err != nil { + if err := processEpochInternal(s, info); err != nil { return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) } @@ -192,7 +205,7 @@ func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.E // // Fulu: https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#modified-process_epoch // Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_epoch -func processEpochInternal(s *stateAccessor) error { +func processEpochInternal(s *stateAccessor, info *TransitionInfo) error { if err := processJustificationAndFinalization(s); err != nil { return fmt.Errorf("process_justification_and_finalization: %w", err) } @@ -223,7 +236,10 @@ func processEpochInternal(s *stateAccessor) error { // Gloas-only: process builder pending payments if s.version >= spec.DataVersionGloas { - processBuilderPendingPayments(s) + delayedCount := processBuilderPendingPayments(s) + if info != nil { + info.DelayedBuilderPayments = delayedCount + } } processEffectiveBalanceUpdates(s) From 3c5282eba2ef422812aa56d4a6f3715ee5268dd3 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 9 Apr 2026 06:09:53 +0200 Subject: [PATCH 89/99] improve withdrawals rendering --- handlers/builder.go | 42 +++++++++++--- handlers/slot.go | 58 ++++++++++++++++++- handlers/validator.go | 17 +++++- handlers/withdrawals.go | 17 +++++- handlers/withdrawals_list.go | 18 +++++- indexer/beacon/epochstate.go | 11 ++-- indexer/beacon/epochstats.go | 12 ++-- indexer/beacon/state_sim.go | 37 ++++++------ templates/builder/withdrawals.html | 16 ++++- templates/slot/withdrawals.html | 30 ++++++++++ templates/validator/withdrawals.html | 16 ++++- templates/withdrawals/withdrawals.html | 12 ++++ .../withdrawals_list/withdrawals_list.html | 12 ++++ types/models/builders.go | 14 +++-- types/models/slot.go | 3 + types/models/validator.go | 2 + types/models/withdrawals.go | 2 + types/models/withdrawals_list.go | 2 + 18 files changed, 266 insertions(+), 55 deletions(-) diff --git a/handlers/builder.go b/handlers/builder.go index d140d4615..679e9a70f 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -266,6 +266,33 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b pageData.AdditionalWithdrawalCount = totalRows - 10 } + // Batch resolve blocks (including ref slot blocks) + blockUids := make([]uint64, 0, len(dbWithdrawals)*2) + blockUidSet := make(map[uint64]bool, len(dbWithdrawals)*2) + for _, w := range dbWithdrawals { + if !blockUidSet[w.BlockUid] { + blockUidSet[w.BlockUid] = true + blockUids = append(blockUids, w.BlockUid) + } + if w.RefSlot != nil && !blockUidSet[*w.RefSlot] { + blockUidSet[*w.RefSlot] = true + blockUids = append(blockUids, *w.RefSlot) + } + } + blockMap := make(map[uint64]*dbtypes.AssignedSlot, len(blockUids)) + if len(blockUids) > 0 { + blockFilter := &dbtypes.BlockFilter{ + BlockUids: blockUids, + WithOrphaned: 1, + } + blocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, 0, uint32(len(blockUids)), 0) + for _, b := range blocks { + if b.Block != nil { + blockMap[b.Block.BlockUid] = b + } + } + } + for _, w := range dbWithdrawals { slot := w.BlockUid >> 16 wd := &models.BuilderPageDataWithdrawal{ @@ -276,14 +303,15 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b Amount: w.Amount, } - // Resolve block root - blockFilter := &dbtypes.BlockFilter{ - BlockUids: []uint64{w.BlockUid}, - WithOrphaned: 1, + if blockInfo, ok := blockMap[w.BlockUid]; ok && blockInfo.Block != nil { + wd.BlockRoot = blockInfo.Block.Root } - blocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, 0, 1, 0) - if len(blocks) > 0 && blocks[0].Block != nil { - wd.BlockRoot = blocks[0].Block.Root + + if w.RefSlot != nil { + wd.RefSlot = *w.RefSlot >> 16 + if refBlock, ok := blockMap[*w.RefSlot]; ok && refBlock.Block != nil { + wd.RefSlotRoot = refBlock.Block.Root + } } pageData.Withdrawals = append(pageData.Withdrawals, wd) diff --git a/handlers/slot.go b/handlers/slot.go index c2b88a939..615e63489 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -9,6 +9,7 @@ import ( "math" "math/big" "net/http" + "slices" "strconv" "strings" "time" @@ -950,6 +951,48 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock pageData.WithdrawalsCount = uint64(len(executionWithdrawals)) pageData.Withdrawals = make([]*models.SlotPageWithdrawal, pageData.WithdrawalsCount) + + // Try to get enriched withdrawal data (type + ref slot) from the chain service. + // This works for both cached (unfinalized) and DB (finalized) blocks. + var enrichedWithdrawals []*dbtypes.Withdrawal + if cacheBlock := services.GlobalBeaconService.GetBeaconIndexer().GetBlockByRoot(blockData.Root); cacheBlock != nil { + isCanonical := slices.Contains(services.GlobalBeaconService.GetCanonicalForkKeys(), cacheBlock.GetForkId()) + enrichedWithdrawals = cacheBlock.GetDbWithdrawals(services.GlobalBeaconService.GetBeaconIndexer(), isCanonical) + } + if len(enrichedWithdrawals) == 0 { + dbWithdrawals, _ := db.GetWithdrawalsByBlockUid(ctx, blockUid) + enrichedWithdrawals = dbWithdrawals + } + + // Build a lookup map by block index for enrichment + enrichedMap := make(map[int16]*dbtypes.Withdrawal, len(enrichedWithdrawals)) + for _, ew := range enrichedWithdrawals { + enrichedMap[ew.BlockIdx] = ew + } + + // Batch resolve ref slot block roots + refBlockUids := make([]uint64, 0) + refBlockUidSet := make(map[uint64]bool) + for _, ew := range enrichedWithdrawals { + if ew.RefSlot != nil && !refBlockUidSet[*ew.RefSlot] { + refBlockUidSet[*ew.RefSlot] = true + refBlockUids = append(refBlockUids, *ew.RefSlot) + } + } + refBlockMap := make(map[uint64]*dbtypes.AssignedSlot, len(refBlockUids)) + if len(refBlockUids) > 0 { + refFilter := &dbtypes.BlockFilter{ + BlockUids: refBlockUids, + WithOrphaned: 1, + } + refBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, refFilter, 0, uint32(len(refBlockUids)), 0) + for _, b := range refBlocks { + if b.Block != nil { + refBlockMap[b.Block.BlockUid] = b + } + } + } + for i, withdrawal := range executionWithdrawals { validatorIndex := uint64(withdrawal.ValidatorIndex) isBuilder := validatorIndex&services.BuilderIndexFlag != 0 @@ -957,7 +1000,7 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock if isBuilder { displayIndex = validatorIndex &^ services.BuilderIndexFlag } - pageData.Withdrawals[i] = &models.SlotPageWithdrawal{ + wd := &models.SlotPageWithdrawal{ Index: uint64(withdrawal.Index), ValidatorIndex: displayIndex, ValidatorName: services.GlobalBeaconService.GetValidatorName(validatorIndex), @@ -965,6 +1008,19 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock Address: withdrawal.Address[:], Amount: uint64(withdrawal.Amount), } + + // Enrich with type and ref slot from chain service data + if enriched, ok := enrichedMap[int16(i)]; ok { + wd.Type = enriched.Type + if enriched.RefSlot != nil { + wd.RefSlot = *enriched.RefSlot >> 16 + if refBlock, ok := refBlockMap[*enriched.RefSlot]; ok && refBlock.Block != nil { + wd.RefSlotRoot = refBlock.Block.Root + } + } + } + + pageData.Withdrawals[i] = wd } } diff --git a/handlers/validator.go b/handlers/validator.go index 2ee55d165..a8663a2b2 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -625,14 +625,18 @@ func buildValidatorPageData(ctx context.Context, validatorIndex uint64, tabView } } - // Batch resolve blocks - blockUids := make([]uint64, 0, len(dbWithdrawals)) - blockUidSet := make(map[uint64]bool, len(dbWithdrawals)) + // Batch resolve blocks (including ref slot blocks) + blockUids := make([]uint64, 0, len(dbWithdrawals)*2) + blockUidSet := make(map[uint64]bool, len(dbWithdrawals)*2) for _, w := range dbWithdrawals { if !blockUidSet[w.BlockUid] { blockUidSet[w.BlockUid] = true blockUids = append(blockUids, w.BlockUid) } + if w.RefSlot != nil && !blockUidSet[*w.RefSlot] { + blockUidSet[*w.RefSlot] = true + blockUids = append(blockUids, *w.RefSlot) + } } blockMap := make(map[uint64]*dbtypes.AssignedSlot, len(blockUids)) if len(blockUids) > 0 { @@ -676,6 +680,13 @@ func buildValidatorPageData(ctx context.Context, validatorIndex uint64, tabView } } + if withdrawal.RefSlot != nil { + withdrawalData.RefSlot = *withdrawal.RefSlot >> 16 + if refBlock, ok := blockMap[*withdrawal.RefSlot]; ok && refBlock.Block != nil { + withdrawalData.RefSlotRoot = refBlock.Block.Root + } + } + pageData.Withdrawals = append(pageData.Withdrawals, withdrawalData) } diff --git a/handlers/withdrawals.go b/handlers/withdrawals.go index 4a2651f22..6a926e123 100644 --- a/handlers/withdrawals.go +++ b/handlers/withdrawals.go @@ -203,14 +203,18 @@ func buildWithdrawalsPageData(ctx context.Context, firstEpoch uint64, pageSize u } } - // Batch resolve blocks - blockUids := make([]uint64, 0, len(dbWithdrawals)) - blockUidSet := make(map[uint64]bool, len(dbWithdrawals)) + // Batch resolve blocks (including ref slot blocks) + blockUids := make([]uint64, 0, len(dbWithdrawals)*2) + blockUidSet := make(map[uint64]bool, len(dbWithdrawals)*2) for _, w := range dbWithdrawals { if !blockUidSet[w.BlockUid] { blockUidSet[w.BlockUid] = true blockUids = append(blockUids, w.BlockUid) } + if w.RefSlot != nil && !blockUidSet[*w.RefSlot] { + blockUidSet[*w.RefSlot] = true + blockUids = append(blockUids, *w.RefSlot) + } } blockMap := make(map[uint64]*dbtypes.AssignedSlot, len(blockUids)) if len(blockUids) > 0 { @@ -260,6 +264,13 @@ func buildWithdrawalsPageData(ctx context.Context, firstEpoch uint64, pageSize u } } + if withdrawal.RefSlot != nil { + withdrawalData.RefSlot = *withdrawal.RefSlot >> 16 + if refBlock, ok := blockMap[*withdrawal.RefSlot]; ok && refBlock.Block != nil { + withdrawalData.RefSlotRoot = refBlock.Block.Root + } + } + pageData.BeaconWithdrawals = append(pageData.BeaconWithdrawals, withdrawalData) } pageData.BeaconWithdrawalCount = uint64(len(pageData.BeaconWithdrawals)) diff --git a/handlers/withdrawals_list.go b/handlers/withdrawals_list.go index 5fc933087..456445770 100644 --- a/handlers/withdrawals_list.go +++ b/handlers/withdrawals_list.go @@ -269,14 +269,18 @@ func buildFilteredWithdrawalsListPageData(ctx context.Context, pageIdx uint64, p } } - // Batch resolve blocks for block root and number - blockUids := make([]uint64, 0, len(dbWithdrawals)) - blockUidSet := make(map[uint64]bool, len(dbWithdrawals)) + // Batch resolve blocks for block root, number, and ref slot + blockUids := make([]uint64, 0, len(dbWithdrawals)*2) + blockUidSet := make(map[uint64]bool, len(dbWithdrawals)*2) for _, w := range dbWithdrawals { if !blockUidSet[w.BlockUid] { blockUidSet[w.BlockUid] = true blockUids = append(blockUids, w.BlockUid) } + if w.RefSlot != nil && !blockUidSet[*w.RefSlot] { + blockUidSet[*w.RefSlot] = true + blockUids = append(blockUids, *w.RefSlot) + } } blockMap := make(map[uint64]*dbtypes.AssignedSlot, len(blockUids)) if len(blockUids) > 0 { @@ -329,6 +333,14 @@ func buildFilteredWithdrawalsListPageData(ctx context.Context, pageIdx uint64, p } } + // Resolve ref slot (block UID of the slot this builder payment refers to) + if withdrawal.RefSlot != nil { + withdrawalData.RefSlot = *withdrawal.RefSlot >> 16 + if refBlock, ok := blockMap[*withdrawal.RefSlot]; ok && refBlock.Block != nil { + withdrawalData.RefSlotRoot = refBlock.Block.Root + } + } + pageData.Withdrawals = append(pageData.Withdrawals, withdrawalData) } pageData.WithdrawalCount = uint64(len(pageData.Withdrawals)) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 40f9f0904..151261149 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -28,7 +28,7 @@ type epochState struct { highPriority bool stateSlot phase0.Slot - sourceBlockSlot phase0.Slot // slot of the source block (before epoch transition) + sourceBlockUid uint64 // block UID of the source block (before epoch transition) validatorBalances []phase0.Gwei builderBalances []phase0.Gwei randaoMixes []phase0.Root @@ -145,10 +145,13 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch specs := client.indexer.consensusPool.GetChainState().GetSpecs() - // Save the source block slot before epoch transition (needed for ref slot of + // Save the source block UID before epoch transition (needed for ref slot of // direct builder payments from the parent epoch's last block). - if sourceSlot, err := resState.Slot(); err == nil { - s.sourceBlockSlot = sourceSlot + if block != nil { + s.sourceBlockUid = block.BlockUID + } else if beaconBlock != nil { + slot, _ := beaconBlock.Slot() + s.sourceBlockUid = uint64(slot) << 16 } // For Fulu+: apply epoch transition to advance the state from the post-block state diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index d88650257..7d82d6f1d 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -60,8 +60,8 @@ type EpochStatsValues struct { FirstDepositIndex uint64 PendingWithdrawals []electra.PendingPartialWithdrawal BuilderPendingWithdrawals []gloas.BuilderPendingWithdrawal - DelayedBuilderPaymentCount uint32 // number of delayed payments at the tail of BuilderPendingWithdrawals - SourceBlockSlot phase0.Slot // slot of the source block (last block of parent epoch, before epoch transition) + DelayedBuilderPaymentCount uint32 // number of delayed payments at the tail of BuilderPendingWithdrawals + SourceBlockUid uint64 // block UID of the source block (last block of parent epoch) PendingConsolidations []electra.PendingConsolidation ConsolidatingBalance phase0.Gwei } @@ -82,7 +82,7 @@ type EpochStatsPacked struct { PendingWithdrawals []electra.PendingPartialWithdrawal `ssz-max:"10000000"` BuilderPendingWithdrawals []gloas.BuilderPendingWithdrawal `ssz-max:"10000000"` DelayedBuilderPaymentCount uint32 - SourceBlockSlot phase0.Slot + SourceBlockUid uint64 PendingConsolidations []electra.PendingConsolidation `ssz-max:"10000000"` ConsolidatingBalance phase0.Gwei } @@ -184,7 +184,7 @@ func (es *EpochStats) buildPackedSSZ() ([]byte, error) { PendingConsolidations: es.values.PendingConsolidations, BuilderPendingWithdrawals: es.values.BuilderPendingWithdrawals, DelayedBuilderPaymentCount: es.values.DelayedBuilderPaymentCount, - SourceBlockSlot: es.values.SourceBlockSlot, + SourceBlockUid: es.values.SourceBlockUid, ConsolidatingBalance: es.values.ConsolidatingBalance, } @@ -240,7 +240,7 @@ func (es *EpochStats) parsePackedSSZ(chainState *consensus.ChainState, ssz []byt PendingConsolidations: packedValues.PendingConsolidations, BuilderPendingWithdrawals: packedValues.BuilderPendingWithdrawals, DelayedBuilderPaymentCount: packedValues.DelayedBuilderPaymentCount, - SourceBlockSlot: packedValues.SourceBlockSlot, + SourceBlockUid: packedValues.SourceBlockUid, ConsolidatingBalance: packedValues.ConsolidatingBalance, } @@ -398,7 +398,7 @@ func (es *EpochStats) processState(indexer *Indexer, validatorSet []*phase0.Vali } } values.DelayedBuilderPaymentCount = dependentState.delayedBuilderPaymentCount - values.SourceBlockSlot = dependentState.sourceBlockSlot + values.SourceBlockUid = dependentState.sourceBlockUid for i, pendingConsolidation := range dependentState.pendingConsolidations { srcIndicee := pendingConsolidation.SourceIndex diff --git a/indexer/beacon/state_sim.go b/indexer/beacon/state_sim.go index 40a1e7805..f996bc4dc 100644 --- a/indexer/beacon/state_sim.go +++ b/indexer/beacon/state_sim.go @@ -655,15 +655,17 @@ func (sim *stateSimulator) classifyBuilderPayments(block *Block, builderCount in // Delayed entries are at the tail: positions [builderCount - delayedCount, builderCount) delayedStart := builderCount - int(delayedCount) - // For direct payment ref slots: use the last replayed block's slot if available, - // otherwise fall back to the source block slot (last block of parent epoch). + // For direct payment ref: use the last replayed block's UID if available, + // otherwise fall back to the source block UID (last block of parent epoch). // At epoch start (no blocks replayed yet), prevState.block is nil, so we use - // SourceBlockSlot for the direct payment that carried over from the parent epoch. - var directRefSlot phase0.Slot + // SourceBlockUid which was saved before the epoch transition was applied. + var directRefBlockUID *uint64 if sim.prevState.block != nil { - directRefSlot = sim.prevState.block.Slot - } else { - directRefSlot = sim.epochStatsValues.SourceBlockSlot + uid := sim.prevState.block.BlockUID + directRefBlockUID = &uid + } else if sim.epochStatsValues.SourceBlockUid > 0 { + uid := sim.epochStatsValues.SourceBlockUid + directRefBlockUID = &uid } for i := range payments { @@ -671,13 +673,12 @@ func (sim *stateSimulator) classifyBuilderPayments(block *Block, builderCount in // Delayed entry (tail of queue, from epoch transition) payments[i].Type = dbtypes.WithdrawalTypeBuilderDelayedPayment payments[i].RefSlot = sim.resolveDelayedPaymentRefSlot(i, block) - } else if directRefSlot > 0 { - // Direct entry with known ref slot (from a previous block's delivered payload) + } else if directRefBlockUID != nil { + // Direct entry with known ref block (from a previous block's delivered payload) payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment - slot := uint64(directRefSlot) - payments[i].RefSlot = &slot + payments[i].RefSlot = directRefBlockUID } else { - // Direct entry with unknown ref slot (rare leftover from earlier epochs) + // Direct entry with unknown ref block (rare leftover from earlier epochs) payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment } } @@ -685,8 +686,8 @@ func (sim *stateSimulator) classifyBuilderPayments(block *Block, builderCount in return payments } -// resolveDelayedPaymentRefSlot finds the slot a delayed builder payment is for -// by matching the builder index against blocks with missed payloads. +// resolveDelayedPaymentRefSlot finds the block UID of the block a delayed builder payment +// refers to, by matching the builder index against blocks with missed payloads. // Due to double-buffering in builder_pending_payments, delayed payments from epoch K // are evaluated at epoch K+2 boundary. So we look TWO epochs back from the block's epoch. // If the target epoch is already pruned from cache, falls back to DB query. @@ -719,8 +720,8 @@ func (sim *stateSimulator) resolveDelayedPaymentRefSlot(delayedIdx int, block *B continue } if blockIndex.BuilderIndex == uint64(builderIndex) && (!b.HasExecutionPayload() || b.isPayloadOrphaned) { - s := uint64(slot) - return &s + uid := b.BlockUID + return &uid } } } @@ -737,8 +738,8 @@ func (sim *stateSimulator) resolveDelayedPaymentRefSlot(delayedIdx int, block *B } if uint64(dbBuilderIndex) == uint64(builderIndex) && (assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusMissing || assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusOrphaned) { - s := assignedSlot.Block.Slot - return &s + uid := assignedSlot.Block.BlockUid + return &uid } } } diff --git a/templates/builder/withdrawals.html b/templates/builder/withdrawals.html index 91f3ac901..ea280d856 100644 --- a/templates/builder/withdrawals.html +++ b/templates/builder/withdrawals.html @@ -8,6 +8,7 @@ Slot Time Type + Ref Amount Status @@ -51,6 +52,17 @@ Unknown {{ end }} + + {{ if gt $withdrawal.RefSlot 0 }} + {{ if $withdrawal.RefSlotRoot }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ else }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ end }} + {{ else }} + - + {{ end }} + {{ formatEthFromGweiP $withdrawal.Amount 6 }} @@ -64,7 +76,7 @@ {{ end }} - + {{ if gt .AdditionalWithdrawalCount 0 }} View {{ formatAddCommas .AdditionalWithdrawalCount }} more withdrawals {{ end }} @@ -72,7 +84,7 @@ {{ else }} - +
    {{ template "timeline_svg" }}
    diff --git a/templates/slot/withdrawals.html b/templates/slot/withdrawals.html index 357bfd5cc..fa1d45f20 100644 --- a/templates/slot/withdrawals.html +++ b/templates/slot/withdrawals.html @@ -5,6 +5,8 @@ Index Validator Index + Type + Ref Recipient Address Amount @@ -20,6 +22,34 @@ {{ formatValidatorWithIndex $withdrawal.ValidatorIndex $withdrawal.ValidatorName }} {{ end }} + + {{ if eq $withdrawal.Type 1 }} + Full + {{ else if eq $withdrawal.Type 2 }} + Sweep + {{ else if eq $withdrawal.Type 3 }} + Requested + {{ else if eq $withdrawal.Type 4 }} + Builder Withdrawal + {{ else if eq $withdrawal.Type 5 }} + Builder Payment + {{ else if eq $withdrawal.Type 6 }} + Builder Delayed + {{ else }} + - + {{ end }} + + + {{ if gt $withdrawal.RefSlot 0 }} + {{ if $withdrawal.RefSlotRoot }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ else }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ end }} + {{ else }} + - + {{ end }} + {{ ethAddressLink $withdrawal.Address }} {{ formatEthFromGweiP $withdrawal.Amount 6 }} diff --git a/templates/validator/withdrawals.html b/templates/validator/withdrawals.html index 883e27c1b..6aa279045 100644 --- a/templates/validator/withdrawals.html +++ b/templates/validator/withdrawals.html @@ -8,6 +8,7 @@ Slot Time Type + Ref Address Amount Status @@ -52,6 +53,17 @@ Unknown {{ end }} + + {{ if gt $withdrawal.RefSlot 0 }} + {{ if $withdrawal.RefSlotRoot }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ else }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ end }} + {{ else }} + - + {{ end }} + {{ if $withdrawal.Address }} @@ -74,7 +86,7 @@ {{ end }} - + {{ if gt .AdditionalWithdrawalCount 0 }} View {{ formatAddCommas .AdditionalWithdrawalCount }} more withdrawals {{ end }} @@ -82,7 +94,7 @@ {{ else }} - +
    {{ template "timeline_svg" }}
    diff --git a/templates/withdrawals/withdrawals.html b/templates/withdrawals/withdrawals.html index 8ef63cad3..4f13bb0d1 100644 --- a/templates/withdrawals/withdrawals.html +++ b/templates/withdrawals/withdrawals.html @@ -343,6 +343,7 @@
    This table displays the most recent beacon chain with Block Time Type + Ref Validator Address Amount @@ -401,6 +402,17 @@
    This table displays the most recent beacon chain with Unknown {{ end }} + + {{ if gt $withdrawal.RefSlot 0 }} + {{ if $withdrawal.RefSlotRoot }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ else }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ end }} + {{ else }} + - + {{ end }} + {{ if $withdrawal.HasValidator }} {{- if $withdrawal.IsBuilder }} diff --git a/templates/withdrawals_list/withdrawals_list.html b/templates/withdrawals_list/withdrawals_list.html index db6807e97..8d5fc9ca5 100644 --- a/templates/withdrawals_list/withdrawals_list.html +++ b/templates/withdrawals_list/withdrawals_list.html @@ -158,6 +158,7 @@

    Block Time Type + Ref Validator Address Amount @@ -216,6 +217,17 @@

    Unknown {{ end }} + + {{ if gt $withdrawal.RefSlot 0 }} + {{ if $withdrawal.RefSlotRoot }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ else }} + {{ formatAddCommas $withdrawal.RefSlot }} + {{ end }} + {{ else }} + - + {{ end }} + {{ if $withdrawal.HasValidator }} {{- if $withdrawal.IsBuilder }} diff --git a/types/models/builders.go b/types/models/builders.go index cc512b5e3..6ccc6b34c 100644 --- a/types/models/builders.go +++ b/types/models/builders.go @@ -132,12 +132,14 @@ type BuilderPageDataDeposit struct { // BuilderPageDataWithdrawal represents a withdrawal on the builder detail page type BuilderPageDataWithdrawal struct { - SlotNumber uint64 `json:"slot"` - BlockRoot []byte `json:"block_root" ssz-size:"32"` - Time time.Time `json:"time"` - Orphaned bool `json:"orphaned"` - Type uint8 `json:"type"` - Amount uint64 `json:"amount"` + SlotNumber uint64 `json:"slot"` + BlockRoot []byte `json:"block_root" ssz-size:"32"` + Time time.Time `json:"time"` + Orphaned bool `json:"orphaned"` + Type uint8 `json:"type"` + Amount uint64 `json:"amount"` + RefSlot uint64 `json:"ref_slot"` + RefSlotRoot []byte `json:"ref_slot_root" ssz-size:"32"` } // BuilderPageDataExitTxDetails contains transaction details for EL-triggered exits diff --git a/types/models/slot.go b/types/models/slot.go index 8e87357d1..da2fd7328 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -236,6 +236,9 @@ type SlotPageWithdrawal struct { IsBuilder bool `json:"is_builder"` Address []byte `json:"address" ssz-size:"20"` Amount uint64 `json:"amount"` + Type uint8 `json:"type"` + RefSlot uint64 `json:"ref_slot"` + RefSlotRoot []byte `json:"ref_slot_root" ssz-size:"32"` } type SlotPageBlob struct { diff --git a/types/models/validator.go b/types/models/validator.go index a4e0c0e2a..13f018a0d 100644 --- a/types/models/validator.go +++ b/types/models/validator.go @@ -197,4 +197,6 @@ type ValidatorPageDataBeaconWithdrawal struct { Type uint8 `json:"type"` Address []byte `json:"address" ssz-size:"20"` Amount uint64 `json:"amount"` + RefSlot uint64 `json:"ref_slot"` + RefSlotRoot []byte `json:"ref_slot_root" ssz-size:"32"` } diff --git a/types/models/withdrawals.go b/types/models/withdrawals.go index 5b9fe2b60..8be173455 100644 --- a/types/models/withdrawals.go +++ b/types/models/withdrawals.go @@ -72,4 +72,6 @@ type WithdrawalsPageDataBeaconWithdrawal struct { ValidatorName string `json:"vname"` Address []byte `json:"address" ssz-size:"20"` Amount uint64 `json:"amount"` // Gwei + RefSlot uint64 `json:"ref_slot"` + RefSlotRoot []byte `json:"ref_slot_root" ssz-size:"32"` } diff --git a/types/models/withdrawals_list.go b/types/models/withdrawals_list.go index 4682a9319..c774b9246 100644 --- a/types/models/withdrawals_list.go +++ b/types/models/withdrawals_list.go @@ -51,4 +51,6 @@ type WithdrawalsListPageDataWithdrawal struct { ValidatorName string `json:"vname"` Address []byte `json:"address" ssz-size:"20"` Amount uint64 `json:"amount"` // Gwei + RefSlot uint64 `json:"ref_slot"` + RefSlotRoot []byte `json:"ref_slot_root" ssz-size:"32"` } From cfa5f65ec988e3b69d0a8481cd39905c9439404b Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 9 Apr 2026 06:43:29 +0200 Subject: [PATCH 90/99] implement full state transition --- .gitignore | 2 + .hack/devnet/run.sh | 4 + go.mod | 2 +- go.sum | 2 + indexer/beacon/epochcache.go | 9 +- indexer/beacon/epochstate.go | 266 ++++++- indexer/beacon/indexer.go | 3 + indexer/beacon/statetransition/block.go | 291 ++++++++ indexer/beacon/statetransition/builder.go | 119 ++- indexer/beacon/statetransition/committees.go | 324 +++++++++ indexer/beacon/statetransition/helpers.go | 17 +- indexer/beacon/statetransition/lookahead.go | 38 +- indexer/beacon/statetransition/operations.go | 685 ++++++++++++++++++ indexer/beacon/statetransition/pending.go | 213 +++--- indexer/beacon/statetransition/registry.go | 118 +-- indexer/beacon/statetransition/resets.go | 55 -- indexer/beacon/statetransition/rewards.go | 78 +- indexer/beacon/statetransition/slashings.go | 38 +- indexer/beacon/statetransition/state.go | 207 +++++- .../beacon/statetransition/statetransition.go | 175 +++-- indexer/beacon/statetransition/withdrawals.go | 267 +++++++ statecache/statecache.go | 301 ++++++++ types/config.go | 6 + 23 files changed, 2835 insertions(+), 385 deletions(-) create mode 100644 indexer/beacon/statetransition/block.go create mode 100644 indexer/beacon/statetransition/committees.go create mode 100644 indexer/beacon/statetransition/operations.go create mode 100644 indexer/beacon/statetransition/withdrawals.go create mode 100644 statecache/statecache.go diff --git a/.gitignore b/.gitignore index 101d69721..9e5d33ae8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ tmp-* .hack/devnet/generated-** .hack/devnet/custom-** +cmd/statetransition-test/ + diff --git a/.hack/devnet/run.sh b/.hack/devnet/run.sh index 330c8d115..9ffab3fc4 100755 --- a/.hack/devnet/run.sh +++ b/.hack/devnet/run.sh @@ -147,6 +147,10 @@ indexer: cachePersistenceDelay: 8 disableIndexWriter: false syncEpochCooldown: 1 + stateCache: + enabled: true + path: "${__dir}/generated-state-cache" + maxStates: 5 executionIndexer: enabled: true retention: 4368h diff --git a/go.mod b/go.mod index 7bc70021b..bc9c6d873 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/mashingan/smapping v0.1.19 github.com/minio/minio-go/v7 v7.0.99 github.com/mitchellh/mapstructure v1.5.0 - github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387 + github.com/pk910/dynamic-ssz v1.3.1-0.20260407212738-e97de623fd84 github.com/pressly/goose/v3 v3.27.0 github.com/probe-lab/eth-das-guardian v0.2.2 github.com/protolambda/bls12-381-util v0.1.0 diff --git a/go.sum b/go.sum index 95aeabcb8..71f099d4c 100644 --- a/go.sum +++ b/go.sum @@ -555,6 +555,8 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387 h1:XkL2iLFDP6/NKAOF0fysZrhaa/qtcAtqzhLUxSIy79s= github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387/go.mod h1:NmeFF4jxzVwWC8cnEhUB7xMI++8hd/0OZvZHFrUvFfs= +github.com/pk910/dynamic-ssz v1.3.1-0.20260407212738-e97de623fd84 h1:J3H3PiaO4+ej5HTK/nG/wnAj0jx+Ek2+0s8o+zujI4I= +github.com/pk910/dynamic-ssz v1.3.1-0.20260407212738-e97de623fd84/go.mod h1:NmeFF4jxzVwWC8cnEhUB7xMI++8hd/0OZvZHFrUvFfs= github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81 h1:mHgrerDiro/np9FCoJ19EYsCyl/CXMHFi34o+j3T+rE= github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81/go.mod h1:lwj0l8l51hIjqdQpODPea01JfE33nyM++1VGjBZau08= github.com/pk910/hashtree-bindings v0.1.0 h1:w7NyRWFi2OaYEFvo9ADcE/QU6PMuVLl3hBgx92KiH9c= diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index 6d5030ecc..825c65b22 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -538,7 +538,7 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { // Advance the already-loaded state to the next target epoch. // Payload is nil since it was already applied on the first PrepareEpochPreState call. var transitionInfo statetransition.TransitionInfo - if err := statetransition.PrepareEpochPreState(state, entry.epochState.targetEpoch, nil, specs, &transitionInfo); err != nil { + if err := statetransition.NewStateTransition(specs, cache.indexer.dynSsz).PrepareEpochPreState(state, entry.epochState.targetEpoch, nil, &transitionInfo); err != nil { cache.indexer.logger.Errorf("error advancing state to epoch %v: %v", entry.epochState.targetEpoch, err) continue } @@ -552,6 +552,13 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { entry.epochState.loadingStatus = 2 + // Store in state cache. + if sc := cache.indexer.stateCache; sc != nil { + if err := sc.Store(entry.epochState.slotRoot, entry.epochState.targetEpoch, state); err != nil { + cache.indexer.logger.Warnf("failed to cache state for epoch %v: %v", entry.epochState.targetEpoch, err) + } + } + // Signal ready. entry.epochState.readyChanMutex.Lock() if entry.epochState.readyChan != nil { diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 151261149..f6459474a 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -12,6 +12,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/indexer/beacon/statetransition" + "github.com/ethpandaops/dora/statecache" ) // epochState represents a beacon state which a epoch status depends on. @@ -128,9 +129,6 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } if beaconBlock != nil { - slot, _ := beaconBlock.Slot() - client.logger.Infof("loading state for block root %v (slot %v)", s.slotRoot.String(), slot) - var err error s.stateRoot, err = beaconBlock.StateRoot() if err != nil { @@ -138,11 +136,6 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch } } - resState, err := LoadBeaconState(ctx, client, s.stateRoot) - if err != nil { - return nil, err - } - specs := client.indexer.consensusPool.GetChainState().GetSpecs() // Save the source block UID before epoch transition (needed for ref slot of @@ -154,34 +147,75 @@ func (s *epochState) loadState(ctx context.Context, client *Client, cache *epoch s.sourceBlockUid = uint64(slot) << 16 } - // For Fulu+: apply epoch transition to advance the state from the post-block state - // of the parent epoch's last block to the pre-state of the target epoch. - // This includes payload processing (Gloas) and epoch transitions across boundaries. - // Skip for genesis (epoch 0) — the genesis state is already the correct pre-state. - if resState.Version >= spec.DataVersionFulu && s.targetEpoch > 0 { - var payloadEnvelope *gloas.ExecutionPayloadEnvelope - if resState.Version >= spec.DataVersionGloas { - var executionPayload *gloas.SignedExecutionPayloadEnvelope - if block != nil { - executionPayload = block.GetExecutionPayload(ctx) - } - if executionPayload == nil { - executionPayload, _ = LoadExecutionPayload(ctx, client, s.slotRoot) + // Try loading from state cache first (post-epoch-transition state). + var resState *spec.VersionedBeaconState + sc := client.indexer.stateCache + if sc != nil && sc.Check(s.slotRoot, s.targetEpoch) { + resState = sc.Load(s.slotRoot, s.targetEpoch) + if resState != nil { + client.logger.Infof("loaded epoch %v state from cache (dep: %v)", s.targetEpoch, s.slotRoot.String()) + } + } + + // Try replaying from parent epoch's cached state + blocks. This is much + // cheaper than loading the full state from the beacon API (which can be + // hundreds of MB on mainnet). On any failure, falls through to API load. + if resState == nil && sc != nil && s.targetEpoch > 0 { + if replayed := s.tryReplayFromParentState(ctx, client, block, beaconBlock, specs, sc); replayed != nil { + resState = replayed + } + } + + if resState == nil { + // Fall back to loading the full state from the beacon API. + apiStart := time.Now() + var err error + resState, err = LoadBeaconState(ctx, client, s.stateRoot) + if err != nil { + return nil, err + } + apiLoadDur := time.Since(apiStart) + + // For Fulu+: apply epoch transition to advance the state from the post-block state + // of the parent epoch's last block to the pre-state of the target epoch. + // Skip for genesis (epoch 0) — the genesis state is already the correct pre-state. + var epochTransitionDur time.Duration + if resState.Version >= spec.DataVersionFulu && s.targetEpoch > 0 { + var payloadEnvelope *gloas.ExecutionPayloadEnvelope + if resState.Version >= spec.DataVersionGloas { + var executionPayload *gloas.SignedExecutionPayloadEnvelope + if block != nil { + executionPayload = block.GetExecutionPayload(ctx) + } + if executionPayload == nil { + executionPayload, _ = LoadExecutionPayload(ctx, client, s.slotRoot) + } + if executionPayload != nil { + payloadEnvelope = executionPayload.Message + } } - if executionPayload != nil { - payloadEnvelope = executionPayload.Message + + epochStart := time.Now() + var transitionInfo statetransition.TransitionInfo + if err := statetransition.NewStateTransition(specs, client.indexer.dynSsz).PrepareEpochPreState(resState, s.targetEpoch, payloadEnvelope, &transitionInfo); err != nil { + return nil, fmt.Errorf("error applying epoch transition for epoch %v: %w", s.targetEpoch, err) } + epochTransitionDur = time.Since(epochStart) + s.delayedBuilderPaymentCount = transitionInfo.DelayedBuilderPayments } - var transitionInfo statetransition.TransitionInfo - if err := statetransition.PrepareEpochPreState(resState, s.targetEpoch, payloadEnvelope, specs, &transitionInfo); err != nil { - return nil, fmt.Errorf("error applying epoch transition for epoch %v: %w", s.targetEpoch, err) + client.logger.Infof("loaded epoch %v state from beacon API in %v + epoch transition %v", + s.targetEpoch, apiLoadDur.Round(time.Millisecond), epochTransitionDur.Round(time.Millisecond)) + + // Store in state cache for future use. + if sc != nil { + if err := sc.Store(s.slotRoot, s.targetEpoch, resState); err != nil { + client.logger.Warnf("failed to cache state for epoch %v: %v", s.targetEpoch, err) + } } - s.delayedBuilderPaymentCount = transitionInfo.DelayedBuilderPayments } - err = s.processState(resState, cache, specs) - if err != nil { + if err := s.processState(resState, cache, specs); err != nil { return nil, err } @@ -304,6 +338,180 @@ func (s *epochState) processState(state *spec.VersionedBeaconState, cache *epoch return nil } +// tryReplayFromParentState attempts to reconstruct the dependent block's post-state +// by loading the parent epoch's pre-state from cache and replaying all parent epoch +// blocks using the state transition. Returns the post-epoch-transition state ready +// for the target epoch, or nil if replay is not possible or verification fails. +// On any failure (missing inputs, ApplyBlock error, HTR mismatch, epoch transition +// error) the function returns nil and the caller falls back to loading the state +// from the beacon API. +func (s *epochState) tryReplayFromParentState( + ctx context.Context, + client *Client, + depBlock *Block, + depBeaconBlock *spec.VersionedSignedBeaconBlock, + specs *consensus.ChainSpec, + sc *statecache.StateCache, +) *spec.VersionedBeaconState { + if depBlock == nil || depBeaconBlock == nil { + return nil + } + + parentEpoch := s.targetEpoch - 1 + slotsPerEpoch := specs.SlotsPerEpoch + + // Walk back from depBlock to find the dependent root for the parent epoch + // (the last block before parentEpoch's first slot). + parentEpochFirstSlot := phase0.Slot(uint64(parentEpoch) * slotsPerEpoch) + walkBlock := depBlock + for walkBlock != nil && walkBlock.Slot >= parentEpochFirstSlot { + parentRoot := walkBlock.GetParentRoot() + if parentRoot == nil { + return nil + } + walkBlock = client.indexer.blockCache.getBlockByRoot(*parentRoot) + } + if walkBlock == nil { + return nil + } + parentDepRoot := walkBlock.Root + + // Parent epoch's pre-state must be in cache. + if !sc.Check(parentDepRoot, parentEpoch) { + return nil + } + parentState := sc.Load(parentDepRoot, parentEpoch) + if parentState == nil { + return nil + } + + // Skip replay across fork boundaries — the state version must match the + // dependent block's version (fork upgrades during state transition are not + // yet implemented). + if depBeaconBlock.Version != parentState.Version { + return nil + } + + // Collect all blocks in the parent epoch in slot order. + var epochBlocks []*Block + walkBlock = depBlock + for walkBlock != nil && walkBlock.Slot >= parentEpochFirstSlot { + epochBlocks = append(epochBlocks, walkBlock) + parentRoot := walkBlock.GetParentRoot() + if parentRoot == nil { + break + } + walkBlock = client.indexer.blockCache.getBlockByRoot(*parentRoot) + } + for i, j := 0, len(epochBlocks)-1; i < j; i, j = i+1, j-1 { + epochBlocks[i], epochBlocks[j] = epochBlocks[j], epochBlocks[i] + } + + // Reusable state transition; caches persist across all blocks in the epoch + // and the trailing epoch transition. + st := statetransition.NewStateTransition(specs, client.indexer.dynSsz) + + // prevStateRoot is the verified post-block HTR from the previous iteration — + // the same value as the next block's pre-state HTR — passed as a hint to + // skip the expensive HTR computation in the first process_slot. + var prevStateRoot phase0.Root + + replayStart := time.Now() + var blockApplyTotal time.Duration + for _, blk := range epochBlocks { + beaconBlock := blk.GetBlock(ctx) + if beaconBlock == nil { + return nil + } + + blockStart := time.Now() + if err := st.ApplyBlockWithStateRoot(parentState, beaconBlock, prevStateRoot); err != nil { + client.logger.Warnf("replay: ApplyBlock failed at slot %v: %v", blk.Slot, err) + return nil + } + + // Verify post-block state root matches the block header (post-block, + // pre-payload for Gloas). Catches state transition implementation bugs. + expectedStateRoot, _ := beaconBlock.StateRoot() + var gotStateRoot phase0.Root + var htrErr error + switch parentState.Version { + case spec.DataVersionFulu: + gotStateRoot, htrErr = parentState.Fulu.HashTreeRoot() + case spec.DataVersionGloas: + gotStateRoot, htrErr = parentState.Gloas.HashTreeRoot() + } + if htrErr != nil { + client.logger.Warnf("replay: HTR failed at slot %v: %v", blk.Slot, htrErr) + return nil + } + if gotStateRoot != expectedStateRoot { + client.logger.Warnf("replay: state root mismatch at slot %v (got %v, expected %v), falling back to API", + blk.Slot, gotStateRoot.String(), expectedStateRoot.String()) + return nil + } + prevStateRoot = gotStateRoot + blockApplyTotal += time.Since(blockStart) + + // For Gloas: apply execution payload if available. Skip the LAST block's + // payload — the dep block's state root is the pre-payload root, so + // PrepareEpochPreState will handle the payload. + isLastBlock := (blk == epochBlocks[len(epochBlocks)-1]) + if parentState.Version >= spec.DataVersionGloas && !isLastBlock { + payload := blk.GetExecutionPayload(ctx) + if payload != nil && payload.Message != nil { + if err := st.ApplyExecutionPayload(parentState, payload); err != nil { + client.logger.Warnf("replay: ApplyExecutionPayload failed at slot %v: %v", blk.Slot, err) + return nil + } + // Post-payload state HTR is recorded in the envelope itself. + prevStateRoot = payload.Message.StateRoot + } else { + // State mutated by something we can't predict — drop the hint. + prevStateRoot = phase0.Root{} + } + } + } + blockReplayDur := time.Since(replayStart) + + // Apply epoch transition to advance the state from the post-block state of + // the parent epoch's last block to the pre-state of the target epoch. + var epochTransitionDur time.Duration + if parentState.Version >= spec.DataVersionFulu { + var payloadEnvelope *gloas.ExecutionPayloadEnvelope + if parentState.Version >= spec.DataVersionGloas { + payload := depBlock.GetExecutionPayload(ctx) + if payload != nil { + payloadEnvelope = payload.Message + } + } + + epochStart := time.Now() + var transitionInfo statetransition.TransitionInfo + if err := st.PrepareEpochPreState(parentState, s.targetEpoch, payloadEnvelope, &transitionInfo); err != nil { + client.logger.Warnf("replay: epoch transition failed for epoch %v: %v", s.targetEpoch, err) + return nil + } + epochTransitionDur = time.Since(epochStart) + s.delayedBuilderPaymentCount = transitionInfo.DelayedBuilderPayments + } + + client.logger.Infof( + "replayed epoch %v: %d blocks in %v (apply %v) + epoch transition %v", + parentEpoch, len(epochBlocks), + blockReplayDur.Round(time.Millisecond), + blockApplyTotal.Round(time.Millisecond), + epochTransitionDur.Round(time.Millisecond), + ) + + // Cache the post-epoch-transition state for the target epoch. + if err := sc.Store(s.slotRoot, s.targetEpoch, parentState); err != nil { + client.logger.Warnf("failed to cache replayed state for epoch %v: %v", s.targetEpoch, err) + } + + return parentState +} + // isGloasPostPayloadState checks whether the Gloas state is post-payload // (i.e. execution payload deposits have been applied) for the given slot. func isGloasPostPayloadState(state *spec.VersionedBeaconState, slot phase0.Slot) bool { diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 015551057..085008367 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -17,6 +17,7 @@ import ( "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/dora/statecache" "github.com/ethpandaops/dora/utils" "github.com/ethpandaops/ethwallclock" ) @@ -41,6 +42,7 @@ type Indexer struct { maxParallelStateCalls uint16 // caches + stateCache *statecache.StateCache blockCache *blockCache epochCache *epochCache forkCache *forkCache @@ -114,6 +116,7 @@ func NewIndexer(ctx context.Context, logger logrus.FieldLogger, consensusPool *c blockDispatcher: &utils.Dispatcher[*Block]{}, } + indexer.stateCache = statecache.New(utils.Config, indexer.dynSsz) indexer.blockCache = newBlockCache(indexer) indexer.epochCache = newEpochCache(indexer) indexer.forkCache = newForkCache(indexer) diff --git a/indexer/beacon/statetransition/block.go b/indexer/beacon/statetransition/block.go new file mode 100644 index 000000000..fbe3fe9b1 --- /dev/null +++ b/indexer/beacon/statetransition/block.go @@ -0,0 +1,291 @@ +package statetransition + +import ( + "bytes" + "crypto/sha256" + "fmt" + + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" + dynssz "github.com/pk910/dynamic-ssz" + "github.com/pk910/dynamic-ssz/sszutils" +) + +// ApplyBlock applies a beacon block to the state in-place (process_block). +// The state must be at the block's slot (call PrepareEpochPreState or ProcessSlots first). +// After this call, the state matches the block's post-state (pre-payload for Gloas). +// +// If parentStateRoot is non-zero, it is used as the hint for the first +// process_slot's state-root caching, skipping the expensive HTR computation. +// This is safe when the caller already knows the HTR of the current state +// (e.g., from the parent block's state_root field). Subsequent process_slot +// calls (when there are skipped slots) still compute HTR normally. +// +// Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-block-processing +func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches, parentStateRoot phase0.Root) error { + if state.Version < spec.DataVersionFulu { + return nil + } + + s, err := newStateAccessorWithCaches(state, specs, caches) + if err != nil { + return fmt.Errorf("failed to create state accessor: %w", err) + } + + blockSlot, err := block.Slot() + if err != nil { + return fmt.Errorf("failed to get block slot: %w", err) + } + + // Advance state to the block's slot via process_slots. + // This implements the spec's process_slots exactly: + // while state.slot < slot: + // process_slot(state) + // if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + // process_epoch(state) + // state.slot += 1 + // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots + slotsPerEpoch := specs.SlotsPerEpoch + stateRootHint := parentStateRoot + for s.Slot < blockSlot { + if err := processSlotRootCaching(s, stateRootHint); err != nil { + return fmt.Errorf("process_slot at slot %d: %w", s.Slot, err) + } + // Hint only applies to the first iteration; subsequent slots have + // mutated state and need fresh HTR. + stateRootHint = phase0.Root{} + if (uint64(s.Slot)+1)%slotsPerEpoch == 0 { + if err := processEpochInternal(s, nil); err != nil { + return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) + } + } + s.Slot++ + s.writeBack() + } + + proposerIndex, err := block.ProposerIndex() + if err != nil { + return fmt.Errorf("failed to get proposer index: %w", err) + } + + parentRoot, err := block.ParentRoot() + if err != nil { + return fmt.Errorf("failed to get parent root: %w", err) + } + + bodyRoot, err := getBlockBodyRoot(block) + if err != nil { + return fmt.Errorf("failed to get body root: %w", err) + } + + // process_block_header + processBlockHeader(s, blockSlot, proposerIndex, parentRoot, bodyRoot) + + // process_withdrawals + processWithdrawals(s) + + // process_execution_payload (Fulu) — caches the execution payload header + if state.Version == spec.DataVersionFulu { + processFuluExecutionPayload(s, block, ds) + } + + // process_execution_payload_bid (Gloas) — records the builder's bid + if state.Version >= spec.DataVersionGloas { + processExecutionPayloadBid(s, block) + } + + // process_randao + processRandao(s, block) + + // process_eth1_data + processEth1Data(s, block) + + // process_operations + processOperations(s, block) + + // process_sync_aggregate + processSyncAggregate(s, block) + + s.writeBack() + + return nil +} + +// processSlotRootCaching implements process_slot: caches state root and block root. +// If stateRootHint is non-zero it is used directly, skipping the expensive HTR +// computation. The caller is responsible for ensuring the hint matches the +// current state's HTR (e.g., by sourcing it from the parent block's state_root). +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slot +func processSlotRootCaching(s *stateAccessor, stateRootHint phase0.Root) error { + // Cache state root — use hint if provided, otherwise compute HTR. + var stateRoot phase0.Root + if stateRootHint != (phase0.Root{}) { + stateRoot = stateRootHint + } else { + var err error + stateRoot, err = s.computeStateHTR() + if err != nil { + return fmt.Errorf("failed to compute state root: %w", err) + } + } + + idx := uint64(s.Slot) % s.specs.SlotsPerHistoricalRoot + s.StateRoots[idx] = stateRoot + + // Fill in latest block header's state root if it's the default zero value + if s.LatestBlockHeader != nil && s.LatestBlockHeader.StateRoot == (phase0.Root{}) { + s.LatestBlockHeader.StateRoot = stateRoot + } + + // Cache block root + blockRoot, err := s.computeLatestBlockHeaderHTR() + if err != nil { + return fmt.Errorf("failed to compute block root: %w", err) + } + + s.BlockRoots[idx] = blockRoot + + // Gloas: clear the next slot's execution payload availability bit. + s.clearNextSlotAvailabilityBit() + + return nil +} + +// processBlockHeader implements process_block_header. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#block-header +func processBlockHeader(s *stateAccessor, slot phase0.Slot, proposerIndex phase0.ValidatorIndex, parentRoot phase0.Root, bodyRoot phase0.Root) { + header := &phase0.BeaconBlockHeader{ + Slot: slot, + ProposerIndex: proposerIndex, + ParentRoot: parentRoot, + StateRoot: phase0.Root{}, // filled in by next process_slot + BodyRoot: bodyRoot, + } + + s.LatestBlockHeader = header +} + +// processRandao mixes in the block's RANDAO reveal. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#randao +func processRandao(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + randaoReveal, err := block.RandaoReveal() + if err != nil { + return + } + + epoch := s.currentEpoch() + idx := uint64(epoch) % s.specs.EpochsPerHistoricalVector + + // Mix in: XOR current mix with hash of the reveal + revealHash := sha256.Sum256(randaoReveal[:]) + var mixed phase0.Root + for i := 0; i < 32; i++ { + mixed[i] = s.RANDAOMixes[idx][i] ^ revealHash[i] + } + s.RANDAOMixes[idx] = mixed +} + +// processEth1Data adds the block's ETH1 vote. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#eth1-data +func processEth1Data(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + eth1Data, err := block.ETH1Data() + if err != nil || eth1Data == nil { + return + } + + s.ETH1DataVotes = append(s.ETH1DataVotes, eth1Data) + + // Check if majority vote + voteCount := 0 + for _, vote := range s.ETH1DataVotes { + if bytes.Equal(vote.BlockHash, eth1Data.BlockHash) && + vote.DepositRoot == eth1Data.DepositRoot && + vote.DepositCount == eth1Data.DepositCount { + voteCount++ + } + } + + threshold := s.specs.EpochsPerEth1VotingPeriod * s.specs.SlotsPerEpoch / 2 + if uint64(voteCount) > threshold { + s.ETH1Data = eth1Data + } +} + +// Transaction/Withdrawal list types for SSZ hash_tree_root computation via dynamic-ssz. +type transactionList []bellatrix.Transaction + +var _ = sszutils.Annotate[transactionList](`ssz-max:"1048576,1073741824" ssz-size:"?,?"`) + +type withdrawalList []*capella.Withdrawal + +var _ = sszutils.Annotate[withdrawalList](`ssz-max:"16"`) + +// processFuluExecutionPayload caches the execution payload header for Fulu blocks. +// https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#modified-process_execution_payload +func processFuluExecutionPayload(s *stateAccessor, block *spec.VersionedSignedBeaconBlock, ds *dynssz.DynSsz) { + if block.Fulu == nil || block.Fulu.Message == nil || block.Fulu.Message.Body == nil { + return + } + + payload := block.Fulu.Message.Body.ExecutionPayload + if payload == nil { + return + } + + if ds == nil { + return + } + + txs := make(transactionList, len(payload.Transactions)) + copy(txs, payload.Transactions) + txRoot, err := ds.HashTreeRoot(txs) + if err != nil { + return + } + + wds := make(withdrawalList, len(payload.Withdrawals)) + copy(wds, payload.Withdrawals) + wdRoot, err := ds.HashTreeRoot(wds) + if err != nil { + return + } + + s.LatestExecutionPayloadHeader = &deneb.ExecutionPayloadHeader{ + ParentHash: payload.ParentHash, + FeeRecipient: payload.FeeRecipient, + StateRoot: payload.StateRoot, + ReceiptsRoot: payload.ReceiptsRoot, + LogsBloom: payload.LogsBloom, + PrevRandao: payload.PrevRandao, + BlockNumber: payload.BlockNumber, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Timestamp: payload.Timestamp, + ExtraData: payload.ExtraData, + BaseFeePerGas: payload.BaseFeePerGas, + BlockHash: payload.BlockHash, + TransactionsRoot: phase0.Root(txRoot), + WithdrawalsRoot: phase0.Root(wdRoot), + BlobGasUsed: payload.BlobGasUsed, + ExcessBlobGas: payload.ExcessBlobGas, + } +} + +// getBlockBodyRoot computes the body root of a signed beacon block. +func getBlockBodyRoot(block *spec.VersionedSignedBeaconBlock) (phase0.Root, error) { + switch block.Version { + case spec.DataVersionFulu: + if block.Fulu != nil && block.Fulu.Message != nil && block.Fulu.Message.Body != nil { + return block.Fulu.Message.Body.HashTreeRoot() + } + case spec.DataVersionGloas: + if block.Gloas != nil && block.Gloas.Message != nil && block.Gloas.Message.Body != nil { + return block.Gloas.Message.Body.HashTreeRoot() + } + } + return phase0.Root{}, fmt.Errorf("unsupported block version: %v", block.Version) +} diff --git a/indexer/beacon/statetransition/builder.go b/indexer/beacon/statetransition/builder.go index e88794743..43e57f44f 100644 --- a/indexer/beacon/statetransition/builder.go +++ b/indexer/beacon/statetransition/builder.go @@ -1,6 +1,7 @@ package statetransition import ( + "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -63,25 +64,24 @@ func getBuilderPaymentQuorumThreshold(s *stateAccessor) uint64 { // Slides the PTC assignment window and computes new assignments for the lookahead epoch. // New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-process_ptc_window func processPtcWindow(s *stateAccessor) { - // PTC window is not available for Fulu states in the accessor - if s.version < 8 { // DataVersionGloas + // PTC window is Gloas-only. + if s.version < spec.DataVersionGloas { return } - g := s.rawState.Gloas - if g == nil || len(g.PTCWindow) == 0 { + if len(s.PTCWindow) == 0 { return } slotsPerEpoch := s.specs.SlotsPerEpoch // Slide window: remove first SLOTS_PER_EPOCH entries - windowLen := uint64(len(g.PTCWindow)) + windowLen := uint64(len(s.PTCWindow)) if windowLen <= slotsPerEpoch { return } - copy(g.PTCWindow, g.PTCWindow[slotsPerEpoch:]) + copy(s.PTCWindow, s.PTCWindow[slotsPerEpoch:]) // Compute new PTC assignments for the last SLOTS_PER_EPOCH entries nextEpoch := s.currentEpoch() + phase0.Epoch(s.specs.MinSeedLookahead) + 1 @@ -89,40 +89,111 @@ func processPtcWindow(s *stateAccessor) { lastStart := windowLen - slotsPerEpoch for i := uint64(0); i < slotsPerEpoch && lastStart+i < windowLen; i++ { - g.PTCWindow[lastStart+i] = computePtc(s, phase0.Slot(startSlot+i)) + s.PTCWindow[lastStart+i] = computePtc(s, phase0.Slot(startSlot+i)) } } // computePtc computes the PTC (Payload Timeliness Committee) for a given slot. -// This selects PTC_SIZE validators from the beacon committees using the PTC seed. +// Concatenates all beacon committees for the slot, then uses balance-weighted +// selection (without shuffling) to pick PTC_SIZE members. // New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-compute_ptc func computePtc(s *stateAccessor, slot phase0.Slot) []phase0.ValidatorIndex { epoch := phase0.Epoch(uint64(slot) / s.specs.SlotsPerEpoch) - activeIndices := s.getActiveValidatorIndices(epoch) - if len(activeIndices) == 0 { + ptcSize := s.specs.PtcSize + if ptcSize == 0 { return nil } - ptcSize := s.specs.PtcSize - if ptcSize == 0 { + // seed = hash(get_seed(state, epoch, DOMAIN_PTC_ATTESTER) + uint_to_bytes(slot)) + epochSeed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainPtcAttester)) + var buf [40]byte + copy(buf[:32], epochSeed[:]) + buf[32] = byte(slot) + buf[33] = byte(slot >> 8) + buf[34] = byte(slot >> 16) + buf[35] = byte(slot >> 24) + buf[36] = byte(slot >> 32) + buf[37] = byte(slot >> 40) + buf[38] = byte(slot >> 48) + buf[39] = byte(slot >> 56) + seed := hash256(buf[:]) + + // Concatenate all committees for this slot + cc := newCommitteeCache() + committeesPerSlot := s.getCommitteeCountPerSlot(epoch) + var indices []phase0.ValidatorIndex + for ci := uint64(0); ci < committeesPerSlot; ci++ { + committee := s.getBeaconCommittee(slot, ci, cc) + indices = append(indices, committee...) + } + + if len(indices) == 0 { + return nil + } + + // compute_balance_weighted_selection(state, indices, seed, size=PTC_SIZE, shuffle_indices=False) + return computeBalanceWeightedSelection(s, indices, seed, ptcSize, false) +} + +// computeBalanceWeightedSelection implements compute_balance_weighted_selection. +// Selects `size` validators from `indices` using balance-weighted rejection sampling. +// If shuffleIndices is true, candidates are sampled via compute_shuffled_index; +// otherwise they are traversed in order. +// https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-compute_balance_weighted_selection +func computeBalanceWeightedSelection(s *stateAccessor, indices []phase0.ValidatorIndex, seed phase0.Root, size uint64, shuffleIndices bool) []phase0.ValidatorIndex { + const maxRandomValue = 65535 // 2^16 - 1 + total := uint64(len(indices)) + if total == 0 { return nil } - // Get the seed for PTC computation - seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainPtcAttester)) + maxEB := uint64(s.specs.MaxEffectiveBalanceElectra) + if maxEB == 0 { + maxEB = uint64(s.specs.MaxEffectiveBalance) + } - // Select PTC members using shuffling - committeeSize := uint64(len(activeIndices)) - result := make([]phase0.ValidatorIndex, 0, ptcSize) + // Pre-compute effective balances for the candidate indices + effectiveBalances := make([]uint64, total) + for j, idx := range indices { + effectiveBalances[j] = uint64(s.Validators[idx].EffectiveBalance) + } - slotIndex := uint64(slot) % s.specs.SlotsPerEpoch - startOffset := (committeeSize * slotIndex * ptcSize) / (s.specs.SlotsPerEpoch * ptcSize) + selected := make([]phase0.ValidatorIndex, 0, size) + var randomBytes phase0.Root + i := uint64(0) + + for uint64(len(selected)) < size { + offset := (i % 16) * 2 + if offset == 0 { + // random_bytes = hash(seed + uint_to_bytes(i // 16)) + var rbuf [40]byte + copy(rbuf[:32], seed[:]) + quotient := i / 16 + rbuf[32] = byte(quotient) + rbuf[33] = byte(quotient >> 8) + rbuf[34] = byte(quotient >> 16) + rbuf[35] = byte(quotient >> 24) + rbuf[36] = byte(quotient >> 32) + rbuf[37] = byte(quotient >> 40) + rbuf[38] = byte(quotient >> 48) + rbuf[39] = byte(quotient >> 56) + randomBytes = hash256(rbuf[:]) + } - for i := uint64(0); i < ptcSize && i < committeeSize; i++ { - idx := (startOffset + i) % committeeSize - shuffledIdx := computeShuffledIndex(idx, committeeSize, seed, s.specs) - result = append(result, activeIndices[shuffledIdx]) + nextIndex := i % total + if shuffleIndices { + nextIndex = computeShuffledIndex(nextIndex, total, seed, s.specs) + } + + weight := effectiveBalances[nextIndex] * maxRandomValue + randomValue := uint64(randomBytes[offset]) | uint64(randomBytes[offset+1])<<8 + threshold := maxEB * randomValue + + if weight >= threshold { + selected = append(selected, indices[nextIndex]) + } + i++ } - return result + return selected } diff --git a/indexer/beacon/statetransition/committees.go b/indexer/beacon/statetransition/committees.go new file mode 100644 index 000000000..73c58cd23 --- /dev/null +++ b/indexer/beacon/statetransition/committees.go @@ -0,0 +1,324 @@ +package statetransition + +import ( + "encoding/binary" + "fmt" + + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/clients/consensus" + blsu "github.com/protolambda/bls12-381-util" +) + +// committeeKey uniquely identifies a beacon committee by slot and index. +type committeeKey struct { + slot phase0.Slot + index uint64 +} + +// committeeCache caches computed beacon committees and shuffled permutations +// to avoid recomputation across multiple attestations. +type committeeCache struct { + cache map[committeeKey][]phase0.ValidatorIndex + shuffled map[shuffleKey][]phase0.ValidatorIndex // epoch seed → shuffled active indices +} + +type shuffleKey struct { + seed phase0.Root + count uint64 +} + +func newCommitteeCache() *committeeCache { + return &committeeCache{ + cache: make(map[committeeKey][]phase0.ValidatorIndex, 64), + shuffled: make(map[shuffleKey][]phase0.ValidatorIndex), + } +} + +// get returns the cached committee, or nil if not cached. +func (c *committeeCache) get(slot phase0.Slot, index uint64) []phase0.ValidatorIndex { + return c.cache[committeeKey{slot: slot, index: index}] +} + +// put stores a committee in the cache. +func (c *committeeCache) put(slot phase0.Slot, index uint64, committee []phase0.ValidatorIndex) { + c.cache[committeeKey{slot: slot, index: index}] = committee +} + +// getShuffled returns a fully shuffled permutation of indices, cached by seed+count. +func (c *committeeCache) getShuffled(indices []phase0.ValidatorIndex, seed phase0.Root, specs *consensus.ChainSpec) []phase0.ValidatorIndex { + key := shuffleKey{seed: seed, count: uint64(len(indices))} + if cached, ok := c.shuffled[key]; ok { + return cached + } + + // Compute full shuffled permutation + indexCount := uint64(len(indices)) + shuffled := make([]phase0.ValidatorIndex, indexCount) + for i := uint64(0); i < indexCount; i++ { + shuffled[i] = indices[computeShuffledIndex(i, indexCount, seed, specs)] + } + + c.shuffled[key] = shuffled + return shuffled +} + +// getCommitteeCountPerSlot returns the number of committees per slot for the given epoch. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_committee_count_per_slot +func (s *stateAccessor) getCommitteeCountPerSlot(epoch phase0.Epoch) uint64 { + activeCount := uint64(len(s.getActiveValidatorIndices(epoch))) + committeesPerSlot := activeCount / s.specs.SlotsPerEpoch / s.specs.TargetCommitteeSize + if committeesPerSlot > s.specs.MaxCommitteesPerSlot { + committeesPerSlot = s.specs.MaxCommitteesPerSlot + } + if committeesPerSlot < 1 { + committeesPerSlot = 1 + } + return committeesPerSlot +} + +// getBeaconCommittee returns the beacon committee for the given slot and committee index. +// Uses the provided cache to avoid recomputation. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_beacon_committee +func (s *stateAccessor) getBeaconCommittee(slot phase0.Slot, committeeIndex uint64, cc *committeeCache) []phase0.ValidatorIndex { + if cc != nil { + if cached := cc.get(slot, committeeIndex); cached != nil { + return cached + } + } + + epoch := phase0.Epoch(uint64(slot) / s.specs.SlotsPerEpoch) + committeesPerSlot := s.getCommitteeCountPerSlot(epoch) + activeIndices := s.getActiveValidatorIndices(epoch) + seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainBeaconAttester)) + + slotIndex := uint64(slot) % s.specs.SlotsPerEpoch + index := slotIndex*committeesPerSlot + committeeIndex + count := committeesPerSlot * s.specs.SlotsPerEpoch + + committee := computeCommittee(activeIndices, seed, index, count, s.specs) + + if cc != nil { + cc.put(slot, committeeIndex, committee) + } + + return committee +} + +// computeCommittee computes a committee from the given parameters (no cache). +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_committee +func computeCommittee(indices []phase0.ValidatorIndex, seed phase0.Root, index, count uint64, specs *consensus.ChainSpec) []phase0.ValidatorIndex { + if count == 0 { + return nil + } + + indexCount := uint64(len(indices)) + start := (indexCount * index) / count + end := (indexCount * (index + 1)) / count + + shuffledIndices := computeShuffledBatch(start, end, indexCount, seed, specs) + committee := make([]phase0.ValidatorIndex, len(shuffledIndices)) + for i, shuffled := range shuffledIndices { + committee[i] = indices[shuffled] + } + + return committee +} + +// computeShuffledBatch computes shuffled indices for a contiguous range [start, end) +// using the swap-or-not shuffle, with per-round pivot caching. +// Much faster than calling computeShuffledIndex individually for each index. +func computeShuffledBatch(start, end, indexCount uint64, seed phase0.Root, specs *consensus.ChainSpec) []uint64 { + n := end - start + result := make([]uint64, n) + for i := uint64(0); i < n; i++ { + result[i] = start + i + } + + for currentRound := uint64(0); currentRound < specs.ShuffleRoundCount; currentRound++ { + // Compute pivot once per round (depends only on seed + round) + var buf [33]byte + copy(buf[0:32], seed[:]) + buf[32] = byte(currentRound) + pivotHash := hash256(buf[:]) + pivot := binary.LittleEndian.Uint64(pivotHash[:8]) % indexCount + + // Pre-compute the seed+round prefix for source hashes + var srcPrefix [33]byte + copy(srcPrefix[0:32], seed[:]) + srcPrefix[32] = byte(currentRound) + + // Cache source hashes by position/256 bucket + sourceCache := make(map[uint32]phase0.Root) + + for i := uint64(0); i < n; i++ { + index := result[i] + flip := (pivot + indexCount - index) % indexCount + position := index + if flip > index { + position = flip + } + + bucket := uint32(position / 256) + source, ok := sourceCache[bucket] + if !ok { + var buf2 [37]byte + copy(buf2[0:33], srcPrefix[:]) + binary.LittleEndian.PutUint32(buf2[33:37], bucket) + source = hash256(buf2[:]) + sourceCache[bucket] = source + } + + byteIdx := (position % 256) / 8 + bitIdx := position % 8 + bit := (source[byteIdx] >> bitIdx) & 1 + + if bit == 1 { + result[i] = flip + } + } + } + + return result +} + +// getAttestingIndices returns the set of attesting indices for an Electra+ attestation. +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-get_attesting_indices +func (s *stateAccessor) getAttestingIndices(slot phase0.Slot, committeeBits []byte, aggregationBits []byte, cc *committeeCache) []phase0.ValidatorIndex { + committeeIndices := getCommitteeIndicesFromBits(committeeBits) + + attestingSet := make(map[phase0.ValidatorIndex]struct{}) + committeeOffset := 0 + + for _, ci := range committeeIndices { + committee := s.getBeaconCommittee(slot, ci, cc) + for i, validatorIndex := range committee { + bitPos := committeeOffset + i + byteIdx := bitPos / 8 + bitIdx := bitPos % 8 + if byteIdx < len(aggregationBits) && aggregationBits[byteIdx]&(1<= maxEB*randomValue { + pubkeys = append(pubkeys, s.Validators[candidateIndex].PublicKey) + } + i++ + } + + aggregate, err := aggregateBLSPubkeys(pubkeys) + if err != nil { + // An aggregation failure means a malformed pubkey ended up in the + // committee, which is impossible for a valid chain — surface it loudly + // rather than silently producing a wrong state root. + panic(fmt.Errorf("failed to aggregate sync committee pubkeys: %w", err)) + } + + return &altair.SyncCommittee{ + Pubkeys: pubkeys, + AggregatePubkey: aggregate, + } +} + +// aggregateBLSPubkeys computes the BLS G1 aggregate of the given pubkeys. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#bls-signatures +func aggregateBLSPubkeys(pubkeys []phase0.BLSPubKey) (phase0.BLSPubKey, error) { + if len(pubkeys) == 0 { + return phase0.BLSPubKey{}, fmt.Errorf("cannot aggregate empty pubkey set") + } + parsed := make([]*blsu.Pubkey, len(pubkeys)) + for i, pk := range pubkeys { + var raw [48]byte = pk + p := new(blsu.Pubkey) + if err := p.Deserialize(&raw); err != nil { + return phase0.BLSPubKey{}, fmt.Errorf("invalid pubkey at index %d: %w", i, err) + } + parsed[i] = p + } + agg, err := blsu.AggregatePubkeys(parsed) + if err != nil { + return phase0.BLSPubKey{}, err + } + out := agg.Serialize() + return phase0.BLSPubKey(out), nil +} diff --git a/indexer/beacon/statetransition/helpers.go b/indexer/beacon/statetransition/helpers.go index c597053eb..57275acd0 100644 --- a/indexer/beacon/statetransition/helpers.go +++ b/indexer/beacon/statetransition/helpers.go @@ -13,9 +13,11 @@ func hash256(data []byte) phase0.Root { return phase0.Root(sha256.Sum256(data)) } -// hashTreeRoot is a placeholder for hash_tree_root of a vector of roots. -// For historical summaries, we need the Merkle root of block_roots and state_roots. -// This is a simplified version — the actual HTR of a fixed-size vector. +// hashTreeRoot computes the SSZ hash_tree_root of a fixed-size vector of +// roots (e.g., state.block_roots / state.state_roots, both +// Vector[Root, SLOTS_PER_HISTORICAL_ROOT]). Each root is already a 32-byte +// chunk, so no chunkification or length mixin is needed — just the binary +// Merkle tree over the leaves padded to the next power of two. func hashTreeRoot(roots []phase0.Root) phase0.Root { if len(roots) == 0 { return phase0.Root{} @@ -101,15 +103,6 @@ func computeShuffledIndex(index, indexCount uint64, seed phase0.Root, specs *con return index } -// getRandomByte extracts a pseudo-random byte from a seed. -func getRandomByte(seed phase0.Root, round, byteIndex uint64) uint8 { - var buf [40]byte - copy(buf[:32], seed[:]) - binary.LittleEndian.PutUint64(buf[32:40], round) - h := hash256(buf[:]) - return h[byteIndex%32] -} - // computeActivationExitEpoch returns the epoch at which a validator activation/exit takes effect. // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_activation_exit_epoch func computeActivationExitEpoch(epoch phase0.Epoch, specs *consensus.ChainSpec) phase0.Epoch { diff --git a/indexer/beacon/statetransition/lookahead.go b/indexer/beacon/statetransition/lookahead.go index 781b45d15..778858c89 100644 --- a/indexer/beacon/statetransition/lookahead.go +++ b/indexer/beacon/statetransition/lookahead.go @@ -53,18 +53,20 @@ func getBeaconProposerIndices(s *stateAccessor, epoch phase0.Epoch) []phase0.Val } // computeProposerIndex selects the proposer for a specific slot using the -// spec's compute_proposer_index function. -// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#compute_proposer_index +// Electra+ compute_proposer_index with per-slot seed from Fulu's compute_proposer_indices. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-compute_proposer_index +// Called via Fulu: https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#new-compute_proposer_indices func computeProposerIndex(s *stateAccessor, activeIndices []phase0.ValidatorIndex, epoch phase0.Epoch, slot phase0.Slot) phase0.ValidatorIndex { if len(activeIndices) == 0 { return 0 } - seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainBeaconProposer)) + epochSeed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainBeaconProposer)) - // Mix in slot + // Compute per-slot seed: hash(epoch_seed + uint_to_bytes(slot)) + // Fulu compute_proposer_indices: seeds = [hash(seed + uint_to_bytes(slot)) for each slot] var buf [40]byte - copy(buf[:32], seed[:]) + copy(buf[:32], epochSeed[:]) buf[32] = byte(slot) buf[33] = byte(slot >> 8) buf[34] = byte(slot >> 16) @@ -73,7 +75,7 @@ func computeProposerIndex(s *stateAccessor, activeIndices []phase0.ValidatorInde buf[37] = byte(slot >> 40) buf[38] = byte(slot >> 48) buf[39] = byte(slot >> 56) - slotSeed := hash256(buf[:]) + seed := hash256(buf[:]) indexCount := uint64(len(activeIndices)) maxEB := uint64(s.specs.MaxEffectiveBalanceElectra) @@ -81,18 +83,36 @@ func computeProposerIndex(s *stateAccessor, activeIndices []phase0.ValidatorInde maxEB = uint64(s.specs.MaxEffectiveBalance) } + // Electra: 16-bit random values (MAX_RANDOM_VALUE = 2^16 - 1) + const maxRandomValue = 65535 + i := uint64(0) for { candidateIndex := activeIndices[computeShuffledIndex(i%indexCount, indexCount, seed, s.specs)] - randomByte := getRandomByte(slotSeed, i/32, i%32) + + // Electra: random_bytes = hash(seed + uint_to_bytes(i // 16)) + // offset = (i % 16) * 2; random_value = LE uint16 from random_bytes[offset:offset+2] + var rbuf [40]byte + copy(rbuf[:32], seed[:]) + quotient := i / 16 + rbuf[32] = byte(quotient) + rbuf[33] = byte(quotient >> 8) + rbuf[34] = byte(quotient >> 16) + rbuf[35] = byte(quotient >> 24) + rbuf[36] = byte(quotient >> 32) + rbuf[37] = byte(quotient >> 40) + rbuf[38] = byte(quotient >> 48) + rbuf[39] = byte(quotient >> 56) + h := hash256(rbuf[:]) + offset := (i % 16) * 2 + randomValue := uint64(h[offset]) | uint64(h[offset+1])<<8 effectiveBalance := uint64(s.Validators[candidateIndex].EffectiveBalance) - if effectiveBalance*255 >= maxEB*uint64(randomByte) { + if effectiveBalance*maxRandomValue >= maxEB*randomValue { return candidateIndex } i++ - // Safety: prevent infinite loop if i > indexCount*100 { return activeIndices[0] } diff --git a/indexer/beacon/statetransition/operations.go b/indexer/beacon/statetransition/operations.go new file mode 100644 index 000000000..e62396a4c --- /dev/null +++ b/indexer/beacon/statetransition/operations.go @@ -0,0 +1,685 @@ +package statetransition + +import ( + "bytes" + "slices" + + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// processOperations implements process_operations. +// Processes all block body operations: slashings, attestations, deposits, exits, etc. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#operations +// Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_operations +func processOperations(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + proposerSlashings, _ := block.ProposerSlashings() + for _, slashing := range proposerSlashings { + processProposerSlashing(s, slashing) + } + + attesterSlashings, _ := block.AttesterSlashings() + for _, slashing := range attesterSlashings { + processAttesterSlashing(s, slashing) + } + + processAttestations(s, block) + + voluntaryExits, _ := block.VoluntaryExits() + for _, exit := range voluntaryExits { + processVoluntaryExit(s, exit) + } + + blsChanges, _ := block.BLSToExecutionChanges() + for _, change := range blsChanges { + processBLSToExecutionChange(s, change) + } + + // Process execution requests (Electra+): deposits, withdrawals, consolidations. + // In Fulu/Gloas, these come from the execution layer via the block body, NOT from + // the legacy deposit mechanism. + // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_execution_requests + requests, err := block.ExecutionRequests() + if err == nil && requests != nil { + for _, deposit := range requests.Deposits { + s.PendingDeposits = append(s.PendingDeposits, &electra.PendingDeposit{ + Pubkey: deposit.Pubkey, + WithdrawalCredentials: deposit.WithdrawalCredentials, + Amount: deposit.Amount, + Signature: deposit.Signature, + Slot: s.Slot, + }) + } + for _, withdrawal := range requests.Withdrawals { + processWithdrawalRequest(s, withdrawal) + } + for _, consolidation := range requests.Consolidations { + processConsolidationRequest(s, consolidation) + } + } +} + +// processProposerSlashing processes a proposer slashing. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#proposer-slashings +func processProposerSlashing(s *stateAccessor, slashing *phase0.ProposerSlashing) { + if slashing == nil || slashing.SignedHeader1 == nil { + return + } + proposerIndex := slashing.SignedHeader1.Message.ProposerIndex + if int(proposerIndex) >= len(s.Validators) { + return + } + slashValidator(s, proposerIndex) +} + +// processAttesterSlashing processes an attester slashing. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#attester-slashings +func processAttesterSlashing(s *stateAccessor, slashing any) { + att1, att2 := getSlashingAttestations(slashing) + if att1 == nil || att2 == nil { + return + } + + att1Indices, _ := att1.AttestingIndices() + att2Indices, _ := att2.AttestingIndices() + if att1Indices == nil || att2Indices == nil { + return + } + + att2Set := make(map[uint64]bool, len(att2Indices)) + for _, idx := range att2Indices { + att2Set[idx] = true + } + + for _, idx := range att1Indices { + if att2Set[idx] && phase0.ValidatorIndex(idx) < phase0.ValidatorIndex(len(s.Validators)) { + slashValidator(s, phase0.ValidatorIndex(idx)) + } + } +} + +// processAttestations processes all attestations in the block. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_attestation +func processAttestations(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + var attestations []*electra.Attestation + + switch block.Version { + case spec.DataVersionFulu: + if block.Fulu != nil && block.Fulu.Message != nil && block.Fulu.Message.Body != nil { + attestations = block.Fulu.Message.Body.Attestations + } + case spec.DataVersionGloas: + if block.Gloas != nil && block.Gloas.Message != nil && block.Gloas.Message.Body != nil { + attestations = block.Gloas.Message.Body.Attestations + } + } + + for _, att := range attestations { + processAttestation(s, att, s.caches.committeeCache) + } +} + +// processAttestation processes a single Electra+ attestation, updating participation flags. +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_attestation +func processAttestation(s *stateAccessor, att *electra.Attestation, cc *committeeCache) { + if att == nil || att.Data == nil { + return + } + + data := att.Data + currentEpoch := s.currentEpoch() + previousEpoch := s.previousEpoch() + + if data.Target.Epoch != currentEpoch && data.Target.Epoch != previousEpoch { + return + } + + attestingIndices := s.getAttestingIndices(data.Slot, []byte(att.CommitteeBits), []byte(att.AggregationBits), cc) + + // Determine which participation flags to set based on attestation properties. + isCurrentEpoch := data.Target.Epoch == currentEpoch + inclusionDelay := uint64(s.Slot) - uint64(data.Slot) + + // Check correctness of source, target, head + justifiedCheckpoint := s.PreviousJustifiedCheckpoint + if isCurrentEpoch { + justifiedCheckpoint = s.CurrentJustifiedCheckpoint + } + + sourceCorrect := data.Source.Epoch == justifiedCheckpoint.Epoch && data.Source.Root == justifiedCheckpoint.Root + targetCorrect := data.Target.Root == getBlockRoot(s, data.Target.Epoch) + headCorrect := data.BeaconBlockRoot == getBlockRootAtSlot(s, data.Slot) + + // Gloas: payload_matches check for head attestation. + // https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_attestation + payloadMatches := true + if s.version >= spec.DataVersionGloas { + if isAttestationSameSlot(s, data) { + payloadMatches = true + } else { + var payloadIndex uint64 + if s.getAvailabilityBit(data.Slot) { + payloadIndex = 1 + } + payloadMatches = uint64(data.Index) == payloadIndex + } + } + + // Determine which flags to set + var participationFlags [3]bool + if sourceCorrect && inclusionDelay <= intSqrt(s.specs.SlotsPerEpoch) { + participationFlags[TimelySourceFlagIndex] = true + } + if sourceCorrect && targetCorrect && inclusionDelay <= s.specs.SlotsPerEpoch { + participationFlags[TimelyTargetFlagIndex] = true + } + if sourceCorrect && targetCorrect && headCorrect && payloadMatches && inclusionDelay == s.specs.MinAttestationInclusionDelay { + participationFlags[TimelyHeadFlagIndex] = true + } + + // Select the right participation array and (Gloas) the corresponding + // builder pending payment slot. The Gloas spec keeps a 2-epoch sliding + // window of payments — current epoch is the second half, previous epoch + // the first half. + var participation []altair.ParticipationFlags + var builderPayment *gloas.BuilderPendingPayment + var builderPaymentIdx uint64 + slotsPerEpoch := s.specs.SlotsPerEpoch + if isCurrentEpoch { + participation = s.CurrentEpochParticipation + builderPaymentIdx = slotsPerEpoch + uint64(data.Slot)%slotsPerEpoch + } else { + participation = s.PreviousEpochParticipation + builderPaymentIdx = uint64(data.Slot) % slotsPerEpoch + } + if s.version >= spec.DataVersionGloas && builderPaymentIdx < uint64(len(s.BuilderPendingPayments)) { + builderPayment = s.BuilderPendingPayments[builderPaymentIdx] + } + + sameSlot := s.version >= spec.DataVersionGloas && isAttestationSameSlot(s, data) + + // Update participation flags, compute proposer reward, and (Gloas) accumulate + // builder payment weight from validators contributing new flags on same-slot + // attestations. + proposerRewardNumerator := uint64(0) + for _, index := range attestingIndices { + if int(index) >= len(participation) { + continue + } + + willSetNewFlag := false + for fi := 0; fi < 3; fi++ { + if !participationFlags[fi] { + continue + } + if !hasFlag(participation[index], fi) { + participation[index] |= altair.ParticipationFlags(1 << fi) + proposerRewardNumerator += uint64(s.getBaseReward(index)) * ParticipationFlagWeights[fi] + willSetNewFlag = true + } + } + + // Gloas: each validator contributes its effective balance to the + // builder payment weight at most once per slot, when it first sets a + // new flag on a same-slot attestation. Only counted when the slot + // actually has a builder payment with non-zero amount. + if willSetNewFlag && sameSlot && builderPayment != nil && + builderPayment.Withdrawal != nil && builderPayment.Withdrawal.Amount > 0 { + builderPayment.Weight += s.Validators[index].EffectiveBalance + } + } + + // Proposer reward + proposerRewardDenominator := uint64((WeightDenominator - ProposerWeight) * WeightDenominator / ProposerWeight) + if proposerRewardDenominator > 0 { + proposerReward := phase0.Gwei(proposerRewardNumerator / proposerRewardDenominator) + proposerIndex := s.getProposerIndex() + s.increaseBalance(proposerIndex, proposerReward) + } +} + +// processVoluntaryExit processes a voluntary exit. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#voluntary-exits +func processVoluntaryExit(s *stateAccessor, exit *phase0.SignedVoluntaryExit) { + if exit == nil || exit.Message == nil { + return + } + validatorIndex := exit.Message.ValidatorIndex + if int(validatorIndex) >= len(s.Validators) { + return + } + initiateValidatorExit(s, validatorIndex) +} + +// processBLSToExecutionChange updates a validator's withdrawal credentials from BLS to execution. +// https://github.com/ethereum/consensus-specs/blob/master/specs/capella/beacon-chain.md#new-process_bls_to_execution_change +func processBLSToExecutionChange(s *stateAccessor, signed *capella.SignedBLSToExecutionChange) { + if signed == nil || signed.Message == nil { + return + } + + change := signed.Message + validatorIndex := change.ValidatorIndex + if int(validatorIndex) >= len(s.Validators) { + return + } + + validator := s.Validators[validatorIndex] + + // Only apply if currently BLS credentials (0x00 prefix) + if len(validator.WithdrawalCredentials) == 0 || validator.WithdrawalCredentials[0] != 0x00 { + return + } + + // Update to execution credentials (0x01 prefix) + newCredentials := make([]byte, 32) + newCredentials[0] = 0x01 + copy(newCredentials[12:], change.ToExecutionAddress[:]) + validator.WithdrawalCredentials = newCredentials +} + +// processWithdrawalRequest processes an EL withdrawal request. +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_withdrawal_request +func processWithdrawalRequest(s *stateAccessor, request *electra.WithdrawalRequest) { + if request == nil { + return + } + isFullExitRequest := request.Amount == 0 + + // If partial withdrawal queue is full, only full exits are processed. + if uint64(len(s.PendingPartialWithdrawals)) == s.specs.PendingPartialWithdrawalsLimit && !isFullExitRequest { + return + } + + // Find validator by pubkey. + validatorIndex := findValidatorByPubkey(s, request.ValidatorPubkey) + if validatorIndex == nil { + return + } + validator := s.Validators[*validatorIndex] + + // Verify withdrawal credentials: must be 0x01 or 0x02 (execution prefix) and + // the source address must match the credential's address bytes. + if !hasExecutionWithdrawalCredential(validator) { + return + } + if !bytes.Equal(validator.WithdrawalCredentials[12:], request.SourceAddress[:]) { + return + } + // Verify the validator is active and exit has not been initiated. + if !isActiveValidator(validator, s.currentEpoch()) { + return + } + if validator.ExitEpoch != FarFutureEpoch { + return + } + // Validator must have been active long enough. + if s.currentEpoch() < validator.ActivationEpoch+phase0.Epoch(s.specs.ShardCommitteePeriod) { + return + } + + pendingBalanceToWithdraw := getPendingBalanceToWithdraw(s, *validatorIndex) + + if isFullExitRequest { + // Only exit if there are no pending partial withdrawals queued. + if pendingBalanceToWithdraw == 0 { + initiateValidatorExit(s, *validatorIndex) + } + return + } + + // Partial withdrawal: only allowed for compounding (0x02) validators with + // effective balance ≥ MIN_ACTIVATION_BALANCE and *actual* balance exceeding + // MIN_ACTIVATION_BALANCE + already-pending withdrawals. + if !hasCompoundingWithdrawalCredential(validator) { + return + } + hasSufficientEffectiveBalance := validator.EffectiveBalance >= phase0.Gwei(s.specs.MinActivationBalance) + hasExcessBalance := s.Balances[*validatorIndex] > phase0.Gwei(s.specs.MinActivationBalance)+pendingBalanceToWithdraw + if !hasSufficientEffectiveBalance || !hasExcessBalance { + return + } + + // Withdraw at most the excess (balance - MIN - pending), capped by request.Amount. + maxWithdrawable := s.Balances[*validatorIndex] - phase0.Gwei(s.specs.MinActivationBalance) - pendingBalanceToWithdraw + toWithdraw := phase0.Gwei(request.Amount) + if maxWithdrawable < toWithdraw { + toWithdraw = maxWithdrawable + } + + exitQueueEpoch := computeExitEpochAndUpdateChurn(s, toWithdraw) + withdrawableEpoch := exitQueueEpoch + phase0.Epoch(s.specs.MinValidatorWithdrawbilityDelay) + + s.PendingPartialWithdrawals = append(s.PendingPartialWithdrawals, &electra.PendingPartialWithdrawal{ + ValidatorIndex: *validatorIndex, + Amount: toWithdraw, + WithdrawableEpoch: withdrawableEpoch, + }) +} + +// blsG2PointAtInfinity is the canonical compressed encoding of the G2 point at +// infinity (0xc0 followed by 95 zero bytes). Used as a placeholder signature +// for synthetic pending deposits created via queueExcessActiveBalance. +var blsG2PointAtInfinity = func() phase0.BLSSignature { + var sig phase0.BLSSignature + sig[0] = 0xc0 + return sig +}() + +// switchToCompoundingValidator switches a validator's withdrawal credentials to +// compounding (0x02) and moves any balance above MIN_ACTIVATION_BALANCE into the +// pending_deposits queue. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-switch_to_compounding_validator +func switchToCompoundingValidator(s *stateAccessor, index phase0.ValidatorIndex) { + validator := s.Validators[index] + validator.WithdrawalCredentials[0] = 0x02 + queueExcessActiveBalance(s, index) +} + +// queueExcessActiveBalance moves any balance above MIN_ACTIVATION_BALANCE from +// the validator's balance into a synthetic pending deposit. Used when a +// validator switches to compounding credentials. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-queue_excess_active_balance +func queueExcessActiveBalance(s *stateAccessor, index phase0.ValidatorIndex) { + balance := s.Balances[index] + minActivation := phase0.Gwei(s.specs.MinActivationBalance) + if balance <= minActivation { + return + } + excess := balance - minActivation + s.Balances[index] = minActivation + + validator := s.Validators[index] + s.PendingDeposits = append(s.PendingDeposits, &electra.PendingDeposit{ + Pubkey: validator.PublicKey, + WithdrawalCredentials: append([]byte(nil), validator.WithdrawalCredentials...), + Amount: excess, + Signature: blsG2PointAtInfinity, + Slot: 0, // GENESIS_SLOT, distinguishes from real deposits + }) +} + +// processConsolidationRequest processes an EL consolidation request. +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_consolidation_request +func processConsolidationRequest(s *stateAccessor, request *electra.ConsolidationRequest) { + if request == nil { + return + } + + sourceIndex := findValidatorByPubkey(s, request.SourcePubkey) + targetIndex := findValidatorByPubkey(s, request.TargetPubkey) + if sourceIndex == nil || targetIndex == nil { + return + } + + sourceValidator := s.Validators[*sourceIndex] + targetValidator := s.Validators[*targetIndex] + + // Validate source credentials + if len(sourceValidator.WithdrawalCredentials) == 0 || sourceValidator.WithdrawalCredentials[0] == 0x00 { + return + } + if !bytes.Equal(sourceValidator.WithdrawalCredentials[12:], request.SourceAddress[:]) { + return + } + + // Self-consolidation: switch source to compounding credentials. + // Per the spec, this also moves any excess balance (above 32 ETH) into the + // pending_deposits queue via switch_to_compounding_validator. + // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_consolidation_request + if *sourceIndex == *targetIndex { + if sourceValidator.WithdrawalCredentials[0] == 0x01 { + switchToCompoundingValidator(s, *sourceIndex) + } + return + } + + // Check queue limit + if uint64(len(s.PendingConsolidations)) >= s.specs.PendingConsolidationsLimit { + return + } + + // Check target has compounding credentials + if targetValidator.WithdrawalCredentials[0] != 0x02 { + return + } + + // Check both active and not exiting + epoch := s.currentEpoch() + if !isActiveValidator(sourceValidator, epoch) || sourceValidator.ExitEpoch != FarFutureEpoch { + return + } + if !isActiveValidator(targetValidator, epoch) || targetValidator.ExitEpoch != FarFutureEpoch { + return + } + + // Check source age + if epoch < sourceValidator.ActivationEpoch+phase0.Epoch(s.specs.ShardCommitteePeriod) { + return + } + + // Check no pending partial withdrawals for source + for _, pw := range s.PendingPartialWithdrawals { + if pw.ValidatorIndex == *sourceIndex { + return + } + } + + s.PendingConsolidations = append(s.PendingConsolidations, &electra.PendingConsolidation{ + SourceIndex: *sourceIndex, + TargetIndex: *targetIndex, + }) + + // Initiate exit for the source + initiateValidatorExit(s, *sourceIndex) +} + +// processExecutionPayloadBid records the builder's bid in builder_pending_payments. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-process_execution_payload_bid +func processExecutionPayloadBid(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + if s.version < spec.DataVersionGloas { + return + } + if block.Gloas == nil || block.Gloas.Message == nil || block.Gloas.Message.Body == nil { + return + } + + signedBid := block.Gloas.Message.Body.SignedExecutionPayloadBid + if signedBid == nil || signedBid.Message == nil { + return + } + + bid := signedBid.Message + amount := bid.Value + + // Record the pending payment if there is some payment + if amount > 0 { + slotsPerEpoch := s.specs.SlotsPerEpoch + paymentIdx := slotsPerEpoch + uint64(s.Slot)%slotsPerEpoch + if paymentIdx < uint64(len(s.BuilderPendingPayments)) { + s.BuilderPendingPayments[paymentIdx] = &gloas.BuilderPendingPayment{ + Weight: 0, + Withdrawal: &gloas.BuilderPendingWithdrawal{ + FeeRecipient: bid.FeeRecipient, + Amount: amount, + BuilderIndex: bid.BuilderIndex, + }, + } + } + } + + // Cache the signed execution payload bid (always, regardless of amount) + s.LatestExecutionPayloadBid = bid +} + +// processSyncAggregate processes the sync committee aggregate. +// https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#sync-aggregate-processing +func processSyncAggregate(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { + syncAggregate, err := block.SyncAggregate() + if err != nil || syncAggregate == nil { + return + } + + committeeSize := s.specs.SyncCommitteeSize + if committeeSize == 0 || s.CurrentSyncCommittee == nil { + return + } + + totalActiveBalance := s.getTotalActiveBalance() + totalActiveIncrements := uint64(totalActiveBalance) / s.specs.EffectiveBalanceIncrement + totalBaseRewards := totalActiveIncrements * uint64(s.getBaseRewardPerIncrement()) + maxParticipantRewards := totalBaseRewards * SyncRewardWeight / WeightDenominator / s.specs.SlotsPerEpoch + participantReward := phase0.Gwei(maxParticipantRewards / committeeSize) + proposerReward := participantReward * ProposerWeight / (WeightDenominator - ProposerWeight) + + // Build pubkey → validator index map for the sync committee + syncCommitteePubkeys := s.CurrentSyncCommittee.Pubkeys + proposerIndex := s.getProposerIndex() + + for i := uint64(0); i < committeeSize && i < uint64(len(syncCommitteePubkeys)); i++ { + validatorIndex := findValidatorByPubkey(s, syncCommitteePubkeys[i]) + if validatorIndex == nil { + continue + } + + byteIdx := i / 8 + bitIdx := i % 8 + if byteIdx < uint64(len(syncAggregate.SyncCommitteeBits)) && + syncAggregate.SyncCommitteeBits[byteIdx]&(1< 0 { + penalty := validator.EffectiveBalance / phase0.Gwei(minPenaltyQuotient) + s.decreaseBalance(index, penalty) + } + + // Proposer + whistleblower reward + proposerIndex := s.getProposerIndex() + whistleblowerRewardQuotient := s.specs.WhistleblowerRewardQuotientElectra + if whistleblowerRewardQuotient == 0 { + whistleblowerRewardQuotient = s.specs.WhitelistRewardQuotient + } + if whistleblowerRewardQuotient > 0 { + whistleblowerReward := validator.EffectiveBalance / phase0.Gwei(whistleblowerRewardQuotient) + proposerReward := whistleblowerReward * ProposerWeight / WeightDenominator + s.increaseBalance(proposerIndex, proposerReward) + } +} + +// getProposerIndex returns the proposer for the current slot from the lookahead. +// Modified in Fulu: https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#modified-get_beacon_proposer_index +func (s *stateAccessor) getProposerIndex() phase0.ValidatorIndex { + if len(s.ProposerLookahead) > 0 { + idx := uint64(s.Slot) % s.specs.SlotsPerEpoch + return s.ProposerLookahead[idx] + } + // Fallback: compute directly + activeIndices := s.getActiveValidatorIndices(s.currentEpoch()) + return computeProposerIndex(s, activeIndices, s.currentEpoch(), s.Slot) +} + +// getBlockRootAtSlot returns the block root at a specific slot. +// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_block_root_at_slot +func getBlockRootAtSlot(s *stateAccessor, slot phase0.Slot) phase0.Root { + return s.BlockRoots[uint64(slot)%s.specs.SlotsPerHistoricalRoot] +} + +// isAttestationSameSlot checks if the attestation is for the block at the attestation slot. +// New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-is_attestation_same_slot +func isAttestationSameSlot(s *stateAccessor, data *phase0.AttestationData) bool { + if data.Slot == 0 { + return true + } + blockRoot := data.BeaconBlockRoot + slotBlockRoot := getBlockRootAtSlot(s, data.Slot) + prevBlockRoot := getBlockRootAtSlot(s, data.Slot-1) + return blockRoot == slotBlockRoot && blockRoot != prevBlockRoot +} + +// findValidatorByPubkey looks up a validator index by BLS public key. +// Uses a cached map for O(1) lookups after first call. +func findValidatorByPubkey(s *stateAccessor, pubkey phase0.BLSPubKey) *phase0.ValidatorIndex { + if s.caches.pubkeyCache == nil { + epoch := s.currentEpoch() + s.caches.pubkeyCache = make(map[phase0.BLSPubKey]phase0.ValidatorIndex, len(s.Validators)) + for i, v := range s.Validators { + if isActiveValidator(v, epoch) { + s.caches.pubkeyCache[v.PublicKey] = phase0.ValidatorIndex(i) + } + } + } + if idx, ok := s.caches.pubkeyCache[pubkey]; ok { + return &idx + } + return nil +} + +func maxEpoch(a, b phase0.Epoch) phase0.Epoch { + if a > b { + return a + } + return b +} + +// getSlashingAttestations extracts both attestations from an attester slashing. +func getSlashingAttestations(slashing any) (att1, att2 interface { + AttestingIndices() ([]uint64, error) +}) { + type attestationPair interface { + Attestation1() (interface{ AttestingIndices() ([]uint64, error) }, error) + Attestation2() (interface{ AttestingIndices() ([]uint64, error) }, error) + } + + if s, ok := slashing.(attestationPair); ok { + a1, _ := s.Attestation1() + a2, _ := s.Attestation2() + return a1, a2 + } + return nil, nil +} + +// Unused import guards. +var ( + _ = (*electra.WithdrawalRequest)(nil) + _ = (*electra.ConsolidationRequest)(nil) + _ = slices.Contains[[]int] +) diff --git a/indexer/beacon/statetransition/pending.go b/indexer/beacon/statetransition/pending.go index ed73172f0..bdc901756 100644 --- a/indexer/beacon/statetransition/pending.go +++ b/indexer/beacon/statetransition/pending.go @@ -8,78 +8,83 @@ import ( // processPendingDeposits implements the Electra+ version of process_pending_deposits. // New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_pending_deposits func processPendingDeposits(s *stateAccessor) error { + nextEpoch := s.currentEpoch() + 1 availableForProcessing := s.DepositBalanceToConsume + s.getActivationExitChurnLimit() - - processedCount := 0 + processedAmount := phase0.Gwei(0) nextDepositIndex := uint64(0) - depositsToPostpone := make([]*electra.PendingDeposit, 0) + var depositsToPostpone []*electra.PendingDeposit isChurnLimitReached := false + finalizedSlot := phase0.Slot(uint64(s.FinalizedCheckpoint.Epoch) * s.specs.SlotsPerEpoch) + + // Build pubkey index map for O(1) validator lookups. Without this, the + // nested loop over PendingDeposits × Validators is O(N×M), which on mainnet + // (51K deposits × 2.2M validators) would take hours. + pubkeyIndex := make(map[phase0.BLSPubKey]phase0.ValidatorIndex, len(s.Validators)) + for i, v := range s.Validators { + pubkeyIndex[v.PublicKey] = phase0.ValidatorIndex(i) + } for _, deposit := range s.PendingDeposits { - // Check if deposit is finalized - if deposit.Slot > phase0.Slot(uint64(s.FinalizedCheckpoint.Epoch)*s.specs.SlotsPerEpoch) { + // Do not process deposit requests if Eth1 bridge deposits are not yet applied. + if deposit.Slot > 0 && s.Eth1DepositIndex < s.DepositRequestsStartIndex { break } - processedCount++ - - // Check if validator exists - validatorIndex := findValidatorIndex(s, deposit.Pubkey) - if validatorIndex == nil { - // New validator: apply deposit - if !isChurnLimitReached { - cost := phase0.Gwei(deposit.Amount) - if cost > availableForProcessing { - isChurnLimitReached = true - depositsToPostpone = append(depositsToPostpone, deposit) - continue - } - availableForProcessing -= cost - applyPendingDeposit(s, deposit) - } else { - depositsToPostpone = append(depositsToPostpone, deposit) - } - continue + // Check if deposit has been finalized. + if deposit.Slot > finalizedSlot { + break } - // Existing validator - v := s.Validators[*validatorIndex] - - // Check for non-applied top-ups - if hasExecutionWithdrawalCredential(v) { - if !isChurnLimitReached { - cost := phase0.Gwei(deposit.Amount) - if cost > availableForProcessing { - isChurnLimitReached = true - depositsToPostpone = append(depositsToPostpone, deposit) - continue - } - availableForProcessing -= cost - } else { - depositsToPostpone = append(depositsToPostpone, deposit) - continue - } + // Check the per-epoch processing limit. + if nextDepositIndex >= s.specs.MaxPendingDepositsPerEpoch { + break } - // Apply deposit to existing validator - applyPendingDepositToExisting(s, *validatorIndex, deposit) + // Read validator state. + isValidatorExited := false + isValidatorWithdrawn := false + if existingIdx, ok := pubkeyIndex[deposit.Pubkey]; ok { + v := s.Validators[existingIdx] + isValidatorExited = v.ExitEpoch < FarFutureEpoch + isValidatorWithdrawn = v.WithdrawableEpoch < nextEpoch + } + + switch { + case isValidatorWithdrawn: + // Deposited balance will never become active. Apply without consuming churn. + applyPendingDeposit(s, deposit, pubkeyIndex) + + case isValidatorExited: + // Validator is exiting; postpone until after withdrawable epoch. + depositsToPostpone = append(depositsToPostpone, deposit) - if deposit.Slot > phase0.Slot(0) { - nextDepositIndex = uint64(deposit.Slot) + default: + // Check if deposit fits in the churn; if not, stop processing this epoch. + if processedAmount+phase0.Gwei(deposit.Amount) > availableForProcessing { + isChurnLimitReached = true + break + } + processedAmount += phase0.Gwei(deposit.Amount) + applyPendingDeposit(s, deposit, pubkeyIndex) } - } - _ = nextDepositIndex // used for deposit index tracking in full spec + if isChurnLimitReached { + break + } - // Remove processed deposits, keep postponed + unprocessed - remaining := make([]*electra.PendingDeposit, 0, len(depositsToPostpone)+len(s.PendingDeposits)-processedCount) - remaining = append(remaining, s.PendingDeposits[processedCount:]...) + // Regardless of how the deposit was handled, advance the queue cursor. + nextDepositIndex++ + } + + // state.pending_deposits = state.pending_deposits[next_deposit_index:] + deposits_to_postpone + remaining := make([]*electra.PendingDeposit, 0, len(s.PendingDeposits)-int(nextDepositIndex)+len(depositsToPostpone)) + remaining = append(remaining, s.PendingDeposits[nextDepositIndex:]...) remaining = append(remaining, depositsToPostpone...) s.PendingDeposits = remaining - // Update deposit balance to consume + // Accumulate churn only if the churn limit has been hit. if isChurnLimitReached { - s.DepositBalanceToConsume = availableForProcessing + s.DepositBalanceToConsume = availableForProcessing - processedAmount } else { s.DepositBalanceToConsume = 0 } @@ -87,23 +92,53 @@ func processPendingDeposits(s *stateAccessor) error { return nil } -// findValidatorIndex finds a validator by pubkey. Returns nil if not found. -func findValidatorIndex(s *stateAccessor, pubkey phase0.BLSPubKey) *phase0.ValidatorIndex { - for i, v := range s.Validators { - if v.PublicKey == pubkey { - idx := phase0.ValidatorIndex(i) - return &idx - } +// applyPendingDeposit implements apply_pending_deposit. If the validator does +// not exist, a new one is added to the registry. Otherwise the deposit amount +// is added to the existing validator's balance. +// +// Signature verification is skipped — we trust blocks fetched from a verified +// beacon node, so the deposit is always applied. +// +// pubkeyIndex is the local pubkey→index map maintained by processPendingDeposits; +// when a new validator is appended, the map is updated so subsequent deposits +// in the same loop find the new validator. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-apply_pending_deposit +func applyPendingDeposit(s *stateAccessor, deposit *electra.PendingDeposit, pubkeyIndex map[phase0.BLSPubKey]phase0.ValidatorIndex) { + if existingIdx, ok := pubkeyIndex[deposit.Pubkey]; ok { + s.increaseBalance(existingIdx, phase0.Gwei(deposit.Amount)) + return } - return nil + // New validator: add to registry. + addValidatorToRegistry(s, deposit.Pubkey, deposit.WithdrawalCredentials, phase0.Gwei(deposit.Amount)) + pubkeyIndex[deposit.Pubkey] = phase0.ValidatorIndex(len(s.Validators) - 1) +} + +// addValidatorToRegistry implements the Electra modified add_validator_to_registry. +// Constructs a validator via get_validator_from_deposit (which sets effective_balance +// based on the deposit amount and the credential type) and appends it to the +// registry along with matching balance/participation/inactivity entries. +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-add_validator_to_registry +func addValidatorToRegistry(s *stateAccessor, pubkey phase0.BLSPubKey, withdrawalCredentials []byte, amount phase0.Gwei) { + v := getValidatorFromDeposit(s, pubkey, withdrawalCredentials, amount) + s.Validators = append(s.Validators, v) + s.Balances = append(s.Balances, amount) + s.PreviousEpochParticipation = append(s.PreviousEpochParticipation, 0) + s.CurrentEpochParticipation = append(s.CurrentEpochParticipation, 0) + s.InactivityScores = append(s.InactivityScores, 0) } -// applyPendingDeposit processes a deposit for a new validator. -func applyPendingDeposit(s *stateAccessor, deposit *electra.PendingDeposit) { - // Add new validator +// getValidatorFromDeposit implements the Electra modified get_validator_from_deposit. +// Returns a validator with effective_balance computed as +// min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, max_effective_balance). +// +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-get_validator_from_deposit +func getValidatorFromDeposit(s *stateAccessor, pubkey phase0.BLSPubKey, withdrawalCredentials []byte, amount phase0.Gwei) *phase0.Validator { + credentials := append([]byte(nil), withdrawalCredentials...) v := &phase0.Validator{ - PublicKey: deposit.Pubkey, - WithdrawalCredentials: deposit.WithdrawalCredentials, + PublicKey: pubkey, + WithdrawalCredentials: credentials, EffectiveBalance: 0, Slashed: false, ActivationEligibilityEpoch: FarFutureEpoch, @@ -111,42 +146,46 @@ func applyPendingDeposit(s *stateAccessor, deposit *electra.PendingDeposit) { ExitEpoch: FarFutureEpoch, WithdrawableEpoch: FarFutureEpoch, } - s.Validators = append(s.Validators, v) - s.Balances = append(s.Balances, 0) - s.PreviousEpochParticipation = append(s.PreviousEpochParticipation, 0) - s.CurrentEpochParticipation = append(s.CurrentEpochParticipation, 0) - s.InactivityScores = append(s.InactivityScores, 0) - - idx := phase0.ValidatorIndex(len(s.Validators) - 1) - s.increaseBalance(idx, phase0.Gwei(deposit.Amount)) -} - -// applyPendingDepositToExisting applies a deposit to an existing validator. -func applyPendingDepositToExisting(s *stateAccessor, index phase0.ValidatorIndex, deposit *electra.PendingDeposit) { - s.increaseBalance(index, phase0.Gwei(deposit.Amount)) + maxEB := s.getMaxEffectiveBalance(v) + increment := phase0.Gwei(s.specs.EffectiveBalanceIncrement) + rounded := amount - amount%increment + if rounded < maxEB { + v.EffectiveBalance = rounded + } else { + v.EffectiveBalance = maxEB + } + return v } // processPendingConsolidations implements the Electra+ version of process_pending_consolidations. // New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_pending_consolidations func processPendingConsolidations(s *stateAccessor) { nextEpoch := s.currentEpoch() + 1 - processedCount := 0 + nextPendingConsolidation := 0 for _, consolidation := range s.PendingConsolidations { sourceValidator := s.Validators[consolidation.SourceIndex] + // Slashed source: skip (count as processed) and continue. + if sourceValidator.Slashed { + nextPendingConsolidation++ + continue + } + // Source not yet withdrawable: stop processing (queue is ordered). if sourceValidator.WithdrawableEpoch > nextEpoch { break } - processedCount++ - // Move balance from source to target - targetValidator := s.Validators[consolidation.TargetIndex] - _ = targetValidator // used for credential check in full spec + // Move only the *active* balance — bounded by the effective balance. + // Any balance above effective_balance becomes a normal withdrawal later. + sourceEffectiveBalance := s.Balances[consolidation.SourceIndex] + if sourceValidator.EffectiveBalance < sourceEffectiveBalance { + sourceEffectiveBalance = sourceValidator.EffectiveBalance + } - activeBalance := s.Balances[consolidation.SourceIndex] - s.decreaseBalance(consolidation.SourceIndex, activeBalance) - s.increaseBalance(consolidation.TargetIndex, activeBalance) + s.decreaseBalance(consolidation.SourceIndex, sourceEffectiveBalance) + s.increaseBalance(consolidation.TargetIndex, sourceEffectiveBalance) + nextPendingConsolidation++ } - s.PendingConsolidations = s.PendingConsolidations[processedCount:] + s.PendingConsolidations = s.PendingConsolidations[nextPendingConsolidation:] } diff --git a/indexer/beacon/statetransition/registry.go b/indexer/beacon/statetransition/registry.go index 9544bd634..5f5e618e7 100644 --- a/indexer/beacon/statetransition/registry.go +++ b/indexer/beacon/statetransition/registry.go @@ -1,65 +1,36 @@ package statetransition import ( - "sort" - "github.com/attestantio/go-eth2-client/spec/phase0" ) // processRegistryUpdates implements the Electra+ version of process_registry_updates. +// Single loop with if/elif/elif chain — Electra removed activation churn here and +// moved it to process_pending_deposits, so every eligible validator activates this +// epoch (no churn limit, no sorting). +// // Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_registry_updates func processRegistryUpdates(s *stateAccessor) error { currentEpoch := s.currentEpoch() - activationExitChurnLimit := s.getActivationExitChurnLimit() + activationEpoch := computeActivationExitEpoch(currentEpoch, s.specs) - // Process ejections for i, v := range s.Validators { - if isActiveValidator(v, currentEpoch) && - v.EffectiveBalance <= phase0.Gwei(s.specs.EjectionBalance) { - initiateValidatorExit(s, phase0.ValidatorIndex(i)) - } - } - - // Set activation eligibility - for _, v := range s.Validators { - if isEligibleForActivationQueue(v, s.specs) { + switch { + case isEligibleForActivationQueue(v, s.specs): v.ActivationEligibilityEpoch = currentEpoch + 1 + case isActiveValidator(v, currentEpoch) && v.EffectiveBalance <= phase0.Gwei(s.specs.EjectionBalance): + initiateValidatorExit(s, phase0.ValidatorIndex(i)) + case isEligibleForActivation(v, s.FinalizedCheckpoint.Epoch): + v.ActivationEpoch = activationEpoch } } - // Dequeue validators for activation (Electra+: balance-based churn) - activationQueue := make([]phase0.ValidatorIndex, 0) - for i, v := range s.Validators { - if isEligibleForActivation(v, s.FinalizedCheckpoint.Epoch) { - activationQueue = append(activationQueue, phase0.ValidatorIndex(i)) - } - } - - // Sort by activation eligibility epoch, then by index - sort.Slice(activationQueue, func(i, j int) bool { - vi := s.Validators[activationQueue[i]] - vj := s.Validators[activationQueue[j]] - if vi.ActivationEligibilityEpoch != vj.ActivationEligibilityEpoch { - return vi.ActivationEligibilityEpoch < vj.ActivationEligibilityEpoch - } - return activationQueue[i] < activationQueue[j] - }) - - // Activate validators up to the churn limit - activatedBalance := phase0.Gwei(0) - for _, idx := range activationQueue { - v := s.Validators[idx] - if activatedBalance+s.getMaxEffectiveBalance(v) > activationExitChurnLimit { - break - } - activatedBalance += s.getMaxEffectiveBalance(v) - v.ActivationEpoch = computeActivationExitEpoch(currentEpoch, s.specs) - } - return nil } -// initiateValidatorExit queues a validator for exit. +// initiateValidatorExit queues a validator for exit, computing the exit epoch +// via compute_exit_epoch_and_update_churn (which handles multi-epoch overflow). +// // Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-initiate_validator_exit func initiateValidatorExit(s *stateAccessor, index phase0.ValidatorIndex) { v := s.Validators[index] @@ -67,21 +38,56 @@ func initiateValidatorExit(s *stateAccessor, index phase0.ValidatorIndex) { return // already exiting } - exitQueueEpoch := computeActivationExitEpoch(s.currentEpoch(), s.specs) - if s.EarliestExitEpoch > exitQueueEpoch { - exitQueueEpoch = s.EarliestExitEpoch + // Spec uses validator.effective_balance, NOT max effective balance. + exitQueueEpoch := computeExitEpochAndUpdateChurn(s, v.EffectiveBalance) + + v.ExitEpoch = exitQueueEpoch + v.WithdrawableEpoch = exitQueueEpoch + phase0.Epoch(s.specs.MinValidatorWithdrawbilityDelay) +} + +// computeExitEpochAndUpdateChurn returns the earliest epoch at which an exit of +// the given balance can be processed, while updating state.earliest_exit_epoch +// and state.exit_balance_to_consume in place. Handles multi-epoch overflow. +// +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-compute_exit_epoch_and_update_churn +func computeExitEpochAndUpdateChurn(s *stateAccessor, exitBalance phase0.Gwei) phase0.Epoch { + earliestExitEpoch := computeActivationExitEpoch(s.currentEpoch(), s.specs) + if s.EarliestExitEpoch > earliestExitEpoch { + earliestExitEpoch = s.EarliestExitEpoch } + perEpochChurn := s.getActivationExitChurnLimit() - // Consume exit churn - exitBalance := s.getMaxEffectiveBalance(v) - if s.ExitBalanceToConsume < exitBalance { - // Not enough churn left, push to next epoch - s.ExitBalanceToConsume += s.getActivationExitChurnLimit() - exitQueueEpoch++ + var exitBalanceToConsume phase0.Gwei + if s.EarliestExitEpoch < earliestExitEpoch { + // New epoch for exits — refill the budget. + exitBalanceToConsume = perEpochChurn + } else { + exitBalanceToConsume = s.ExitBalanceToConsume } - s.ExitBalanceToConsume -= exitBalance - s.EarliestExitEpoch = exitQueueEpoch - v.ExitEpoch = exitQueueEpoch - v.WithdrawableEpoch = exitQueueEpoch + phase0.Epoch(s.specs.MinValidatorWithdrawbilityDelay) + // If exit doesn't fit, push it forward by enough epochs to fit the balance. + if exitBalance > exitBalanceToConsume { + balanceToProcess := exitBalance - exitBalanceToConsume + additionalEpochs := (balanceToProcess-1)/perEpochChurn + 1 + earliestExitEpoch += phase0.Epoch(additionalEpochs) + exitBalanceToConsume += phase0.Gwei(additionalEpochs) * perEpochChurn + } + + s.ExitBalanceToConsume = exitBalanceToConsume - exitBalance + s.EarliestExitEpoch = earliestExitEpoch + return s.EarliestExitEpoch +} + +// getPendingBalanceToWithdraw returns the sum of pending partial withdrawal +// amounts for the given validator. +// +// New in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-get_pending_balance_to_withdraw +func getPendingBalanceToWithdraw(s *stateAccessor, validatorIndex phase0.ValidatorIndex) phase0.Gwei { + total := phase0.Gwei(0) + for _, w := range s.PendingPartialWithdrawals { + if w.ValidatorIndex == validatorIndex { + total += w.Amount + } + } + return total } diff --git a/indexer/beacon/statetransition/resets.go b/indexer/beacon/statetransition/resets.go index 74578497b..132d13507 100644 --- a/indexer/beacon/statetransition/resets.go +++ b/indexer/beacon/statetransition/resets.go @@ -3,7 +3,6 @@ package statetransition import ( "github.com/attestantio/go-eth2-client/spec/altair" "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/phase0" ) // processEth1DataReset resets the ETH1 data votes at the start of a new voting period. @@ -60,57 +59,3 @@ func processHistoricalSummariesUpdate(s *stateAccessor) { StateSummaryRoot: stateSummary, }) } - -// processSyncCommitteeUpdates rotates the sync committee at period boundaries. -// New in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#sync-committee-updates -func processSyncCommitteeUpdates(s *stateAccessor) { - nextEpoch := s.currentEpoch() + 1 - if s.specs.EpochsPerSyncCommitteePeriod == 0 { - return - } - if uint64(nextEpoch)%s.specs.EpochsPerSyncCommitteePeriod != 0 { - return - } - - s.CurrentSyncCommittee = s.NextSyncCommittee - s.NextSyncCommittee = computeNextSyncCommittee(s) -} - -// computeNextSyncCommittee computes the next sync committee. -// https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#get_next_sync_committee -func computeNextSyncCommittee(s *stateAccessor) *altair.SyncCommittee { - indices := s.getActiveValidatorIndices(s.currentEpoch() + 1) - if len(indices) == 0 { - return s.NextSyncCommittee // fallback: keep current - } - - epoch := s.currentEpoch() + 1 - seed := getSeed(s, epoch, phase0.DomainType(s.specs.DomainSyncCommittee)) - - syncCommitteeSize := s.specs.SyncCommitteeSize - committee := make([]phase0.ValidatorIndex, 0, syncCommitteeSize) - pubkeys := make([]phase0.BLSPubKey, 0, syncCommitteeSize) - - i := uint64(0) - for uint64(len(committee)) < syncCommitteeSize { - shuffledIndex := computeShuffledIndex(i%uint64(len(indices)), uint64(len(indices)), seed, s.specs) - candidateIndex := indices[shuffledIndex] - - randomByte := getRandomByte(seed, i/32, i%32) - effectiveBalance := s.Validators[candidateIndex].EffectiveBalance - maxEB := s.getMaxEffectiveBalance(s.Validators[candidateIndex]) - - if effectiveBalance*255 >= maxEB*phase0.Gwei(randomByte) { - committee = append(committee, candidateIndex) - pubkeys = append(pubkeys, s.Validators[candidateIndex].PublicKey) - } - i++ - } - - // Compute aggregate pubkey placeholder (we don't actually need it for the explorer, - // but the struct requires it). Use empty value. - return &altair.SyncCommittee{ - Pubkeys: pubkeys, - AggregatePubkey: phase0.BLSPubKey{}, - } -} diff --git a/indexer/beacon/statetransition/rewards.go b/indexer/beacon/statetransition/rewards.go index ff4dfc67e..06e3d7c31 100644 --- a/indexer/beacon/statetransition/rewards.go +++ b/indexer/beacon/statetransition/rewards.go @@ -5,10 +5,12 @@ import ( ) // processInactivityUpdates implements process_inactivity_updates (Altair+). +// Skips the genesis epoch — score updates are based on the previous epoch's +// participation, which doesn't exist at epoch 0. // https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#inactivity-scores func processInactivityUpdates(s *stateAccessor) error { currentEpoch := s.currentEpoch() - if currentEpoch <= 1 { + if currentEpoch == 0 { return nil } @@ -21,36 +23,30 @@ func processInactivityUpdates(s *stateAccessor) error { targetParticipants[idx] = true } + // Iterate over eligible validator indices: active in previous epoch OR (slashed and not yet withdrawable) for i, v := range s.Validators { - if !isActiveValidator(v, previousEpoch) || v.Slashed { + if !isActiveValidator(v, previousEpoch) && !(v.Slashed && previousEpoch+1 < v.WithdrawableEpoch) { continue } idx := phase0.ValidatorIndex(i) if targetParticipants[idx] { - // Decrease inactivity score - if s.InactivityScores[i] > 0 { - decrease := s.specs.InactivityScoreRecoveryRate - if s.InactivityScores[i] < decrease { - s.InactivityScores[i] = 0 - } else { - s.InactivityScores[i] -= decrease - } + // Decrease inactivity score by min(1, score) + if s.InactivityScores[i] >= 1 { + s.InactivityScores[i] -= 1 } } else { - // Increase inactivity score + // Increase inactivity score by INACTIVITY_SCORE_BIAS s.InactivityScores[i] += s.specs.InactivityScoreBias } if !isInactivityLeak { - // Not in inactivity leak: decrease score faster - if s.InactivityScores[i] > 0 { - decrease := uint64(1) - if s.InactivityScores[i] < decrease { - s.InactivityScores[i] = 0 - } else { - s.InactivityScores[i] -= decrease - } + // Not in inactivity leak: decrease score by min(INACTIVITY_SCORE_RECOVERY_RATE, score) + recovery := s.specs.InactivityScoreRecoveryRate + if s.InactivityScores[i] >= recovery { + s.InactivityScores[i] -= recovery + } else { + s.InactivityScores[i] = 0 } } } @@ -59,10 +55,11 @@ func processInactivityUpdates(s *stateAccessor) error { } // processRewardsAndPenalties implements the Altair+ version of process_rewards_and_penalties. +// Skips the genesis epoch — rewards are for work done in the previous epoch. // Modified in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#modified-get_flag_index_deltas func processRewardsAndPenalties(s *stateAccessor) error { currentEpoch := s.currentEpoch() - if currentEpoch <= 1 { + if currentEpoch == 0 { return nil } @@ -70,12 +67,14 @@ func processRewardsAndPenalties(s *stateAccessor) error { totalActiveBalance := s.getTotalActiveBalance() isInactivityLeak := isInInactivityLeak(s) - // Precompute participating balances and sets for each flag + // Precompute participating increments for each flag (matching spec: get_flag_index_deltas) type flagData struct { - participatingBalance phase0.Gwei - participants map[phase0.ValidatorIndex]bool + participatingIncrements uint64 + participants map[phase0.ValidatorIndex]bool } + activeIncrements := uint64(totalActiveBalance) / s.specs.EffectiveBalanceIncrement + flags := make([]flagData, ParticipationFlagCount) for fi := 0; fi < ParticipationFlagCount; fi++ { indices := s.getUnslashedParticipatingIndices(fi, previousEpoch) @@ -88,41 +87,46 @@ func processRewardsAndPenalties(s *stateAccessor) error { if balance < phase0.Gwei(s.specs.EffectiveBalanceIncrement) { balance = phase0.Gwei(s.specs.EffectiveBalanceIncrement) } - flags[fi] = flagData{participatingBalance: balance, participants: pMap} + flags[fi] = flagData{ + participatingIncrements: uint64(balance) / s.specs.EffectiveBalanceIncrement, + participants: pMap, + } } for i, v := range s.Validators { + // is_eligible_validator: active in previous epoch OR (slashed and not yet withdrawable) if !isActiveValidator(v, previousEpoch) && !(v.Slashed && previousEpoch+1 < v.WithdrawableEpoch) { continue } idx := phase0.ValidatorIndex(i) - baseReward := s.getBaseReward(idx) + baseReward := uint64(s.getBaseReward(idx)) for fi := 0; fi < ParticipationFlagCount; fi++ { weight := ParticipationFlagWeights[fi] if flags[fi].participants[idx] && !v.Slashed { if !isInactivityLeak { - // Reward - rewardNumerator := baseReward * phase0.Gwei(weight) * flags[fi].participatingBalance - reward := rewardNumerator / (totalActiveBalance * WeightDenominator) - s.increaseBalance(idx, reward) + // Reward (spec: rewards[index] += base_reward * weight * participating_increments / (active_increments * WEIGHT_DENOMINATOR)) + rewardNumerator := baseReward * weight * flags[fi].participatingIncrements + reward := rewardNumerator / (activeIncrements * WeightDenominator) + s.increaseBalance(idx, phase0.Gwei(reward)) } - } else { - // Penalty - penalty := baseReward * phase0.Gwei(weight) / WeightDenominator - s.decreaseBalance(idx, penalty) + } else if fi != TimelyHeadFlagIndex { + // Penalty (spec: skip TIMELY_HEAD_FLAG_INDEX for penalties) + penalty := baseReward * weight / WeightDenominator + s.decreaseBalance(idx, phase0.Gwei(penalty)) } } - // Inactivity penalty (additional penalty for validators not participating in target) + // Inactivity penalty (spec: get_inactivity_penalty_deltas) + // penalty = effective_balance * inactivity_score / (INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_BELLATRIX) if !flags[TimelyTargetFlagIndex].participants[idx] || v.Slashed { - penaltyNumerator := v.EffectiveBalance * phase0.Gwei(s.InactivityScores[i]) - penaltyDenominator := phase0.Gwei(s.specs.InactivityScoreRecoveryRate * s.getInactivityPenaltyQuotient()) + penaltyNumerator := uint64(v.EffectiveBalance) * s.InactivityScores[i] + penaltyDenominator := s.specs.InactivityScoreBias * s.getInactivityPenaltyQuotient() if penaltyDenominator > 0 { penalty := penaltyNumerator / penaltyDenominator - s.decreaseBalance(idx, penalty) + s.decreaseBalance(idx, phase0.Gwei(penalty)) } } } diff --git a/indexer/beacon/statetransition/slashings.go b/indexer/beacon/statetransition/slashings.go index b60d6d8c9..2c2f5f9d1 100644 --- a/indexer/beacon/statetransition/slashings.go +++ b/indexer/beacon/statetransition/slashings.go @@ -5,8 +5,8 @@ import ( ) // processSlashings implements the Electra+ version of process_slashings. -// https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#slashings -// Modified in Altair: https://github.com/ethereum/consensus-specs/blob/master/specs/altair/beacon-chain.md#modified-process_slashings +// +// Modified in Electra: https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-process_slashings func processSlashings(s *stateAccessor) error { currentEpoch := s.currentEpoch() totalBalance := s.getTotalActiveBalance() @@ -16,35 +16,33 @@ func processSlashings(s *stateAccessor) error { totalSlashings += slashing } - // Electra+: PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3 - proportionalSlashingMultiplier := s.specs.ProportionalSlashingMultiplierBellatrix - if proportionalSlashingMultiplier == 0 { - proportionalSlashingMultiplier = s.specs.ProportionalSlashingMultiplier - } - - adjustedTotalSlashingBalance := totalSlashings * phase0.Gwei(proportionalSlashingMultiplier) + adjustedTotalSlashingBalance := totalSlashings * phase0.Gwei(s.specs.ProportionalSlashingMultiplierBellatrix) if adjustedTotalSlashingBalance > totalBalance { adjustedTotalSlashingBalance = totalBalance } + // Spec computes penalty_per_effective_balance_increment ONCE outside the loop: + // penalty_per_effective_balance_increment = adjusted // (total // increment) + // then per validator: + // penalty = penalty_per_effective_balance_increment * (effective_balance // increment) + // Doing the divisions in a different order loses precision for small slashings. + increment := phase0.Gwei(s.specs.EffectiveBalanceIncrement) + if increment == 0 || totalBalance < increment { + return nil + } + penaltyPerIncrement := adjustedTotalSlashingBalance / (totalBalance / increment) + + halfSlashingsVector := phase0.Epoch(s.specs.EpochsPerSlashingVector / 2) for i, v := range s.Validators { if !v.Slashed { continue } - - withdrawableEpoch := v.WithdrawableEpoch - halfSlashingsVector := phase0.Epoch(s.specs.EpochsPerSlashingVector / 2) - - if currentEpoch+halfSlashingsVector != withdrawableEpoch { + if currentEpoch+halfSlashingsVector != v.WithdrawableEpoch { continue } - // Electra+: use per-validator max effective balance - effectiveBalance := v.EffectiveBalance - increment := phase0.Gwei(s.specs.EffectiveBalanceIncrement) - penaltyNumerator := effectiveBalance / increment * adjustedTotalSlashingBalance - penalty := penaltyNumerator / totalBalance * increment - + effectiveBalanceIncrements := v.EffectiveBalance / increment + penalty := penaltyPerIncrement * effectiveBalanceIncrements s.decreaseBalance(phase0.ValidatorIndex(i), penalty) } diff --git a/indexer/beacon/statetransition/state.go b/indexer/beacon/statetransition/state.go index bdb432abd..338106855 100644 --- a/indexer/beacon/statetransition/state.go +++ b/indexer/beacon/statetransition/state.go @@ -7,6 +7,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/altair" "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" @@ -39,6 +40,7 @@ type stateAccessor struct { BlockRoots []phase0.Root StateRoots []phase0.Root HistoricalSummaries []*capella.HistoricalSummary + Eth1DepositIndex uint64 DepositRequestsStartIndex uint64 DepositBalanceToConsume phase0.Gwei ExitBalanceToConsume phase0.Gwei @@ -50,19 +52,80 @@ type stateAccessor struct { PendingConsolidations []*electra.PendingConsolidation ProposerLookahead []phase0.ValidatorIndex - // Gloas-only fields (nil for Fulu) - BuilderPendingPayments []*gloas.BuilderPendingPayment - BuilderPendingWithdrawals []*gloas.BuilderPendingWithdrawal + // Fields accessed via rawState that should be unified in the accessor + LatestBlockHeader *phase0.BeaconBlockHeader + ETH1Data *phase0.ETH1Data + NextWithdrawalIndex capella.WithdrawalIndex + NextWithdrawalValidatorIndex phase0.ValidatorIndex + + // Fulu-only fields (nil for Gloas) + LatestExecutionPayloadHeader *deneb.ExecutionPayloadHeader + + // Gloas-only fields (nil/zero for Fulu) + BuilderPendingPayments []*gloas.BuilderPendingPayment + BuilderPendingWithdrawals []*gloas.BuilderPendingWithdrawal + Builders []*gloas.Builder + NextWithdrawalBuilderIndex gloas.BuilderIndex + LatestExecutionPayloadBid *gloas.ExecutionPayloadBid + LatestBlockHash phase0.Hash32 + ExecutionPayloadAvailability []byte + PayloadExpectedWithdrawals []*capella.Withdrawal + PTCWindow [][]phase0.ValidatorIndex + + // Caches (lazily populated, not written back). + // Shared via StateTransition to persist across multiple ApplyBlock calls. + caches *stateTransitionCaches // Back-references for writing mutated slices/values back to the underlying state. rawState *spec.VersionedBeaconState } +// stateTransitionCaches holds lazily-populated caches that persist across +// multiple ApplyBlock / epoch transition calls on the same state. +// +// Caches that depend on the active validator set (totalActiveBal, +// baseRewardPerIncr, activeIndices) are keyed by epoch and auto-invalidate +// when crossing an epoch boundary. Effective-balance-dependent caches must +// also be cleared explicitly via invalidateBalanceCaches after the epoch +// transition runs processEffectiveBalanceUpdates / processRegistryUpdates / +// processPendingDeposits, since those mutate effective balances within an +// epoch without changing the cache key. +type stateTransitionCaches struct { + pubkeyCache map[phase0.BLSPubKey]phase0.ValidatorIndex + activeIndicesEpoch phase0.Epoch + activeIndices []phase0.ValidatorIndex + totalActiveBalEpoch phase0.Epoch + totalActiveBalCache *phase0.Gwei + baseRewardPerIncrEpoch phase0.Epoch + baseRewardPerIncrCache *phase0.Gwei + committeeCache *committeeCache +} + +func newStateTransitionCaches() *stateTransitionCaches { + return &stateTransitionCaches{ + committeeCache: newCommitteeCache(), + } +} + +// invalidateBalanceCaches clears caches that depend on validator effective +// balances (must be called after processEffectiveBalanceUpdates). +func (c *stateTransitionCaches) invalidateBalanceCaches() { + c.totalActiveBalCache = nil + c.baseRewardPerIncrCache = nil + c.activeIndices = nil + c.committeeCache = newCommitteeCache() +} + func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSpec) (*stateAccessor, error) { + return newStateAccessorWithCaches(state, specs, newStateTransitionCaches()) +} + +func newStateAccessorWithCaches(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, caches *stateTransitionCaches) (*stateAccessor, error) { s := &stateAccessor{ version: state.Version, specs: specs, rawState: state, + caches: caches, } switch state.Version { @@ -89,6 +152,7 @@ func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSp s.BlockRoots = f.BlockRoots s.StateRoots = f.StateRoots s.HistoricalSummaries = f.HistoricalSummaries + s.Eth1DepositIndex = f.ETH1DepositIndex s.DepositRequestsStartIndex = f.DepositRequestsStartIndex s.DepositBalanceToConsume = f.DepositBalanceToConsume s.ExitBalanceToConsume = f.ExitBalanceToConsume @@ -99,6 +163,11 @@ func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSp s.PendingPartialWithdrawals = f.PendingPartialWithdrawals s.PendingConsolidations = f.PendingConsolidations s.ProposerLookahead = f.ProposerLookahead + s.LatestBlockHeader = f.LatestBlockHeader + s.ETH1Data = f.ETH1Data + s.NextWithdrawalIndex = f.NextWithdrawalIndex + s.NextWithdrawalValidatorIndex = f.NextWithdrawalValidatorIndex + s.LatestExecutionPayloadHeader = f.LatestExecutionPayloadHeader case spec.DataVersionGloas: if state.Gloas == nil { return nil, fmt.Errorf("nil gloas state") @@ -122,6 +191,7 @@ func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSp s.BlockRoots = g.BlockRoots s.StateRoots = g.StateRoots s.HistoricalSummaries = g.HistoricalSummaries + s.Eth1DepositIndex = g.ETH1DepositIndex s.DepositRequestsStartIndex = g.DepositRequestsStartIndex s.DepositBalanceToConsume = g.DepositBalanceToConsume s.ExitBalanceToConsume = g.ExitBalanceToConsume @@ -134,6 +204,17 @@ func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSp s.ProposerLookahead = g.ProposerLookahead s.BuilderPendingPayments = g.BuilderPendingPayments s.BuilderPendingWithdrawals = g.BuilderPendingWithdrawals + s.LatestBlockHeader = g.LatestBlockHeader + s.ETH1Data = g.ETH1Data + s.NextWithdrawalIndex = g.NextWithdrawalIndex + s.NextWithdrawalValidatorIndex = g.NextWithdrawalValidatorIndex + s.Builders = g.Builders + s.NextWithdrawalBuilderIndex = g.NextWithdrawalBuilderIndex + s.LatestExecutionPayloadBid = g.LatestExecutionPayloadBid + s.LatestBlockHash = g.LatestBlockHash + s.ExecutionPayloadAvailability = g.ExecutionPayloadAvailability + s.PayloadExpectedWithdrawals = g.PayloadExpectedWithdrawals + s.PTCWindow = g.PTCWindow default: return nil, fmt.Errorf("unsupported state version: %v", state.Version) } @@ -157,9 +238,18 @@ func (s *stateAccessor) writeBack() { f.PreviousEpochParticipation = s.PreviousEpochParticipation f.CurrentEpochParticipation = s.CurrentEpochParticipation f.JustificationBits = s.JustificationBits + f.PreviousJustifiedCheckpoint = s.PreviousJustifiedCheckpoint + f.CurrentJustifiedCheckpoint = s.CurrentJustifiedCheckpoint + f.FinalizedCheckpoint = s.FinalizedCheckpoint f.InactivityScores = s.InactivityScores + f.CurrentSyncCommittee = s.CurrentSyncCommittee + f.NextSyncCommittee = s.NextSyncCommittee f.ETH1DataVotes = s.ETH1DataVotes + f.BlockRoots = s.BlockRoots + f.StateRoots = s.StateRoots f.HistoricalSummaries = s.HistoricalSummaries + f.ETH1DepositIndex = s.Eth1DepositIndex + f.DepositRequestsStartIndex = s.DepositRequestsStartIndex f.DepositBalanceToConsume = s.DepositBalanceToConsume f.ExitBalanceToConsume = s.ExitBalanceToConsume f.EarliestExitEpoch = s.EarliestExitEpoch @@ -169,6 +259,11 @@ func (s *stateAccessor) writeBack() { f.PendingPartialWithdrawals = s.PendingPartialWithdrawals f.PendingConsolidations = s.PendingConsolidations f.ProposerLookahead = s.ProposerLookahead + f.LatestBlockHeader = s.LatestBlockHeader + f.ETH1Data = s.ETH1Data + f.NextWithdrawalIndex = s.NextWithdrawalIndex + f.NextWithdrawalValidatorIndex = s.NextWithdrawalValidatorIndex + f.LatestExecutionPayloadHeader = s.LatestExecutionPayloadHeader case spec.DataVersionGloas: g := s.rawState.Gloas g.Slot = s.Slot @@ -179,9 +274,18 @@ func (s *stateAccessor) writeBack() { g.PreviousEpochParticipation = s.PreviousEpochParticipation g.CurrentEpochParticipation = s.CurrentEpochParticipation g.JustificationBits = s.JustificationBits + g.PreviousJustifiedCheckpoint = s.PreviousJustifiedCheckpoint + g.CurrentJustifiedCheckpoint = s.CurrentJustifiedCheckpoint + g.FinalizedCheckpoint = s.FinalizedCheckpoint g.InactivityScores = s.InactivityScores + g.CurrentSyncCommittee = s.CurrentSyncCommittee + g.NextSyncCommittee = s.NextSyncCommittee g.ETH1DataVotes = s.ETH1DataVotes + g.BlockRoots = s.BlockRoots + g.StateRoots = s.StateRoots g.HistoricalSummaries = s.HistoricalSummaries + g.ETH1DepositIndex = s.Eth1DepositIndex + g.DepositRequestsStartIndex = s.DepositRequestsStartIndex g.DepositBalanceToConsume = s.DepositBalanceToConsume g.ExitBalanceToConsume = s.ExitBalanceToConsume g.EarliestExitEpoch = s.EarliestExitEpoch @@ -193,7 +297,78 @@ func (s *stateAccessor) writeBack() { g.ProposerLookahead = s.ProposerLookahead g.BuilderPendingPayments = s.BuilderPendingPayments g.BuilderPendingWithdrawals = s.BuilderPendingWithdrawals + g.LatestBlockHeader = s.LatestBlockHeader + g.ETH1Data = s.ETH1Data + g.NextWithdrawalIndex = s.NextWithdrawalIndex + g.NextWithdrawalValidatorIndex = s.NextWithdrawalValidatorIndex + g.Builders = s.Builders + g.NextWithdrawalBuilderIndex = s.NextWithdrawalBuilderIndex + g.LatestExecutionPayloadBid = s.LatestExecutionPayloadBid + g.LatestBlockHash = s.LatestBlockHash + g.ExecutionPayloadAvailability = s.ExecutionPayloadAvailability + g.PayloadExpectedWithdrawals = s.PayloadExpectedWithdrawals + g.PTCWindow = s.PTCWindow + } +} + +// computeStateHTR computes the hash tree root of the underlying state. +// Must call writeBack() first to ensure all accessor fields are synced. +func (s *stateAccessor) computeStateHTR() (phase0.Root, error) { + s.writeBack() + switch s.version { + case spec.DataVersionFulu: + return s.rawState.Fulu.HashTreeRoot() + case spec.DataVersionGloas: + return s.rawState.Gloas.HashTreeRoot() + default: + return phase0.Root{}, fmt.Errorf("unsupported version: %v", s.version) + } +} + +// computeLatestBlockHeaderHTR computes hash_tree_root(state.latest_block_header). +func (s *stateAccessor) computeLatestBlockHeaderHTR() (phase0.Root, error) { + if s.LatestBlockHeader == nil { + return phase0.Root{}, nil + } + return s.LatestBlockHeader.HashTreeRoot() +} + +// clearNextSlotAvailabilityBit clears the execution payload availability bit +// for the next slot (Gloas-specific process_slot step). +func (s *stateAccessor) clearNextSlotAvailabilityBit() { + if s.version < spec.DataVersionGloas || len(s.ExecutionPayloadAvailability) == 0 { + return + } + nextIdx := (uint64(s.Slot) + 1) % s.specs.SlotsPerHistoricalRoot + byteIdx := nextIdx / 8 + bitIdx := nextIdx % 8 + if byteIdx < uint64(len(s.ExecutionPayloadAvailability)) { + s.ExecutionPayloadAvailability[byteIdx] &^= 1 << bitIdx + } +} + +// setAvailabilityBit sets the execution payload availability bit for the current slot. +func (s *stateAccessor) setAvailabilityBit() { + if s.version < spec.DataVersionGloas || len(s.ExecutionPayloadAvailability) == 0 { + return + } + bitfieldLen := uint64(len(s.ExecutionPayloadAvailability)) * 8 + idx := uint64(s.Slot) % bitfieldLen + s.ExecutionPayloadAvailability[idx/8] |= 1 << (idx % 8) +} + +// getAvailabilityBit returns the execution payload availability bit for a given slot. +func (s *stateAccessor) getAvailabilityBit(slot phase0.Slot) bool { + if s.version < spec.DataVersionGloas || len(s.ExecutionPayloadAvailability) == 0 { + return false + } + idx := uint64(slot) % s.specs.SlotsPerHistoricalRoot + byteIdx := idx / 8 + bitIdx := idx % 8 + if byteIdx >= uint64(len(s.ExecutionPayloadAvailability)) { + return false } + return s.ExecutionPayloadAvailability[byteIdx]&(1<= targetSlot { - break + // Apply epoch transition at epoch boundary (last slot of epoch). + if (uint64(s.Slot)+1)%slotsPerEpoch == 0 { + if err := processEpochInternal(s, info); err != nil { + return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) + } } - s.Slot = nextBoundary - if err := processEpochInternal(s, info); err != nil { - return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) - } - - s.Slot++ // cross into next epoch + s.Slot++ } - s.Slot = targetSlot s.writeBack() return nil } -// isPrePayloadState checks whether a Gloas state is pre-payload +// processSlotBlockRootCaching implements the essential parts of process_slot: +// computes the state root, fills latest_block_header.state_root if zero, +// then caches the block root. The state root must be computed first because +// the block root depends on the header's state_root field. +func processSlotBlockRootCaching(s *stateAccessor) { + stateRoot, err := s.computeStateHTR() + if err != nil { + return + } + + idx := uint64(s.Slot) % s.specs.SlotsPerHistoricalRoot + s.StateRoots[idx] = stateRoot + + // Fill latest_block_header.state_root if zero (set after each processBlockHeader). + if s.LatestBlockHeader != nil && s.LatestBlockHeader.StateRoot == (phase0.Root{}) { + s.LatestBlockHeader.StateRoot = stateRoot + } + + blockRoot, err := s.computeLatestBlockHeaderHTR() + if err != nil { + return + } + + s.BlockRoots[idx] = blockRoot + + // Gloas: clear the next slot's execution payload availability bit. + s.clearNextSlotAvailabilityBit() +} + +// IsPrePayloadState checks whether a Gloas state is pre-payload // (the execution payload for the latest block has NOT been processed yet). -func isPrePayloadState(state *spec.VersionedBeaconState) bool { +func IsPrePayloadState(state *spec.VersionedBeaconState) bool { if state.Version < spec.DataVersionGloas || state.Gloas == nil { return false } @@ -149,54 +218,62 @@ func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.E return nil } - g := state.Gloas - slotsPerEpoch := specs.SlotsPerEpoch + s, err := newStateAccessor(state, specs) + if err != nil { + return fmt.Errorf("failed to create state accessor: %w", err) + } + + // Cache latest block header state root (spec: fill before payload processing). + // https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_execution_payload + if s.LatestBlockHeader != nil && s.LatestBlockHeader.StateRoot == (phase0.Root{}) { + stateRoot, htrErr := s.computeStateHTR() + if htrErr != nil { + return fmt.Errorf("failed to compute state root for header fill: %w", htrErr) + } + s.LatestBlockHeader.StateRoot = stateRoot + } - // Process deposit requests → convert to pending deposits. - // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-process_deposit_request + // Process execution requests (deposits, withdrawals, consolidations). if envelope.ExecutionRequests != nil { for _, deposit := range envelope.ExecutionRequests.Deposits { - g.PendingDeposits = append(g.PendingDeposits, &electra.PendingDeposit{ + s.PendingDeposits = append(s.PendingDeposits, &electra.PendingDeposit{ Pubkey: deposit.Pubkey, WithdrawalCredentials: deposit.WithdrawalCredentials, Amount: deposit.Amount, Signature: deposit.Signature, - Slot: g.Slot, + Slot: s.Slot, }) } - // Note: withdrawal requests and consolidation requests are NOT processed here - // since they require full validator lookup and exit queue logic. - // For our purpose (advancing to epoch boundary), the pending deposits - // are the critical part. Withdrawal/consolidation requests affect the - // state sim which replays them separately. + for _, withdrawal := range envelope.ExecutionRequests.Withdrawals { + processWithdrawalRequest(s, withdrawal) + } + + for _, consolidation := range envelope.ExecutionRequests.Consolidations { + processConsolidationRequest(s, consolidation) + } } // Queue the builder payment (direct withdrawal for delivered payload). - // The bid was recorded in builder_pending_payments during process_execution_payload_bid. - // Now that the payload is delivered, we move the payment to builder_pending_withdrawals - // and clear the pending payment entry. - paymentIdx := slotsPerEpoch + uint64(g.Slot)%slotsPerEpoch - if paymentIdx < uint64(len(g.BuilderPendingPayments)) { - payment := g.BuilderPendingPayments[paymentIdx] + slotsPerEpoch := specs.SlotsPerEpoch + paymentIdx := slotsPerEpoch + uint64(s.Slot)%slotsPerEpoch + if paymentIdx < uint64(len(s.BuilderPendingPayments)) { + payment := s.BuilderPendingPayments[paymentIdx] if payment != nil && payment.Withdrawal != nil && payment.Withdrawal.Amount > 0 { - g.BuilderPendingWithdrawals = append(g.BuilderPendingWithdrawals, payment.Withdrawal) + s.BuilderPendingWithdrawals = append(s.BuilderPendingWithdrawals, payment.Withdrawal) } - g.BuilderPendingPayments[paymentIdx] = &gloas.BuilderPendingPayment{} + s.BuilderPendingPayments[paymentIdx] = &gloas.BuilderPendingPayment{} } // Set execution payload availability bit. - bitfieldLen := uint64(len(g.ExecutionPayloadAvailability)) * 8 - if bitfieldLen > 0 { - idx := uint64(g.Slot) % bitfieldLen - g.ExecutionPayloadAvailability[idx/8] |= 1 << (idx % 8) - } + s.setAvailabilityBit() // Cache the execution payload block hash. if envelope.Payload != nil { - copy(g.LatestBlockHash[:], envelope.Payload.BlockHash[:]) + s.LatestBlockHash = envelope.Payload.BlockHash } + s.writeBack() return nil } @@ -243,6 +320,8 @@ func processEpochInternal(s *stateAccessor, info *TransitionInfo) error { } processEffectiveBalanceUpdates(s) + // Effective balances may have changed; clear caches that depend on them. + s.caches.invalidateBalanceCaches() processSlashingsReset(s) processRandaoMixesReset(s) processHistoricalSummariesUpdate(s) diff --git a/indexer/beacon/statetransition/withdrawals.go b/indexer/beacon/statetransition/withdrawals.go new file mode 100644 index 000000000..afb9ddd7a --- /dev/null +++ b/indexer/beacon/statetransition/withdrawals.go @@ -0,0 +1,267 @@ +package statetransition + +import ( + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// expectedWithdrawals holds the result of get_expected_withdrawals. +type expectedWithdrawals struct { + withdrawals []*capella.Withdrawal + processedBuilderWithdrawalsCount uint64 + processedPartialWithdrawalsCount uint64 + processedBuildersSweepCount uint64 + processedValidatorsSweepCount uint64 +} + +// processWithdrawals implements the Gloas version of process_withdrawals. +// Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_withdrawals +func processWithdrawals(s *stateAccessor) { + if !isParentBlockFull(s) { + return + } + + expected := getExpectedWithdrawals(s) + + // apply_withdrawals + for _, w := range expected.withdrawals { + if isBuilderIndex(w.ValidatorIndex) { + // Builder withdrawal: decrease builder balance + builderIdx := convertValidatorIndexToBuilderIndex(w.ValidatorIndex) + if int(builderIdx) < len(s.Builders) { + builder := s.Builders[builderIdx] + amount := phase0.Gwei(w.Amount) + if amount > builder.Balance { + amount = builder.Balance + } + builder.Balance -= amount + } + } else { + if int(w.ValidatorIndex) < len(s.Balances) { + s.decreaseBalance(w.ValidatorIndex, phase0.Gwei(w.Amount)) + } + } + } + + // update_next_withdrawal_index + if len(expected.withdrawals) > 0 { + lastIdx := expected.withdrawals[len(expected.withdrawals)-1].Index + s.NextWithdrawalIndex = lastIdx + 1 + } + + // update_payload_expected_withdrawals + s.PayloadExpectedWithdrawals = expected.withdrawals + + // update_builder_pending_withdrawals: remove processed entries from the front + if expected.processedBuilderWithdrawalsCount > 0 { + n := expected.processedBuilderWithdrawalsCount + if n > uint64(len(s.BuilderPendingWithdrawals)) { + n = uint64(len(s.BuilderPendingWithdrawals)) + } + s.BuilderPendingWithdrawals = s.BuilderPendingWithdrawals[n:] + } + + // update_pending_partial_withdrawals: remove processed entries from the front + if expected.processedPartialWithdrawalsCount > 0 { + n := expected.processedPartialWithdrawalsCount + if n > uint64(len(s.PendingPartialWithdrawals)) { + n = uint64(len(s.PendingPartialWithdrawals)) + } + s.PendingPartialWithdrawals = s.PendingPartialWithdrawals[n:] + } + + // update_next_withdrawal_builder_index + if expected.processedBuildersSweepCount > 0 && len(s.Builders) > 0 { + nextIdx := uint64(s.NextWithdrawalBuilderIndex) + expected.processedBuildersSweepCount + s.NextWithdrawalBuilderIndex = gloas.BuilderIndex(nextIdx % uint64(len(s.Builders))) + } + + // update_next_withdrawal_validator_index (Capella spec, unchanged in Gloas) + if uint64(len(expected.withdrawals)) == s.specs.MaxWithdrawalsPerPayload { + lastW := expected.withdrawals[len(expected.withdrawals)-1] + s.NextWithdrawalValidatorIndex = phase0.ValidatorIndex((uint64(lastW.ValidatorIndex) + 1) % uint64(len(s.Validators))) + } else { + s.NextWithdrawalValidatorIndex = phase0.ValidatorIndex((uint64(s.NextWithdrawalValidatorIndex) + s.specs.MaxValidatorsPerWithdrawalsSweep) % uint64(len(s.Validators))) + } +} + +// BuilderIndexFlag separates builder indices from validator indices. +const BuilderIndexFlag = uint64(1 << 40) + +func isBuilderIndex(idx phase0.ValidatorIndex) bool { + return uint64(idx)&BuilderIndexFlag != 0 +} + +func convertBuilderIndexToValidatorIndex(builderIdx gloas.BuilderIndex) phase0.ValidatorIndex { + return phase0.ValidatorIndex(uint64(builderIdx) | BuilderIndexFlag) +} + +func convertValidatorIndexToBuilderIndex(validatorIdx phase0.ValidatorIndex) gloas.BuilderIndex { + return gloas.BuilderIndex(uint64(validatorIdx) &^ BuilderIndexFlag) +} + +// getExpectedWithdrawals computes the expected withdrawals for the current slot. +// Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-get_expected_withdrawals +func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { + result := &expectedWithdrawals{} + nextIdx := s.NextWithdrawalIndex + maxWithdrawals := s.specs.MaxWithdrawalsPerPayload + // Builder/partial/builder-sweep withdrawals use MAX-1 limit + subLimit := maxWithdrawals - 1 + + // 1. Builder pending withdrawals + for _, bpw := range s.BuilderPendingWithdrawals { + if uint64(len(result.withdrawals)) >= subLimit { + break + } + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: convertBuilderIndexToValidatorIndex(bpw.BuilderIndex), + Address: bpw.FeeRecipient, + Amount: bpw.Amount, + }) + nextIdx++ + result.processedBuilderWithdrawalsCount++ + } + + // 2. Pending partial withdrawals + epoch := s.currentEpoch() + for _, pw := range s.PendingPartialWithdrawals { + if uint64(len(result.withdrawals)) >= subLimit { + break + } + if pw.WithdrawableEpoch > epoch { + break + } + result.processedPartialWithdrawalsCount++ + + validator := s.Validators[pw.ValidatorIndex] + if validator.ExitEpoch != FarFutureEpoch || !hasExecutionWithdrawalCredential(validator) { + continue + } + + balance := s.Balances[pw.ValidatorIndex] + minBalance := phase0.Gwei(s.specs.MinActivationBalance) + if balance <= minBalance { + continue + } + + withdrawableAmount := balance - minBalance + amount := phase0.Gwei(pw.Amount) + if withdrawableAmount < amount { + amount = withdrawableAmount + } + + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: pw.ValidatorIndex, + Address: getWithdrawalAddress(validator), + Amount: amount, + }) + nextIdx++ + } + + // 3. Builder sweep withdrawals (Gloas-specific) + if len(s.Builders) > 0 { + buildersLimit := uint64(len(s.Builders)) + if s.specs.MaxBuildersPerWithdrawalsSweep > 0 && s.specs.MaxBuildersPerWithdrawalsSweep < buildersLimit { + buildersLimit = s.specs.MaxBuildersPerWithdrawalsSweep + } + + builderIdx := s.NextWithdrawalBuilderIndex + for i := uint64(0); i < buildersLimit; i++ { + if uint64(len(result.withdrawals)) >= subLimit { + break + } + + builder := s.Builders[builderIdx] + if builder.WithdrawableEpoch <= epoch && builder.Balance > 0 { + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: convertBuilderIndexToValidatorIndex(builderIdx), + Address: builder.ExecutionAddress, + Amount: builder.Balance, + }) + nextIdx++ + } + + builderIdx = gloas.BuilderIndex((uint64(builderIdx) + 1) % uint64(len(s.Builders))) + result.processedBuildersSweepCount++ + } + } + + // 4. Validator sweep withdrawals (uses full MAX limit) + validatorCount := uint64(len(s.Validators)) + if validatorCount > 0 { + startIdx := uint64(s.NextWithdrawalValidatorIndex) + bound := s.specs.MaxValidatorsPerWithdrawalsSweep + if validatorCount < bound { + bound = validatorCount + } + + for i := uint64(0); i < bound && uint64(len(result.withdrawals)) < maxWithdrawals; i++ { + vidx := phase0.ValidatorIndex((startIdx + i) % validatorCount) + validator := s.Validators[vidx] + balance := s.Balances[vidx] + + result.processedValidatorsSweepCount++ + + if !hasExecutionWithdrawalCredential(validator) { + continue + } + + // Full withdrawal: exited and withdrawable + if validator.ExitEpoch != FarFutureEpoch && validator.WithdrawableEpoch <= epoch && balance > 0 { + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: vidx, + Address: getWithdrawalAddress(validator), + Amount: balance, + }) + nextIdx++ + continue + } + + // Partial (sweep) withdrawal: excess balance + maxEB := s.getMaxEffectiveBalance(validator) + if balance > maxEB { + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: vidx, + Address: getWithdrawalAddress(validator), + Amount: balance - maxEB, + }) + nextIdx++ + } + } + + } + + return result +} + +// getWithdrawalAddress extracts the withdrawal address from validator credentials. +func getWithdrawalAddress(v *phase0.Validator) [20]byte { + var addr [20]byte + if len(v.WithdrawalCredentials) >= 32 { + copy(addr[:], v.WithdrawalCredentials[12:32]) + } + return addr +} + +// isParentBlockFull checks if the parent block had an execution payload (Gloas). +// Spec: return state.latest_execution_payload_bid.block_hash == state.latest_block_hash +// https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#new-is_parent_block_full +func isParentBlockFull(s *stateAccessor) bool { + if s.version < spec.DataVersionGloas { + return true // Pre-Gloas: always full + } + + if s.LatestExecutionPayloadBid == nil { + return false + } + + return s.LatestExecutionPayloadBid.BlockHash == s.LatestBlockHash +} diff --git a/statecache/statecache.go b/statecache/statecache.go new file mode 100644 index 000000000..e12fcccec --- /dev/null +++ b/statecache/statecache.go @@ -0,0 +1,301 @@ +// Package statecache provides an optional file-system-backed cache for beacon states. +// States are stored as compressed SSZ files keyed by (dependentRoot, targetEpoch). +// The cache limits the number of stored states and re-initializes from the +// filesystem on restart (no in-memory index — just scans the directory). +package statecache + +import ( + "compress/gzip" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/fulu" + "github.com/attestantio/go-eth2-client/spec/gloas" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/dora/types" + + dynssz "github.com/pk910/dynamic-ssz" +) + +// StateCache manages cached beacon states on the local filesystem. +// It is safe for concurrent use. +type StateCache struct { + mu sync.Mutex + path string + maxStates uint + dynSsz *dynssz.DynSsz +} + +// New creates a new StateCache. Returns nil if the config disables caching. +// The directory is created if it doesn't exist. +func New(cfg *types.Config, dynSsz *dynssz.DynSsz) *StateCache { + scCfg := cfg.Indexer.StateCache + if !scCfg.Enabled || scCfg.Path == "" { + return nil + } + + maxStates := scCfg.MaxStates + if maxStates == 0 { + maxStates = 5 + } + + if err := os.MkdirAll(scCfg.Path, 0o750); err != nil { + return nil + } + + return &StateCache{ + path: scCfg.Path, + maxStates: maxStates, + dynSsz: dynSsz, + } +} + +// stateKey identifies a cached state by dependent root and target epoch. +type stateKey struct { + DependentRoot phase0.Root + TargetEpoch phase0.Epoch +} + +// filename returns the cache filename for a state key. +// Format: _.ssz.gz +func (k stateKey) filename() string { + return fmt.Sprintf("%d_%s.ssz.gz", k.TargetEpoch, hex.EncodeToString(k.DependentRoot[:])) +} + +// parseFilename parses a cache filename back into a stateKey. +// Returns false if the filename doesn't match the expected format. +func parseFilename(name string) (stateKey, bool) { + name = strings.TrimSuffix(name, ".ssz.gz") + parts := strings.SplitN(name, "_", 2) + if len(parts) != 2 { + return stateKey{}, false + } + + epoch, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return stateKey{}, false + } + + rootBytes, err := hex.DecodeString(parts[1]) + if err != nil || len(rootBytes) != 32 { + return stateKey{}, false + } + + var key stateKey + key.TargetEpoch = phase0.Epoch(epoch) + copy(key.DependentRoot[:], rootBytes) + return key, true +} + +// Check returns true if a cached state exists for the given key. +func (sc *StateCache) Check(dependentRoot phase0.Root, targetEpoch phase0.Epoch) bool { + if sc == nil { + return false + } + + key := stateKey{DependentRoot: dependentRoot, TargetEpoch: targetEpoch} + path := filepath.Join(sc.path, key.filename()) + _, err := os.Stat(path) + return err == nil +} + +// Load reads a cached state from disk. Returns nil if not found. +func (sc *StateCache) Load(dependentRoot phase0.Root, targetEpoch phase0.Epoch) *spec.VersionedBeaconState { + if sc == nil { + return nil + } + + key := stateKey{DependentRoot: dependentRoot, TargetEpoch: targetEpoch} + path := filepath.Join(sc.path, key.filename()) + + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return nil + } + defer gz.Close() + + sszData, err := io.ReadAll(gz) + if err != nil { + return nil + } + + // Read version marker (first byte) + if len(sszData) < 1 { + return nil + } + version := spec.DataVersion(sszData[0]) + sszData = sszData[1:] + + state, err := unmarshalState(sc.dynSsz, version, sszData) + if err != nil { + return nil + } + + return state +} + +// Store writes a state to disk and enforces the max states limit. +func (sc *StateCache) Store(dependentRoot phase0.Root, targetEpoch phase0.Epoch, state *spec.VersionedBeaconState) error { + if sc == nil { + return nil + } + + sc.mu.Lock() + defer sc.mu.Unlock() + + sszData, err := marshalState(sc.dynSsz, state) + if err != nil { + return fmt.Errorf("failed to marshal state: %w", err) + } + + // Prepend version marker + versioned := make([]byte, 1+len(sszData)) + versioned[0] = byte(state.Version) + copy(versioned[1:], sszData) + + key := stateKey{DependentRoot: dependentRoot, TargetEpoch: targetEpoch} + path := filepath.Join(sc.path, key.filename()) + + if err := os.MkdirAll(sc.path, 0o750); err != nil { + return fmt.Errorf("failed to ensure cache directory: %w", err) + } + + f, err := os.CreateTemp(sc.path, "state-*.tmp") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := f.Name() + + gz := gzip.NewWriter(f) + if _, err := gz.Write(versioned); err != nil { + gz.Close() + f.Close() + os.Remove(tmpPath) + return fmt.Errorf("failed to write compressed data: %w", err) + } + gz.Close() + f.Close() + + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("failed to rename temp file: %w", err) + } + + // Enforce max states limit + sc.evict() + + return nil +} + +// evict removes the oldest cached states to stay within the max limit. +// Must be called with sc.mu held. +func (sc *StateCache) evict() { + entries, err := os.ReadDir(sc.path) + if err != nil { + return + } + + type cachedEntry struct { + name string + modTime int64 + } + + var cached []cachedEntry + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".ssz.gz") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + cached = append(cached, cachedEntry{name: entry.Name(), modTime: info.ModTime().UnixNano()}) + } + + if uint(len(cached)) <= sc.maxStates { + return + } + + // Sort by modification time ascending (oldest first) + sort.Slice(cached, func(i, j int) bool { + return cached[i].modTime < cached[j].modTime + }) + + // Remove oldest until within limit + toRemove := uint(len(cached)) - sc.maxStates + for i := uint(0); i < toRemove; i++ { + os.Remove(filepath.Join(sc.path, cached[i].name)) + } +} + +// marshalState serializes a VersionedBeaconState to SSZ bytes. +func marshalState(dynSsz *dynssz.DynSsz, state *spec.VersionedBeaconState) ([]byte, error) { + switch state.Version { + case spec.DataVersionFulu: + if state.Fulu == nil { + return nil, fmt.Errorf("nil fulu state") + } + if dynSsz != nil { + return dynSsz.MarshalSSZ(state.Fulu) + } + return state.Fulu.MarshalSSZ() + case spec.DataVersionGloas: + if state.Gloas == nil { + return nil, fmt.Errorf("nil gloas state") + } + if dynSsz != nil { + return dynSsz.MarshalSSZ(state.Gloas) + } + return state.Gloas.MarshalSSZ() + default: + return nil, fmt.Errorf("unsupported state version: %v", state.Version) + } +} + +// unmarshalState deserializes SSZ bytes into a VersionedBeaconState. +func unmarshalState(dynSsz *dynssz.DynSsz, version spec.DataVersion, data []byte) (*spec.VersionedBeaconState, error) { + state := &spec.VersionedBeaconState{Version: version} + + switch version { + case spec.DataVersionFulu: + state.Fulu = new(fulu.BeaconState) + var err error + if dynSsz != nil { + err = dynSsz.UnmarshalSSZ(state.Fulu, data) + } else { + err = state.Fulu.UnmarshalSSZ(data) + } + if err != nil { + return nil, fmt.Errorf("failed to unmarshal fulu state: %w", err) + } + case spec.DataVersionGloas: + state.Gloas = new(gloas.BeaconState) + var err error + if dynSsz != nil { + err = dynSsz.UnmarshalSSZ(state.Gloas, data) + } else { + err = state.Gloas.UnmarshalSSZ(data) + } + if err != nil { + return nil, fmt.Errorf("failed to unmarshal gloas state: %w", err) + } + default: + return nil, fmt.Errorf("unsupported state version: %v", version) + } + + return state, nil +} diff --git a/types/config.go b/types/config.go index f011b5bcd..48c2d3cc3 100644 --- a/types/config.go +++ b/types/config.go @@ -129,6 +129,12 @@ type Config struct { PubkeyCachePath string `yaml:"pubkeyCachePath" envconfig:"INDEXER_PUBKEY_CACHE_PATH"` BadChainRoots []string `yaml:"badChainRoots" envconfig:"INDEXER_BAD_CHAIN_ROOTS"` + + StateCache struct { + Enabled bool `yaml:"enabled" envconfig:"INDEXER_STATE_CACHE_ENABLED"` + Path string `yaml:"path" envconfig:"INDEXER_STATE_CACHE_PATH"` + MaxStates uint `yaml:"maxStates" envconfig:"INDEXER_STATE_CACHE_MAX_STATES"` + } `yaml:"stateCache"` } `yaml:"indexer"` TxSignature struct { From 0b37393db843bafc2d01e469137434ebaa6a4900 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 10 Apr 2026 01:40:20 +0200 Subject: [PATCH 91/99] state transition fixes --- indexer/beacon/epochstate.go | 45 ++++-- indexer/beacon/statetransition/block.go | 9 +- indexer/beacon/statetransition/operations.go | 35 ++++- indexer/beacon/statetransition/state.go | 14 +- .../beacon/statetransition/statetransition.go | 31 +++- indexer/beacon/statetransition/withdrawals.go | 143 +++++++++++------- 6 files changed, 188 insertions(+), 89 deletions(-) diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index f6459474a..e384186b6 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -418,7 +418,7 @@ func (s *epochState) tryReplayFromParentState( replayStart := time.Now() var blockApplyTotal time.Duration - for _, blk := range epochBlocks { + for i, blk := range epochBlocks { beaconBlock := blk.GetBlock(ctx) if beaconBlock == nil { return nil @@ -450,25 +450,44 @@ func (s *epochState) tryReplayFromParentState( blk.Slot, gotStateRoot.String(), expectedStateRoot.String()) return nil } + // Default: post-block HTR is the correct pre-state HTR for the next block + // (correct for Fulu and for Gloas when no payload is applied). prevStateRoot = gotStateRoot blockApplyTotal += time.Since(blockStart) - // For Gloas: apply execution payload if available. Skip the LAST block's - // payload — the dep block's state root is the pre-payload root, so - // PrepareEpochPreState will handle the payload. - isLastBlock := (blk == epochBlocks[len(epochBlocks)-1]) + // For Gloas: apply execution payload if delivered AND accepted. + // A payload is "accepted" iff the next block in the canonical chain + // references it via bid.parent_block_hash; otherwise it was orphaned + // (the next block built on the parent payload instead) and must NOT + // be applied to the state. + // Skip the LAST block's payload — its acceptance is determined by the + // first block of the target epoch, which we don't have here. The + // PrepareEpochPreState call below handles it via the dep block payload. + isLastBlock := (i == len(epochBlocks)-1) if parentState.Version >= spec.DataVersionGloas && !isLastBlock { payload := blk.GetExecutionPayload(ctx) - if payload != nil && payload.Message != nil { - if err := st.ApplyExecutionPayload(parentState, payload); err != nil { - client.logger.Warnf("replay: ApplyExecutionPayload failed at slot %v: %v", blk.Slot, err) + if payload != nil && payload.Message != nil && payload.Message.Payload != nil { + nextBeaconBlock := epochBlocks[i+1].GetBlock(ctx) + if nextBeaconBlock == nil { return nil } - // Post-payload state HTR is recorded in the envelope itself. - prevStateRoot = payload.Message.StateRoot - } else { - // State mutated by something we can't predict — drop the hint. - prevStateRoot = phase0.Root{} + nextParentBlockHash, err := getBlockExecutionParentHash(nextBeaconBlock) + if err != nil { + client.logger.Warnf("replay: failed to read next bid parent hash at slot %v: %v", epochBlocks[i+1].Slot, err) + return nil + } + + if payload.Message.Payload.BlockHash == nextParentBlockHash { + // Payload accepted by the next block — apply it. + if err := st.ApplyExecutionPayload(parentState, payload); err != nil { + client.logger.Warnf("replay: ApplyExecutionPayload failed at slot %v: %v", blk.Slot, err) + return nil + } + // Post-payload state HTR is recorded in the envelope itself. + prevStateRoot = payload.Message.StateRoot + } + // else: payload was orphaned (next block built on parent payload). + // Leave state unchanged; gotStateRoot is the correct hint. } } } diff --git a/indexer/beacon/statetransition/block.go b/indexer/beacon/statetransition/block.go index fbe3fe9b1..10bf08f90 100644 --- a/indexer/beacon/statetransition/block.go +++ b/indexer/beacon/statetransition/block.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "fmt" + "time" "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/bellatrix" @@ -26,12 +27,12 @@ import ( // calls (when there are skipped slots) still compute HTR normally. // // Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-block-processing -func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches, parentStateRoot phase0.Root) error { +func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches, parentStateRoot phase0.Root, info *ApplyInfo) error { if state.Version < spec.DataVersionFulu { return nil } - s, err := newStateAccessorWithCaches(state, specs, caches) + s, err := newStateAccessorWithCaches(state, specs, ds, caches) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } @@ -59,9 +60,13 @@ func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedS // mutated state and need fresh HTR. stateRootHint = phase0.Root{} if (uint64(s.Slot)+1)%slotsPerEpoch == 0 { + epochStart := time.Now() if err := processEpochInternal(s, nil); err != nil { return fmt.Errorf("process_epoch at slot %d: %w", s.Slot, err) } + if info != nil { + info.EpochTransitionDur += time.Since(epochStart) + } } s.Slot++ s.writeBack() diff --git a/indexer/beacon/statetransition/operations.go b/indexer/beacon/statetransition/operations.go index e62396a4c..38372d40c 100644 --- a/indexer/beacon/statetransition/operations.go +++ b/indexer/beacon/statetransition/operations.go @@ -151,9 +151,13 @@ func processAttestation(s *stateAccessor, att *electra.Attestation, cc *committe justifiedCheckpoint = s.CurrentJustifiedCheckpoint } - sourceCorrect := data.Source.Epoch == justifiedCheckpoint.Epoch && data.Source.Root == justifiedCheckpoint.Root - targetCorrect := data.Target.Root == getBlockRoot(s, data.Target.Epoch) - headCorrect := data.BeaconBlockRoot == getBlockRootAtSlot(s, data.Slot) + // Spec variable mapping (Deneb+): + // is_matching_source = data.source == justified_checkpoint + // is_matching_target = is_matching_source and target_root_matches + // is_matching_head = is_matching_target and head_root_matches + isMatchingSource := data.Source.Epoch == justifiedCheckpoint.Epoch && data.Source.Root == justifiedCheckpoint.Root + isMatchingTarget := isMatchingSource && data.Target.Root == getBlockRoot(s, data.Target.Epoch) + isMatchingHead := isMatchingTarget && data.BeaconBlockRoot == getBlockRootAtSlot(s, data.Slot) // Gloas: payload_matches check for head attestation. // https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_attestation @@ -170,15 +174,17 @@ func processAttestation(s *stateAccessor, att *electra.Attestation, cc *committe } } - // Determine which flags to set + // Determine which flags to set. + // Modified in Deneb (EIP-7045): TIMELY_TARGET has no delay constraint. + // https://github.com/ethereum/consensus-specs/blob/master/specs/deneb/beacon-chain.md#modified-get_attestation_participation_flag_indices var participationFlags [3]bool - if sourceCorrect && inclusionDelay <= intSqrt(s.specs.SlotsPerEpoch) { + if isMatchingSource && inclusionDelay <= intSqrt(s.specs.SlotsPerEpoch) { participationFlags[TimelySourceFlagIndex] = true } - if sourceCorrect && targetCorrect && inclusionDelay <= s.specs.SlotsPerEpoch { + if isMatchingTarget { participationFlags[TimelyTargetFlagIndex] = true } - if sourceCorrect && targetCorrect && headCorrect && payloadMatches && inclusionDelay == s.specs.MinAttestationInclusionDelay { + if isMatchingHead && payloadMatches && inclusionDelay == s.specs.MinAttestationInclusionDelay { participationFlags[TimelyHeadFlagIndex] = true } @@ -640,9 +646,22 @@ func isAttestationSameSlot(s *stateAccessor, data *phase0.AttestationData) bool func findValidatorByPubkey(s *stateAccessor, pubkey phase0.BLSPubKey) *phase0.ValidatorIndex { if s.caches.pubkeyCache == nil { epoch := s.currentEpoch() + + // Include active validators AND recently-exited validators that could + // still be serving on a sync committee. Sync committees are recomputed + // every EPOCHS_PER_SYNC_COMMITTEE_PERIOD epochs; a committee chosen at + // the start of the current period serves for its full duration, so we + // need validators that were active at any point in the last 2 periods. + syncPeriod := phase0.Epoch(s.specs.EpochsPerSyncCommitteePeriod) + cutoff := phase0.Epoch(0) + if epoch > 2*syncPeriod { + cutoff = epoch - 2*syncPeriod + } + s.caches.pubkeyCache = make(map[phase0.BLSPubKey]phase0.ValidatorIndex, len(s.Validators)) for i, v := range s.Validators { - if isActiveValidator(v, epoch) { + // Active now, or exited recently enough to still be on a sync committee. + if isActiveValidator(v, epoch) || v.ExitEpoch >= cutoff { s.caches.pubkeyCache[v.PublicKey] = phase0.ValidatorIndex(i) } } diff --git a/indexer/beacon/statetransition/state.go b/indexer/beacon/statetransition/state.go index 338106855..06a534dbe 100644 --- a/indexer/beacon/statetransition/state.go +++ b/indexer/beacon/statetransition/state.go @@ -12,6 +12,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + dynssz "github.com/pk910/dynamic-ssz" ) // stateAccessor provides a unified interface to access and mutate beacon state fields @@ -20,6 +21,7 @@ import ( type stateAccessor struct { version spec.DataVersion specs *consensus.ChainSpec + dynSsz *dynssz.DynSsz // Common fields shared by Fulu and Gloas (pointers into the underlying state). Slot phase0.Slot @@ -116,14 +118,15 @@ func (c *stateTransitionCaches) invalidateBalanceCaches() { c.committeeCache = newCommitteeCache() } -func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSpec) (*stateAccessor, error) { - return newStateAccessorWithCaches(state, specs, newStateTransitionCaches()) +func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, ds *dynssz.DynSsz) (*stateAccessor, error) { + return newStateAccessorWithCaches(state, specs, ds, newStateTransitionCaches()) } -func newStateAccessorWithCaches(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, caches *stateTransitionCaches) (*stateAccessor, error) { +func newStateAccessorWithCaches(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches) (*stateAccessor, error) { s := &stateAccessor{ version: state.Version, specs: specs, + dynSsz: ds, rawState: state, caches: caches, } @@ -312,14 +315,15 @@ func (s *stateAccessor) writeBack() { } // computeStateHTR computes the hash tree root of the underlying state. +// Uses dynamic-ssz when available (roughly 2x faster than fastssz). // Must call writeBack() first to ensure all accessor fields are synced. func (s *stateAccessor) computeStateHTR() (phase0.Root, error) { s.writeBack() switch s.version { case spec.DataVersionFulu: - return s.rawState.Fulu.HashTreeRoot() + return s.dynSsz.HashTreeRoot(s.rawState.Fulu) case spec.DataVersionGloas: - return s.rawState.Gloas.HashTreeRoot() + return s.dynSsz.HashTreeRoot(s.rawState.Gloas) default: return phase0.Root{}, fmt.Errorf("unsupported version: %v", s.version) } diff --git a/indexer/beacon/statetransition/statetransition.go b/indexer/beacon/statetransition/statetransition.go index dd65b8b44..0b6b664b6 100644 --- a/indexer/beacon/statetransition/statetransition.go +++ b/indexer/beacon/statetransition/statetransition.go @@ -14,6 +14,7 @@ package statetransition import ( "fmt" + "time" "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/electra" @@ -41,9 +42,17 @@ func NewStateTransition(specs *consensus.ChainSpec, ds *dynssz.DynSsz) *StateTra } } +// ApplyInfo collects optional timing information from block application. +// Pass a non-nil pointer to ApplyBlockWithInfo to receive this data. +type ApplyInfo struct { + // EpochTransitionDur is non-zero when the block's process_slots crossed an + // epoch boundary, triggering process_epoch. + EpochTransitionDur time.Duration +} + // ApplyBlock applies a beacon block to the state in-place. func (st *StateTransition) ApplyBlock(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock) error { - return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, phase0.Root{}) + return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, phase0.Root{}, nil) } // ApplyBlockWithStateRoot is like ApplyBlock but accepts the current state's @@ -52,7 +61,13 @@ func (st *StateTransition) ApplyBlock(state *spec.VersionedBeaconState, block *s // sourced from the previously applied block's state_root field. Passing an // incorrect hint will produce an inconsistent state and is undefined behavior. func (st *StateTransition) ApplyBlockWithStateRoot(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, parentStateRoot phase0.Root) error { - return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, parentStateRoot) + return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, parentStateRoot, nil) +} + +// ApplyBlockWithInfo is like ApplyBlockWithStateRoot but also populates info +// with timing details (e.g. epoch transition duration). +func (st *StateTransition) ApplyBlockWithInfo(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, parentStateRoot phase0.Root, info *ApplyInfo) error { + return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, parentStateRoot, info) } // ApplyExecutionPayload applies a Gloas execution payload to the state. @@ -65,7 +80,7 @@ func (st *StateTransition) ApplyExecutionPayload(state *spec.VersionedBeaconStat // PrepareEpochPreState advances a post-block state to the pre-state of the target epoch. func (st *StateTransition) PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, info *TransitionInfo) error { - return prepareEpochPreStateInternal(state, epoch, payload, st.specs, info, st.caches) + return prepareEpochPreStateInternal(state, epoch, payload, st.specs, st.dynSsz, info, st.caches) } // TransitionInfo collects metadata from the state transition that callers may @@ -94,7 +109,7 @@ type TransitionInfo struct { // conversions, balance updates, proposer lookahead, etc. // // If info is non-nil, it is populated with metadata from the transition. -func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec, info *TransitionInfo, caches *stateTransitionCaches) error { +func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec, ds *dynssz.DynSsz, info *TransitionInfo, caches *stateTransitionCaches) error { if state.Version < spec.DataVersionFulu { return nil } @@ -108,7 +123,7 @@ func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0 // Step 2: Advance to the first slot of the target epoch. targetSlot := phase0.Slot(uint64(epoch) * specs.SlotsPerEpoch) - if err := processSlots(state, targetSlot, specs, info, caches); err != nil { + if err := processSlots(state, targetSlot, specs, ds, info, caches); err != nil { return fmt.Errorf("process_slots to epoch %d (slot %d): %w", epoch, targetSlot, err) } @@ -123,7 +138,7 @@ func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0 // outputs we need. Jumps directly to each epoch boundary. // // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots -func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec, info *TransitionInfo, caches *stateTransitionCaches) error { +func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec, ds *dynssz.DynSsz, info *TransitionInfo, caches *stateTransitionCaches) error { currentSlot, err := state.Slot() if err != nil { return fmt.Errorf("failed to get state slot: %w", err) @@ -133,7 +148,7 @@ func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, spec return nil } - s, err := newStateAccessorWithCaches(state, specs, caches) + s, err := newStateAccessorWithCaches(state, specs, ds, caches) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } @@ -218,7 +233,7 @@ func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.E return nil } - s, err := newStateAccessor(state, specs) + s, err := newStateAccessor(state, specs, nil) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } diff --git a/indexer/beacon/statetransition/withdrawals.go b/indexer/beacon/statetransition/withdrawals.go index afb9ddd7a..3c5f42022 100644 --- a/indexer/beacon/statetransition/withdrawals.go +++ b/indexer/beacon/statetransition/withdrawals.go @@ -102,69 +102,109 @@ func convertValidatorIndexToBuilderIndex(validatorIdx phase0.ValidatorIndex) glo return gloas.BuilderIndex(uint64(validatorIdx) &^ BuilderIndexFlag) } +// getBalanceAfterWithdrawals returns a validator's balance minus any amounts +// already scheduled in prior withdrawals within the same batch. +// https://github.com/ethereum/consensus-specs/blob/master/specs/capella/beacon-chain.md#new-get_balance_after_withdrawals +func getBalanceAfterWithdrawals(s *stateAccessor, vidx phase0.ValidatorIndex, withdrawals []*capella.Withdrawal) phase0.Gwei { + balance := s.Balances[vidx] + for _, w := range withdrawals { + if w.ValidatorIndex == vidx { + if balance >= w.Amount { + balance -= w.Amount + } else { + balance = 0 + } + } + } + return balance +} + +// isFullyWithdrawableValidator checks if a validator is fully withdrawable. +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-is_fully_withdrawable_validator +func isFullyWithdrawableValidator(v *phase0.Validator, balance phase0.Gwei, epoch phase0.Epoch) bool { + return hasExecutionWithdrawalCredential(v) && v.WithdrawableEpoch <= epoch && balance > 0 +} + +// isPartiallyWithdrawableValidator checks if a validator is partially withdrawable (sweep). +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-is_partially_withdrawable_validator +func isPartiallyWithdrawableValidator(v *phase0.Validator, balance phase0.Gwei, maxEB phase0.Gwei) bool { + return hasExecutionWithdrawalCredential(v) && v.EffectiveBalance == maxEB && balance > maxEB +} + +// isEligibleForPartialWithdrawals checks if a validator can process a pending partial withdrawal. +// https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-is_eligible_for_partial_withdrawals +func isEligibleForPartialWithdrawals(v *phase0.Validator, balance phase0.Gwei, minActivationBalance phase0.Gwei) bool { + return v.ExitEpoch == FarFutureEpoch && + v.EffectiveBalance >= minActivationBalance && + balance > minActivationBalance +} + // getExpectedWithdrawals computes the expected withdrawals for the current slot. // Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-get_expected_withdrawals func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { result := &expectedWithdrawals{} nextIdx := s.NextWithdrawalIndex maxWithdrawals := s.specs.MaxWithdrawalsPerPayload - // Builder/partial/builder-sweep withdrawals use MAX-1 limit - subLimit := maxWithdrawals - 1 + epoch := s.currentEpoch() - // 1. Builder pending withdrawals - for _, bpw := range s.BuilderPendingWithdrawals { - if uint64(len(result.withdrawals)) >= subLimit { - break + // 1. Builder pending withdrawals (Gloas-specific) + if len(s.BuilderPendingWithdrawals) > 0 { + builderLimit := maxWithdrawals - 1 + for _, bpw := range s.BuilderPendingWithdrawals { + if uint64(len(result.withdrawals)) >= builderLimit { + break + } + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: convertBuilderIndexToValidatorIndex(bpw.BuilderIndex), + Address: bpw.FeeRecipient, + Amount: bpw.Amount, + }) + nextIdx++ + result.processedBuilderWithdrawalsCount++ } - result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ - Index: nextIdx, - ValidatorIndex: convertBuilderIndexToValidatorIndex(bpw.BuilderIndex), - Address: bpw.FeeRecipient, - Amount: bpw.Amount, - }) - nextIdx++ - result.processedBuilderWithdrawalsCount++ } // 2. Pending partial withdrawals - epoch := s.currentEpoch() - for _, pw := range s.PendingPartialWithdrawals { - if uint64(len(result.withdrawals)) >= subLimit { - break - } - if pw.WithdrawableEpoch > epoch { - break + // Limit: min(prior_count + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, MAX_WITHDRAWALS_PER_PAYLOAD - 1) + // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#new-get_pending_partial_withdrawals + { + partialLimit := uint64(len(result.withdrawals)) + s.specs.MaxPendingPartialsPerWithdrawalsSweep + if partialLimit > maxWithdrawals-1 { + partialLimit = maxWithdrawals - 1 } - result.processedPartialWithdrawalsCount++ - validator := s.Validators[pw.ValidatorIndex] - if validator.ExitEpoch != FarFutureEpoch || !hasExecutionWithdrawalCredential(validator) { - continue - } - - balance := s.Balances[pw.ValidatorIndex] minBalance := phase0.Gwei(s.specs.MinActivationBalance) - if balance <= minBalance { - continue - } + for _, pw := range s.PendingPartialWithdrawals { + if pw.WithdrawableEpoch > epoch || uint64(len(result.withdrawals)) >= partialLimit { + break + } + result.processedPartialWithdrawalsCount++ - withdrawableAmount := balance - minBalance - amount := phase0.Gwei(pw.Amount) - if withdrawableAmount < amount { - amount = withdrawableAmount - } + validator := s.Validators[pw.ValidatorIndex] + balance := getBalanceAfterWithdrawals(s, pw.ValidatorIndex, result.withdrawals) + if !isEligibleForPartialWithdrawals(validator, balance, minBalance) { + continue + } - result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ - Index: nextIdx, - ValidatorIndex: pw.ValidatorIndex, - Address: getWithdrawalAddress(validator), - Amount: amount, - }) - nextIdx++ + amount := balance - minBalance + if phase0.Gwei(pw.Amount) < amount { + amount = phase0.Gwei(pw.Amount) + } + + result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ + Index: nextIdx, + ValidatorIndex: pw.ValidatorIndex, + Address: getWithdrawalAddress(validator), + Amount: amount, + }) + nextIdx++ + } } // 3. Builder sweep withdrawals (Gloas-specific) if len(s.Builders) > 0 { + builderSweepLimit := maxWithdrawals - 1 buildersLimit := uint64(len(s.Builders)) if s.specs.MaxBuildersPerWithdrawalsSweep > 0 && s.specs.MaxBuildersPerWithdrawalsSweep < buildersLimit { buildersLimit = s.specs.MaxBuildersPerWithdrawalsSweep @@ -172,7 +212,7 @@ func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { builderIdx := s.NextWithdrawalBuilderIndex for i := uint64(0); i < buildersLimit; i++ { - if uint64(len(result.withdrawals)) >= subLimit { + if uint64(len(result.withdrawals)) >= builderSweepLimit { break } @@ -193,6 +233,7 @@ func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { } // 4. Validator sweep withdrawals (uses full MAX limit) + // https://github.com/ethereum/consensus-specs/blob/master/specs/electra/beacon-chain.md#modified-get_validators_sweep_withdrawals validatorCount := uint64(len(s.Validators)) if validatorCount > 0 { startIdx := uint64(s.NextWithdrawalValidatorIndex) @@ -204,16 +245,12 @@ func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { for i := uint64(0); i < bound && uint64(len(result.withdrawals)) < maxWithdrawals; i++ { vidx := phase0.ValidatorIndex((startIdx + i) % validatorCount) validator := s.Validators[vidx] - balance := s.Balances[vidx] + balance := getBalanceAfterWithdrawals(s, vidx, result.withdrawals) result.processedValidatorsSweepCount++ - if !hasExecutionWithdrawalCredential(validator) { - continue - } - // Full withdrawal: exited and withdrawable - if validator.ExitEpoch != FarFutureEpoch && validator.WithdrawableEpoch <= epoch && balance > 0 { + if isFullyWithdrawableValidator(validator, balance, epoch) { result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ Index: nextIdx, ValidatorIndex: vidx, @@ -224,9 +261,10 @@ func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { continue } - // Partial (sweep) withdrawal: excess balance + // Partial (sweep) withdrawal: excess balance above max effective balance, + // only when effective_balance has reached the max. maxEB := s.getMaxEffectiveBalance(validator) - if balance > maxEB { + if isPartiallyWithdrawableValidator(validator, balance, maxEB) { result.withdrawals = append(result.withdrawals, &capella.Withdrawal{ Index: nextIdx, ValidatorIndex: vidx, @@ -236,7 +274,6 @@ func getExpectedWithdrawals(s *stateAccessor) *expectedWithdrawals { nextIdx++ } } - } return result From d8538af633af197e8c83d81c1c23dbe30f048775 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 10 Apr 2026 14:52:19 +0200 Subject: [PATCH 92/99] switch eth-das-guardian to ethpandaops fork for gloas support Point dora at ethpandaops/eth-das-guardian@gloas-support, which adds Gloas (EIP-7732) fork support on top of probe-lab/eth-das-guardian. - go.mod: replace probe-lab/eth-das-guardian v0.2.2 with ethpandaops/eth-das-guardian pinned to the gloas-support head - bump the pk910/go-eth2-client replace target to match the commit required by the gloas-support branch - implement GetGloasForkEpoch on dasGuardianAPI (new method on the BeaconAPI interface) - route GetStateVersion through "gloas" once the gloas fork epoch is reached so scans go through the shared PeerDAS path Co-Authored-By: Claude Opus 4.6 (1M context) --- go.mod | 6 +++--- go.sum | 12 ++++++------ handlers/api/api_das_guardian.go | 2 +- handlers/api/api_das_guardian_mass.go | 2 +- services/dasguardian.go | 24 ++++++++++++++++++++---- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index c1ecb9cd7..e0cd8c8c4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/attestantio/go-eth2-client v0.28.0 github.com/cockroachdb/pebble v1.1.5 github.com/ethereum/go-ethereum v1.17.0 + github.com/ethpandaops/eth-das-guardian v0.0.0-20260410123341-c9663e34164d github.com/ethpandaops/ethcore v0.0.0-20260225052739-1d2ffa8e298c github.com/ethpandaops/ethwallclock v0.4.0 github.com/glebarez/go-sqlite v1.22.0 @@ -22,9 +23,8 @@ require ( github.com/mashingan/smapping v0.1.19 github.com/minio/minio-go/v7 v7.0.98 github.com/mitchellh/mapstructure v1.5.0 - github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387 + github.com/pk910/dynamic-ssz v1.3.0 github.com/pressly/goose/v3 v3.27.0 - github.com/probe-lab/eth-das-guardian v0.2.2 github.com/protolambda/bls12-381-util v0.1.0 github.com/protolambda/zrnt v0.34.1 github.com/protolambda/ztyp v0.2.2 @@ -255,4 +255,4 @@ require ( modernc.org/sqlite v1.46.1 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260401105128-4544cf21b737 diff --git a/go.sum b/go.sum index a0e943de1..67dbf53f9 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/ethereum/go-ethereum v1.17.0 h1:2D+1Fe23CwZ5tQoAS5DfwKFNI1HGcTwi65/kRlAVxes= github.com/ethereum/go-ethereum v1.17.0/go.mod h1:2W3msvdosS/MCWytpqTcqgFiRYbTH59FxDJzqah120o= +github.com/ethpandaops/eth-das-guardian v0.0.0-20260410123341-c9663e34164d h1:qyBM/DoN7kTrELetvPE/zZbNyESai/Jp0NDtQIV6i24= +github.com/ethpandaops/eth-das-guardian v0.0.0-20260410123341-c9663e34164d/go.mod h1:+1ZU+Z8YZXBp5jWcd1ATJ0xIJfHuj/sufIOL+qJbOLw= github.com/ethpandaops/ethcore v0.0.0-20260225052739-1d2ffa8e298c h1:KJ7LDEC1cEMbpg8oe3lR2QvKxguIW7E5R5hP7TC0/WM= github.com/ethpandaops/ethcore v0.0.0-20260225052739-1d2ffa8e298c/go.mod h1:NblaW7ZBfBA85g9SsJyiDzeecTgvxMnFZTWUAj40vg8= github.com/ethpandaops/ethwallclock v0.4.0 h1:+sgnhf4pk6hLPukP076VxkiLloE4L0Yk1yat+ZyHh1g= @@ -558,10 +560,10 @@ github.com/pion/turn/v4 v4.1.1 h1:9UnY2HB99tpDyz3cVVZguSxcqkJ1DsTSZ+8TGruh4fc= github.com/pion/turn/v4 v4.1.1/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8= github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= -github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387 h1:XkL2iLFDP6/NKAOF0fysZrhaa/qtcAtqzhLUxSIy79s= -github.com/pk910/dynamic-ssz v1.2.3-0.20260318065836-323b83c1a387/go.mod h1:NmeFF4jxzVwWC8cnEhUB7xMI++8hd/0OZvZHFrUvFfs= -github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81 h1:mHgrerDiro/np9FCoJ19EYsCyl/CXMHFi34o+j3T+rE= -github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81/go.mod h1:lwj0l8l51hIjqdQpODPea01JfE33nyM++1VGjBZau08= +github.com/pk910/dynamic-ssz v1.3.0 h1:b6v5v3HWAmdxSVKWk4GS7Y/lhURZPAHRQ7EEMUparQk= +github.com/pk910/dynamic-ssz v1.3.0/go.mod h1:NmeFF4jxzVwWC8cnEhUB7xMI++8hd/0OZvZHFrUvFfs= +github.com/pk910/go-eth2-client v0.0.0-20260401105128-4544cf21b737 h1:+tHLCpEGiohePzGMb1QhDjNLLPH43/dEeslfS4ajpRc= +github.com/pk910/go-eth2-client v0.0.0-20260401105128-4544cf21b737/go.mod h1:cmccJOSI3vYd1LKyU7jo8tR71vGaLmcWzK4xfefei0I= github.com/pk910/hashtree-bindings v0.1.0 h1:w7NyRWFi2OaYEFvo9ADcE/QU6PMuVLl3hBgx92KiH9c= github.com/pk910/hashtree-bindings v0.1.0/go.mod h1:zrWt88783JmhBfcgni6kkIMYRdXTZi/FL//OyI5T/l4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -573,8 +575,6 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.27.0 h1:/D30gVTuQhu0WsNZYbJi4DMOsx1lNq+6SkLe+Wp59BM= github.com/pressly/goose/v3 v3.27.0/go.mod h1:3ZBeCXqzkgIRvrEMDkYh1guvtoJTU5oMMuDdkutoM78= -github.com/probe-lab/eth-das-guardian v0.2.2 h1:ygQjHt9jVD6yvujtBLuWPA3fYnPe6D5b2S7xOASYfm4= -github.com/probe-lab/eth-das-guardian v0.2.2/go.mod h1:xkhVpnhL1y2tmQERuaJTUGFAMb4eKWSVEBr0rhAB6ZM= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= diff --git a/handlers/api/api_das_guardian.go b/handlers/api/api_das_guardian.go index 8d85bfea1..4f14c6ef9 100644 --- a/handlers/api/api_das_guardian.go +++ b/handlers/api/api_das_guardian.go @@ -12,7 +12,7 @@ import ( "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" - dasguardian "github.com/probe-lab/eth-das-guardian" + dasguardian "github.com/ethpandaops/eth-das-guardian" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/api_das_guardian_mass.go b/handlers/api/api_das_guardian_mass.go index d0afbd0b3..b890c035f 100644 --- a/handlers/api/api_das_guardian_mass.go +++ b/handlers/api/api_das_guardian_mass.go @@ -13,7 +13,7 @@ import ( "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" - dasguardian "github.com/probe-lab/eth-das-guardian" + dasguardian "github.com/ethpandaops/eth-das-guardian" "github.com/sirupsen/logrus" ) diff --git a/services/dasguardian.go b/services/dasguardian.go index 40feb624b..11f88ab3b 100644 --- a/services/dasguardian.go +++ b/services/dasguardian.go @@ -10,8 +10,8 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common/hexutil" - dasguardian "github.com/probe-lab/eth-das-guardian" - "github.com/probe-lab/eth-das-guardian/api" + dasguardian "github.com/ethpandaops/eth-das-guardian" + "github.com/ethpandaops/eth-das-guardian/api" "github.com/sirupsen/logrus" ) @@ -131,10 +131,13 @@ func (d *dasGuardianAPI) Init(ctx context.Context) error { } func (d *dasGuardianAPI) GetStateVersion() string { - fuluForkEpoch := d.GetFuluForkEpoch() currentEpoch := GlobalBeaconService.GetChainState().CurrentEpoch() - if currentEpoch >= phase0.Epoch(fuluForkEpoch) { + if currentEpoch >= phase0.Epoch(d.GetGloasForkEpoch()) { + return "gloas" + } + + if currentEpoch >= phase0.Epoch(d.GetFuluForkEpoch()) { return "fulu" } @@ -174,6 +177,19 @@ func (d *dasGuardianAPI) GetFuluForkEpoch() uint64 { return *specs.FuluForkEpoch } +func (d *dasGuardianAPI) GetGloasForkEpoch() uint64 { + specs := GlobalBeaconService.GetChainState().GetSpecs() + if specs == nil { + return 0 + } + + if specs.GloasForkEpoch == nil { + return math.MaxInt64 + } + + return *specs.GloasForkEpoch +} + func (d *dasGuardianAPI) GetNodeIdentity(ctx context.Context) (*api.NodeIdentity, error) { // Get the first available consensus client consensusClients := GlobalBeaconService.GetConsensusClients() From a506eb3bfff91ec566e79e69d784dd3db0ae126c Mon Sep 17 00:00:00 2001 From: pk910 Date: Sat, 11 Apr 2026 03:55:39 +0200 Subject: [PATCH 93/99] clean up state transition & fix builder paymanet type detection --- handlers/builder.go | 6 + indexer/beacon/indexer_getter.go | 39 +++- indexer/beacon/state_sim.go | 204 +++++++++++++----- indexer/beacon/statetransition/block.go | 20 +- indexer/beacon/statetransition/state.go | 14 +- .../beacon/statetransition/statetransition.go | 67 +++--- indexer/beacon/writedb.go | 20 ++ services/chainservice_builder.go | 5 + 8 files changed, 261 insertions(+), 114 deletions(-) diff --git a/handlers/builder.go b/handlers/builder.go index 679e9a70f..6f9884fb5 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -163,6 +163,12 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b return nil, 0 } + // Override balance from the latest epoch state (builder cache doesn't track balance changes within epochs). + balances := services.GlobalBeaconService.GetBuilderBalances() + if int(builderIndex) < len(balances) { + builder.Balance = balances[builderIndex] + } + // Determine state finalizedEpoch, _ := chainState.GetFinalizedCheckpoint() state := "Active" diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 2d7988408..42876af77 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -545,6 +545,8 @@ func (indexer *Indexer) GetBuilderByIndex(index gloas.BuilderIndex, overrideFork } // GetRecentBuilderBalances returns the most recent builder balances for the given fork. +// Starts with epoch-boundary balances and deducts any builder withdrawals processed +// in blocks since the epoch start, reflecting the current head's state. func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phase0.Gwei { chainState := indexer.consensusPool.GetChainState() @@ -556,6 +558,7 @@ func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phas headEpoch := chainState.EpochOfSlot(canonicalHead.Slot) var epochStats *EpochStats + var statsEpoch phase0.Epoch for { cEpoch := chainState.EpochOfSlot(canonicalHead.Slot) if headEpoch-cEpoch > 2 { @@ -574,6 +577,7 @@ func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phas } epochStats = stats + statsEpoch = cEpoch break } @@ -581,5 +585,38 @@ func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phas return nil } - return epochStats.dependentState.builderBalances + // Copy epoch-boundary balances so we can mutate them. + src := epochStats.dependentState.builderBalances + balances := make([]phase0.Gwei, len(src)) + copy(balances, src) + + // Deduct builder withdrawals from blocks in the current epoch. + // Block UIDs encode the slot in the upper bits: uid = slot << 16. + epochStartSlot := chainState.EpochToSlot(statsEpoch) + nextEpochSlot := chainState.EpochToSlot(statsEpoch + 1) + minBlockUid := uint64(epochStartSlot) << 16 + maxBlockUid := uint64(nextEpochSlot) << 16 + + withdrawals, err := db.GetWithdrawalsByBlockUidRange(indexer.ctx, minBlockUid, maxBlockUid, nil) + if err == nil { + for _, w := range withdrawals { + if w.Orphaned { + continue + } + // Builder withdrawals have the BuilderIndexFlag set in the Validator field. + if w.Validator&BuilderIndexFlag == 0 { + continue + } + builderIdx := w.Validator &^ BuilderIndexFlag + if builderIdx < uint64(len(balances)) { + if balances[builderIdx] >= phase0.Gwei(w.Amount) { + balances[builderIdx] -= phase0.Gwei(w.Amount) + } else { + balances[builderIdx] = 0 + } + } + } + } + + return balances } diff --git a/indexer/beacon/state_sim.go b/indexer/beacon/state_sim.go index f996bc4dc..3756e2fa1 100644 --- a/indexer/beacon/state_sim.go +++ b/indexer/beacon/state_sim.go @@ -21,11 +21,20 @@ type stateSimulator struct { validatorSet []*phase0.Validator } +// trackedBuilderWithdrawal pairs a builder pending withdrawal with the optional +// BlockUID of the block whose payload delivery created it. Entries loaded from +// the epoch boundary state have RefBlockUID == nil; entries added during replay +// carry the source block's UID. +type trackedBuilderWithdrawal struct { + gloas.BuilderPendingWithdrawal + RefBlockUID *uint64 +} + type stateSimulatorState struct { epochRoot phase0.Root block *Block pendingWithdrawals []electra.PendingPartialWithdrawal - builderPendingWithdrawals []gloas.BuilderPendingWithdrawal + builderPendingWithdrawals []trackedBuilderWithdrawal builderDelayedCount uint32 // how many entries in builderPendingWithdrawals are delayed/quorum payments additionalWithdrawals []phase0.ValidatorIndex pendingConsolidationCount uint64 @@ -100,16 +109,20 @@ func (sim *stateSimulator) resetState(block *Block) *stateSimulatorState { } } - builderPendingWithdrawals := sim.epochStatsValues.BuilderPendingWithdrawals - if builderPendingWithdrawals == nil { - builderPendingWithdrawals = []gloas.BuilderPendingWithdrawal{} + rawBuilderWithdrawals := sim.epochStatsValues.BuilderPendingWithdrawals + if rawBuilderWithdrawals == nil { + rawBuilderWithdrawals = []gloas.BuilderPendingWithdrawal{} + } + trackedWithdrawals := make([]trackedBuilderWithdrawal, len(rawBuilderWithdrawals)) + for i := range rawBuilderWithdrawals { + trackedWithdrawals[i] = trackedBuilderWithdrawal{BuilderPendingWithdrawal: rawBuilderWithdrawals[i]} } state := &stateSimulatorState{ block: nil, epochRoot: epochRoot, pendingWithdrawals: pendingWithdrawals, - builderPendingWithdrawals: builderPendingWithdrawals, + builderPendingWithdrawals: trackedWithdrawals, builderDelayedCount: sim.epochStatsValues.DelayedBuilderPaymentCount, // delayed payments from epoch transition are at the tail pendingConsolidationCount: 0, additionalWithdrawals: []phase0.ValidatorIndex{}, @@ -137,9 +150,67 @@ func (sim *stateSimulator) resetState(block *Block) *stateSimulatorState { // get pending withdrawals from state state.pendingWithdrawals = sim.epochStatsValues.PendingWithdrawals + // Resolve RefBlockUIDs for initial direct entries by matching them to + // blocks with delivered payloads in the previous epoch (FIFO order). + directCount := len(trackedWithdrawals) - int(state.builderDelayedCount) + if directCount > 0 { + sim.resolveInitialDirectRefs(state, directCount) + } + return state } +// resolveInitialDirectRefs populates RefBlockUID for the first directCount entries +// in the builder pending withdrawals queue. These are direct payments from the +// previous epoch's delivered payloads, loaded from the epoch boundary state without +// source block information. We scan the previous epoch's blocks in slot order and +// match delivered payloads to queue entries by FIFO position. +func (sim *stateSimulator) resolveInitialDirectRefs(state *stateSimulatorState, directCount int) { + chainState := sim.indexer.consensusPool.GetChainState() + + if sim.epochStats.epoch == 0 { + return + } + prevEpoch := sim.epochStats.epoch - 1 + prevStart := chainState.EpochToSlot(prevEpoch) + prevEnd := chainState.EpochToSlot(prevEpoch + 1) + + resolved := 0 + _, prunedEpoch := sim.indexer.GetBlockCacheState() + if prevEpoch >= prunedEpoch { + // Previous epoch is in cache + for slot := prevStart; slot < prevEnd && resolved < directCount; slot++ { + blocks := sim.indexer.GetBlocksBySlot(slot) + for _, b := range blocks { + if b.HasExecutionPayload() && !b.isPayloadOrphaned { + uid := b.BlockUID + state.builderPendingWithdrawals[resolved].RefBlockUID = &uid + resolved++ + if resolved >= directCount { + break + } + } + } + } + } else { + // Previous epoch is finalized/pruned — query DB + dbSlots := db.GetSlotsRange(sim.indexer.ctx, uint64(prevEnd-1), uint64(prevStart), false, false) + for _, assignedSlot := range dbSlots { + if resolved >= directCount { + break + } + if assignedSlot.Block == nil { + continue + } + if assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusCanonical { + uid := assignedSlot.Block.BlockUid + state.builderPendingWithdrawals[resolved].RefBlockUID = &uid + resolved++ + } + } + } +} + func (sim *stateSimulator) getValidator(index phase0.ValidatorIndex) *phase0.Validator { if validator, ok := sim.prevState.validatorMap[index]; ok { return validator @@ -382,8 +453,12 @@ func (sim *stateSimulator) applyBlock(block *Block) [][]uint8 { if block.HasExecutionPayload() && !block.isPayloadOrphaned { blockIndex := block.GetBlockIndex(sim.indexer.ctx) if blockIndex != nil && blockIndex.BuilderIndex != math.MaxUint64 { - sim.prevState.builderPendingWithdrawals = append(sim.prevState.builderPendingWithdrawals, gloas.BuilderPendingWithdrawal{ - BuilderIndex: gloas.BuilderIndex(blockIndex.BuilderIndex), + uid := block.BlockUID + sim.prevState.builderPendingWithdrawals = append(sim.prevState.builderPendingWithdrawals, trackedBuilderWithdrawal{ + BuilderPendingWithdrawal: gloas.BuilderPendingWithdrawal{ + BuilderIndex: gloas.BuilderIndex(blockIndex.BuilderIndex), + }, + RefBlockUID: &uid, }) // builderDelayedCount stays the same — the new entry is direct, not delayed } @@ -655,52 +730,51 @@ func (sim *stateSimulator) classifyBuilderPayments(block *Block, builderCount in // Delayed entries are at the tail: positions [builderCount - delayedCount, builderCount) delayedStart := builderCount - int(delayedCount) - // For direct payment ref: use the last replayed block's UID if available, - // otherwise fall back to the source block UID (last block of parent epoch). - // At epoch start (no blocks replayed yet), prevState.block is nil, so we use - // SourceBlockUid which was saved before the epoch transition was applied. - var directRefBlockUID *uint64 - if sim.prevState.block != nil { - uid := sim.prevState.block.BlockUID - directRefBlockUID = &uid - } else if sim.epochStatsValues.SourceBlockUid > 0 { - uid := sim.epochStatsValues.SourceBlockUid - directRefBlockUID = &uid - } + // Resolve delayed entries by matching against blocks with missed/orphaned payloads + // from the source epoch (2 epochs back). The delayed entries are generated in slot + // order by process_builder_pending_payments, so we collect all candidate blocks + // and assign them to delayed entries in order. + delayedRefs := sim.resolveDelayedPaymentRefSlots(builderCount, block) for i := range payments { if i >= delayedStart && delayedStart >= 0 { // Delayed entry (tail of queue, from epoch transition) payments[i].Type = dbtypes.WithdrawalTypeBuilderDelayedPayment - payments[i].RefSlot = sim.resolveDelayedPaymentRefSlot(i, block) - } else if directRefBlockUID != nil { - // Direct entry with known ref block (from a previous block's delivered payload) - payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment - payments[i].RefSlot = directRefBlockUID + delayedOff := i - delayedStart + if delayedOff < len(delayedRefs) { + payments[i].RefSlot = delayedRefs[delayedOff] + } } else { - // Direct entry with unknown ref block (rare leftover from earlier epochs) + // Direct entry (from a delivered payload) payments[i].Type = dbtypes.WithdrawalTypeBuilderPayment + // Use the tracked source block UID if available (set during replay). + if i < len(sim.prevState.builderPendingWithdrawals) { + payments[i].RefSlot = sim.prevState.builderPendingWithdrawals[i].RefBlockUID + } } } return payments } -// resolveDelayedPaymentRefSlot finds the block UID of the block a delayed builder payment -// refers to, by matching the builder index against blocks with missed payloads. -// Due to double-buffering in builder_pending_payments, delayed payments from epoch K -// are evaluated at epoch K+2 boundary. So we look TWO epochs back from the block's epoch. -// If the target epoch is already pruned from cache, falls back to DB query. -func (sim *stateSimulator) resolveDelayedPaymentRefSlot(delayedIdx int, block *Block) *uint64 { - if delayedIdx < 0 || delayedIdx >= len(sim.prevState.builderPendingWithdrawals) { +// resolveDelayedPaymentRefSlots resolves reference block UIDs for all delayed entries +// in the builder pending withdrawals queue. Delayed payments originate from +// process_builder_pending_payments during the epoch transition, which processes +// BuilderPendingPayments entries from 2 epochs ago in slot order. Each delayed +// entry corresponds to a block where the builder's payload was missed/orphaned. +// +// We scan the source epoch's blocks in slot order, collecting those with missed +// payloads, and assign them to delayed entries in FIFO order. +func (sim *stateSimulator) resolveDelayedPaymentRefSlots(builderCount int, block *Block) []*uint64 { + delayedCount := int(sim.prevState.builderDelayedCount) + if delayedCount == 0 { return nil } - builderIndex := sim.prevState.builderPendingWithdrawals[delayedIdx].BuilderIndex chainState := sim.indexer.consensusPool.GetChainState() blockEpoch := chainState.EpochOfSlot(block.Slot) if blockEpoch < 2 { - return nil + return make([]*uint64, delayedCount) } // Delayed payments from epoch K-2 are evaluated at epoch K boundary @@ -708,41 +782,73 @@ func (sim *stateSimulator) resolveDelayedPaymentRefSlot(delayedIdx int, block *B sourceEpochFirstSlot := chainState.EpochToSlot(sourceEpoch) sourceEpochEndSlot := chainState.EpochToSlot(sourceEpoch + 1) - // Check if the source epoch is still in the block cache + // Collect all blocks with missed/orphaned payloads from the source epoch, in slot order. + type missedBlock struct { + builderIndex uint64 + blockUID uint64 + } + var missedBlocks []missedBlock + _, prunedEpoch := sim.indexer.GetBlockCacheState() if sourceEpoch >= prunedEpoch { - // Source epoch is in cache — search cached blocks for slot := sourceEpochFirstSlot; slot < sourceEpochEndSlot; slot++ { blocks := sim.indexer.GetBlocksBySlot(slot) for _, b := range blocks { blockIndex := b.GetBlockIndex(sim.indexer.ctx) - if blockIndex == nil { + if blockIndex == nil || blockIndex.BuilderIndex == math.MaxUint64 { continue } - if blockIndex.BuilderIndex == uint64(builderIndex) && (!b.HasExecutionPayload() || b.isPayloadOrphaned) { - uid := b.BlockUID - return &uid + if !b.HasExecutionPayload() || b.isPayloadOrphaned { + missedBlocks = append(missedBlocks, missedBlock{ + builderIndex: blockIndex.BuilderIndex, + blockUID: b.BlockUID, + }) } } } } else { - // Source epoch is finalized/pruned — query DB for blocks with missed payloads dbSlots := db.GetSlotsRange(sim.indexer.ctx, uint64(sourceEpochEndSlot-1), uint64(sourceEpochFirstSlot), false, false) for _, assignedSlot := range dbSlots { if assignedSlot.Block == nil { continue } - dbBuilderIndex := assignedSlot.Block.BuilderIndex - if dbBuilderIndex < 0 { - continue // self-built (MaxUint64 stored as -1 in int64) + if assignedSlot.Block.BuilderIndex < 0 { + continue } - if uint64(dbBuilderIndex) == uint64(builderIndex) && - (assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusMissing || assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusOrphaned) { - uid := assignedSlot.Block.BlockUid - return &uid + if assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusMissing || assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusOrphaned { + missedBlocks = append(missedBlocks, missedBlock{ + builderIndex: uint64(assignedSlot.Block.BuilderIndex), + blockUID: assignedSlot.Block.BlockUid, + }) + } + } + } + + // Match delayed entries to missed blocks in order. Each delayed entry's builder + // index must match the missed block's builder index. Multiple delayed entries for + // the same builder consume successive missed blocks for that builder. + delayedStart := builderCount - delayedCount + refs := make([]*uint64, delayedCount) + consumed := make([]bool, len(missedBlocks)) + + for i := range delayedCount { + queueIdx := delayedStart + i + if queueIdx < 0 || queueIdx >= len(sim.prevState.builderPendingWithdrawals) { + continue + } + wantBuilder := sim.prevState.builderPendingWithdrawals[queueIdx].BuilderIndex + for j, mb := range missedBlocks { + if consumed[j] { + continue + } + if mb.builderIndex == uint64(wantBuilder) { + uid := mb.blockUID + refs[i] = &uid + consumed[j] = true + break } } } - return nil + return refs } diff --git a/indexer/beacon/statetransition/block.go b/indexer/beacon/statetransition/block.go index 10bf08f90..5f2787f8e 100644 --- a/indexer/beacon/statetransition/block.go +++ b/indexer/beacon/statetransition/block.go @@ -11,8 +11,6 @@ import ( "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/ethpandaops/dora/clients/consensus" - dynssz "github.com/pk910/dynamic-ssz" "github.com/pk910/dynamic-ssz/sszutils" ) @@ -27,12 +25,12 @@ import ( // calls (when there are skipped slots) still compute HTR normally. // // Modified in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-block-processing -func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches, parentStateRoot phase0.Root, info *ApplyInfo) error { +func (st *StateTransition) applyBlock(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, parentStateRoot phase0.Root, info *ApplyInfo) error { if state.Version < spec.DataVersionFulu { return nil } - s, err := newStateAccessorWithCaches(state, specs, ds, caches) + s, err := st.newAccessor(state) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } @@ -50,7 +48,7 @@ func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedS // process_epoch(state) // state.slot += 1 // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots - slotsPerEpoch := specs.SlotsPerEpoch + slotsPerEpoch := st.specs.SlotsPerEpoch stateRootHint := parentStateRoot for s.Slot < blockSlot { if err := processSlotRootCaching(s, stateRootHint); err != nil { @@ -95,7 +93,7 @@ func applyBlockInternal(state *spec.VersionedBeaconState, block *spec.VersionedS // process_execution_payload (Fulu) — caches the execution payload header if state.Version == spec.DataVersionFulu { - processFuluExecutionPayload(s, block, ds) + processFuluExecutionPayload(s, block) } // process_execution_payload_bid (Gloas) — records the builder's bid @@ -231,7 +229,7 @@ var _ = sszutils.Annotate[withdrawalList](`ssz-max:"16"`) // processFuluExecutionPayload caches the execution payload header for Fulu blocks. // https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/beacon-chain.md#modified-process_execution_payload -func processFuluExecutionPayload(s *stateAccessor, block *spec.VersionedSignedBeaconBlock, ds *dynssz.DynSsz) { +func processFuluExecutionPayload(s *stateAccessor, block *spec.VersionedSignedBeaconBlock) { if block.Fulu == nil || block.Fulu.Message == nil || block.Fulu.Message.Body == nil { return } @@ -241,20 +239,16 @@ func processFuluExecutionPayload(s *stateAccessor, block *spec.VersionedSignedBe return } - if ds == nil { - return - } - txs := make(transactionList, len(payload.Transactions)) copy(txs, payload.Transactions) - txRoot, err := ds.HashTreeRoot(txs) + txRoot, err := s.dynSsz.HashTreeRoot(txs) if err != nil { return } wds := make(withdrawalList, len(payload.Withdrawals)) copy(wds, payload.Withdrawals) - wdRoot, err := ds.HashTreeRoot(wds) + wdRoot, err := s.dynSsz.HashTreeRoot(wds) if err != nil { return } diff --git a/indexer/beacon/statetransition/state.go b/indexer/beacon/statetransition/state.go index 06a534dbe..9d5ebf219 100644 --- a/indexer/beacon/statetransition/state.go +++ b/indexer/beacon/statetransition/state.go @@ -118,17 +118,15 @@ func (c *stateTransitionCaches) invalidateBalanceCaches() { c.committeeCache = newCommitteeCache() } -func newStateAccessor(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, ds *dynssz.DynSsz) (*stateAccessor, error) { - return newStateAccessorWithCaches(state, specs, ds, newStateTransitionCaches()) -} - -func newStateAccessorWithCaches(state *spec.VersionedBeaconState, specs *consensus.ChainSpec, ds *dynssz.DynSsz, caches *stateTransitionCaches) (*stateAccessor, error) { +// newAccessor creates a stateAccessor from the given state, pulling specs, +// dynSsz, and caches from the StateTransition instance. +func (st *StateTransition) newAccessor(state *spec.VersionedBeaconState) (*stateAccessor, error) { s := &stateAccessor{ version: state.Version, - specs: specs, - dynSsz: ds, + specs: st.specs, + dynSsz: st.dynSsz, rawState: state, - caches: caches, + caches: st.caches, } switch state.Version { diff --git a/indexer/beacon/statetransition/statetransition.go b/indexer/beacon/statetransition/statetransition.go index 0b6b664b6..3c914fac6 100644 --- a/indexer/beacon/statetransition/statetransition.go +++ b/indexer/beacon/statetransition/statetransition.go @@ -52,7 +52,7 @@ type ApplyInfo struct { // ApplyBlock applies a beacon block to the state in-place. func (st *StateTransition) ApplyBlock(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock) error { - return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, phase0.Root{}, nil) + return st.applyBlock(state, block, phase0.Root{}, nil) } // ApplyBlockWithStateRoot is like ApplyBlock but accepts the current state's @@ -61,13 +61,13 @@ func (st *StateTransition) ApplyBlock(state *spec.VersionedBeaconState, block *s // sourced from the previously applied block's state_root field. Passing an // incorrect hint will produce an inconsistent state and is undefined behavior. func (st *StateTransition) ApplyBlockWithStateRoot(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, parentStateRoot phase0.Root) error { - return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, parentStateRoot, nil) + return st.applyBlock(state, block, parentStateRoot, nil) } // ApplyBlockWithInfo is like ApplyBlockWithStateRoot but also populates info // with timing details (e.g. epoch transition duration). func (st *StateTransition) ApplyBlockWithInfo(state *spec.VersionedBeaconState, block *spec.VersionedSignedBeaconBlock, parentStateRoot phase0.Root, info *ApplyInfo) error { - return applyBlockInternal(state, block, st.specs, st.dynSsz, st.caches, parentStateRoot, info) + return st.applyBlock(state, block, parentStateRoot, info) } // ApplyExecutionPayload applies a Gloas execution payload to the state. @@ -75,61 +75,42 @@ func (st *StateTransition) ApplyExecutionPayload(state *spec.VersionedBeaconStat if payload == nil || payload.Message == nil { return nil } - return processExecutionPayload(state, payload.Message, st.specs) + return st.processExecutionPayload(state, payload.Message) } // PrepareEpochPreState advances a post-block state to the pre-state of the target epoch. func (st *StateTransition) PrepareEpochPreState(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, info *TransitionInfo) error { - return prepareEpochPreStateInternal(state, epoch, payload, st.specs, st.dynSsz, info, st.caches) -} - -// TransitionInfo collects metadata from the state transition that callers may -// need for downstream processing. Pass a non-nil pointer to PrepareEpochPreState -// to receive this information; pass nil if not needed. -type TransitionInfo struct { - // DelayedBuilderPayments is the number of delayed builder payments appended - // to BuilderPendingWithdrawals by the last epoch transition's - // process_builder_pending_payments. This tells the state simulator how many - // entries at the tail of the queue are delayed (vs direct payments from block payloads). - DelayedBuilderPayments uint32 -} - -// PrepareEpochPreState is the standalone entry point (creates fresh caches). -// Prefer StateTransition.PrepareEpochPreState for repeated use. -// -// The input state is typically the post-state of the last block of a parent epoch. -// The function: -// 1. For Gloas+: if the state is pre-payload and a payload envelope is provided, -// applies the execution payload transition first. -// 2. Advances the state to the first slot of the target epoch, applying epoch -// transitions at every epoch boundary crossed (handles skipped slots and epochs). -// -// After this call, the state represents the pre-block state at the first slot of -// the target epoch, with all epoch transitions applied — including builder payment -// conversions, balance updates, proposer lookahead, etc. -// -// If info is non-nil, it is populated with metadata from the transition. -func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0.Epoch, payload *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec, ds *dynssz.DynSsz, info *TransitionInfo, caches *stateTransitionCaches) error { if state.Version < spec.DataVersionFulu { return nil } // Step 1: For Gloas+ pre-payload states, apply the execution payload transition. if payload != nil && IsPrePayloadState(state) { - if err := processExecutionPayload(state, payload, specs); err != nil { + if err := st.processExecutionPayload(state, payload); err != nil { return fmt.Errorf("process_execution_payload: %w", err) } } // Step 2: Advance to the first slot of the target epoch. - targetSlot := phase0.Slot(uint64(epoch) * specs.SlotsPerEpoch) - if err := processSlots(state, targetSlot, specs, ds, info, caches); err != nil { + targetSlot := phase0.Slot(uint64(epoch) * st.specs.SlotsPerEpoch) + if err := st.processSlots(state, targetSlot, info); err != nil { return fmt.Errorf("process_slots to epoch %d (slot %d): %w", epoch, targetSlot, err) } return nil } +// TransitionInfo collects metadata from the state transition that callers may +// need for downstream processing. Pass a non-nil pointer to PrepareEpochPreState +// to receive this information; pass nil if not needed. +type TransitionInfo struct { + // DelayedBuilderPayments is the number of delayed builder payments appended + // to BuilderPendingWithdrawals by the last epoch transition's + // process_builder_pending_payments. This tells the state simulator how many + // entries at the tail of the queue are delayed (vs direct payments from block payloads). + DelayedBuilderPayments uint32 +} + // processSlots advances the state from its current slot to targetSlot, applying // epoch transitions at every epoch boundary crossed. // @@ -138,7 +119,7 @@ func prepareEpochPreStateInternal(state *spec.VersionedBeaconState, epoch phase0 // outputs we need. Jumps directly to each epoch boundary. // // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#process_slots -func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, specs *consensus.ChainSpec, ds *dynssz.DynSsz, info *TransitionInfo, caches *stateTransitionCaches) error { +func (st *StateTransition) processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, info *TransitionInfo) error { currentSlot, err := state.Slot() if err != nil { return fmt.Errorf("failed to get state slot: %w", err) @@ -148,12 +129,12 @@ func processSlots(state *spec.VersionedBeaconState, targetSlot phase0.Slot, spec return nil } - s, err := newStateAccessorWithCaches(state, specs, ds, caches) + s, err := st.newAccessor(state) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } - slotsPerEpoch := specs.SlotsPerEpoch + slotsPerEpoch := st.specs.SlotsPerEpoch for s.Slot < targetSlot { processSlotBlockRootCaching(s) @@ -228,12 +209,12 @@ func IsPrePayloadState(state *spec.VersionedBeaconState) bool { // payment, transitioning the state from pre-payload to post-payload. // // New in Gloas: https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/beacon-chain.md#modified-process_execution_payload -func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.ExecutionPayloadEnvelope, specs *consensus.ChainSpec) error { +func (st *StateTransition) processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.ExecutionPayloadEnvelope) error { if state.Version < spec.DataVersionGloas || state.Gloas == nil || envelope == nil { return nil } - s, err := newStateAccessor(state, specs, nil) + s, err := st.newAccessor(state) if err != nil { return fmt.Errorf("failed to create state accessor: %w", err) } @@ -270,7 +251,7 @@ func processExecutionPayload(state *spec.VersionedBeaconState, envelope *gloas.E } // Queue the builder payment (direct withdrawal for delivered payload). - slotsPerEpoch := specs.SlotsPerEpoch + slotsPerEpoch := st.specs.SlotsPerEpoch paymentIdx := slotsPerEpoch + uint64(s.Slot)%slotsPerEpoch if paymentIdx < uint64(len(s.BuilderPendingPayments)) { payment := s.BuilderPendingPayments[paymentIdx] diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 4c5cd299a..91fcc3cff 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -318,6 +318,26 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override } } + // Extract execution payload bid from Gloas blocks and add to bid cache. + // This ensures bids are persisted even when syncing from blocks (not just SSE events). + if blockBody.Version >= spec.DataVersionGloas && blockBody.Gloas != nil && + blockBody.Gloas.Message != nil && blockBody.Gloas.Message.Body != nil && + blockBody.Gloas.Message.Body.SignedExecutionPayloadBid != nil && + blockBody.Gloas.Message.Body.SignedExecutionPayloadBid.Message != nil { + bidMsg := blockBody.Gloas.Message.Body.SignedExecutionPayloadBid.Message + dbw.indexer.blockBidCache.AddBid(&dbtypes.BlockBid{ + ParentRoot: bidMsg.ParentBlockRoot[:], + ParentHash: bidMsg.ParentBlockHash[:], + BlockHash: bidMsg.BlockHash[:], + FeeRecipient: bidMsg.FeeRecipient[:], + GasLimit: uint64(bidMsg.GasLimit), + BuilderIndex: int64(bidMsg.BuilderIndex), + Slot: uint64(bidMsg.Slot), + Value: uint64(bidMsg.Value), + ElPayment: uint64(bidMsg.ExecutionPayment), + }) + } + dbBlock := dbtypes.Slot{ Slot: uint64(block.header.Message.Slot), Proposer: uint64(block.header.Message.ProposerIndex), diff --git a/services/chainservice_builder.go b/services/chainservice_builder.go index 49a4c9162..dcdfd6f34 100644 --- a/services/chainservice_builder.go +++ b/services/chainservice_builder.go @@ -251,6 +251,11 @@ func (bs *ChainService) GetBuilderByIndex(index gloas.BuilderIndex) *gloas.Build return bs.beaconIndexer.GetBuilderByIndex(index, nil) } +// GetBuilderBalances returns the current builder balances (epoch-start adjusted for in-epoch withdrawals). +func (bs *ChainService) GetBuilderBalances() []phase0.Gwei { + return bs.beaconIndexer.GetRecentBuilderBalances(nil) +} + // getBuilderStatus determines the status of a builder func getBuilderStatus(builder *gloas.Builder, currentEpoch phase0.Epoch, superseded bool) dbtypes.BuilderStatus { if superseded { From 27e147e7a1b73e5f084e62386d3807d2219d23be Mon Sep 17 00:00:00 2001 From: pk910 Date: Sat, 11 Apr 2026 04:57:44 +0200 Subject: [PATCH 94/99] fix staticcheck issues --- indexer/beacon/block_helper.go | 48 -------------------- indexer/beacon/epochstate.go | 14 ------ indexer/beacon/statetransition/committees.go | 31 ++----------- statecache/statecache.go | 26 ----------- 4 files changed, 3 insertions(+), 116 deletions(-) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 5452be764..98ce2b7c8 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -721,54 +721,6 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator } } -// getLatestBlockHeaderParentRoot returns the parent root from the latest block header in the state. -func getLatestBlockHeaderParentRoot(v *spec.VersionedBeaconState) (phase0.Root, error) { - switch v.Version { - case spec.DataVersionPhase0: - return phase0.Root{}, errors.New("no latest block header parent root in phase0 state") - case spec.DataVersionAltair: - return phase0.Root{}, errors.New("no latest block header parent root in altair state") - case spec.DataVersionBellatrix: - if v.Bellatrix == nil || v.Bellatrix.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no bellatrix state") - } - - return v.Bellatrix.LatestBlockHeader.ParentRoot, nil - case spec.DataVersionCapella: - if v.Capella == nil || v.Capella.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no capella state") - } - - return v.Capella.LatestBlockHeader.ParentRoot, nil - case spec.DataVersionDeneb: - if v.Deneb == nil || v.Deneb.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no deneb state") - } - - return v.Deneb.LatestBlockHeader.ParentRoot, nil - case spec.DataVersionElectra: - if v.Electra == nil || v.Electra.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no electra state") - } - - return v.Electra.LatestBlockHeader.ParentRoot, nil - case spec.DataVersionFulu: - if v.Fulu == nil || v.Fulu.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no fulu state") - } - - return v.Fulu.LatestBlockHeader.ParentRoot, nil - case spec.DataVersionGloas: - if v.Gloas == nil || v.Gloas.LatestBlockHeader == nil { - return phase0.Root{}, errors.New("no gloas state") - } - - return v.Gloas.LatestBlockHeader.ParentRoot, nil - default: - return phase0.Root{}, errors.New("unknown version") - } -} - // getBlockSize returns the block size from a versioned beacon block. func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) (int, error) { switch block.Version { diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index e384186b6..51d98c951 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -530,17 +530,3 @@ func (s *epochState) tryReplayFromParentState( return parentState } - -// isGloasPostPayloadState checks whether the Gloas state is post-payload -// (i.e. execution payload deposits have been applied) for the given slot. -func isGloasPostPayloadState(state *spec.VersionedBeaconState, slot phase0.Slot) bool { - if state.Gloas == nil { - return false - } - bitfieldLen := uint64(len(state.Gloas.ExecutionPayloadAvailability)) * 8 - if bitfieldLen == 0 { - return false - } - idx := uint64(slot) % bitfieldLen - return state.Gloas.ExecutionPayloadAvailability[idx/8]&(1<<(idx%8)) != 0 -} diff --git a/indexer/beacon/statetransition/committees.go b/indexer/beacon/statetransition/committees.go index 73c58cd23..4d46507a9 100644 --- a/indexer/beacon/statetransition/committees.go +++ b/indexer/beacon/statetransition/committees.go @@ -16,22 +16,15 @@ type committeeKey struct { index uint64 } -// committeeCache caches computed beacon committees and shuffled permutations +// committeeCache caches computed beacon committees // to avoid recomputation across multiple attestations. type committeeCache struct { - cache map[committeeKey][]phase0.ValidatorIndex - shuffled map[shuffleKey][]phase0.ValidatorIndex // epoch seed → shuffled active indices -} - -type shuffleKey struct { - seed phase0.Root - count uint64 + cache map[committeeKey][]phase0.ValidatorIndex } func newCommitteeCache() *committeeCache { return &committeeCache{ - cache: make(map[committeeKey][]phase0.ValidatorIndex, 64), - shuffled: make(map[shuffleKey][]phase0.ValidatorIndex), + cache: make(map[committeeKey][]phase0.ValidatorIndex, 64), } } @@ -45,24 +38,6 @@ func (c *committeeCache) put(slot phase0.Slot, index uint64, committee []phase0. c.cache[committeeKey{slot: slot, index: index}] = committee } -// getShuffled returns a fully shuffled permutation of indices, cached by seed+count. -func (c *committeeCache) getShuffled(indices []phase0.ValidatorIndex, seed phase0.Root, specs *consensus.ChainSpec) []phase0.ValidatorIndex { - key := shuffleKey{seed: seed, count: uint64(len(indices))} - if cached, ok := c.shuffled[key]; ok { - return cached - } - - // Compute full shuffled permutation - indexCount := uint64(len(indices)) - shuffled := make([]phase0.ValidatorIndex, indexCount) - for i := uint64(0); i < indexCount; i++ { - shuffled[i] = indices[computeShuffledIndex(i, indexCount, seed, specs)] - } - - c.shuffled[key] = shuffled - return shuffled -} - // getCommitteeCountPerSlot returns the number of committees per slot for the given epoch. // https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md#get_committee_count_per_slot func (s *stateAccessor) getCommitteeCountPerSlot(epoch phase0.Epoch) uint64 { diff --git a/statecache/statecache.go b/statecache/statecache.go index e12fcccec..23a8e0fd1 100644 --- a/statecache/statecache.go +++ b/statecache/statecache.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "sync" @@ -70,31 +69,6 @@ func (k stateKey) filename() string { return fmt.Sprintf("%d_%s.ssz.gz", k.TargetEpoch, hex.EncodeToString(k.DependentRoot[:])) } -// parseFilename parses a cache filename back into a stateKey. -// Returns false if the filename doesn't match the expected format. -func parseFilename(name string) (stateKey, bool) { - name = strings.TrimSuffix(name, ".ssz.gz") - parts := strings.SplitN(name, "_", 2) - if len(parts) != 2 { - return stateKey{}, false - } - - epoch, err := strconv.ParseUint(parts[0], 10, 64) - if err != nil { - return stateKey{}, false - } - - rootBytes, err := hex.DecodeString(parts[1]) - if err != nil || len(rootBytes) != 32 { - return stateKey{}, false - } - - var key stateKey - key.TargetEpoch = phase0.Epoch(epoch) - copy(key.DependentRoot[:], rootBytes) - return key, true -} - // Check returns true if a cached state exists for the given key. func (sc *StateCache) Check(dependentRoot phase0.Root, targetEpoch phase0.Epoch) bool { if sc == nil { From 01ecf1b337f431f325903f0f3ed53e8c785dc23b Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 12 Apr 2026 02:59:41 +0200 Subject: [PATCH 95/99] add builder column & filter to slot lists, various fixes for builder details page --- db/block_bids.go | 44 +++++ handlers/blocks.go | 14 ++ handlers/builder.go | 99 ++++++++--- handlers/slots.go | 14 ++ handlers/slots_filtered.go | 38 ++++- handlers/validator.go | 2 +- templates/_shared/txDetailsModal.html | 167 +++++++++++++++++++ templates/blocks/blocks.html | 8 + templates/builder/builder.html | 3 + templates/builder/recentBlocks.html | 7 + templates/builder/recentDeposits.html | 33 +++- templates/slots/slots.html | 8 + templates/slots_filtered/slots_filtered.html | 17 ++ templates/validator/validator.html | 6 +- types/models/blocks.go | 4 + types/models/builders.go | 26 ++- types/models/slots.go | 4 + types/models/slots_filtered.go | 5 + 18 files changed, 460 insertions(+), 39 deletions(-) create mode 100644 templates/_shared/txDetailsModal.html diff --git a/db/block_bids.go b/db/block_bids.go index 7b2b9c6ca..4e44772d5 100644 --- a/db/block_bids.go +++ b/db/block_bids.go @@ -155,6 +155,50 @@ func GetBidsByBlockHashes(ctx context.Context, blockHashes [][]byte, builderInde return result } +// GetBidsBySlots returns the highest-value bid for each slot for a specific builder. +// Returns a map keyed by slot number for easy lookup. +func GetBidsBySlots(ctx context.Context, slots []uint64, builderIndex int64) map[uint64]*dbtypes.BlockBid { + result := make(map[uint64]*dbtypes.BlockBid, len(slots)) + if len(slots) == 0 { + return result + } + + var sql strings.Builder + args := make([]any, 0, len(slots)+1) + + fmt.Fprint(&sql, ` + SELECT + parent_root, parent_hash, block_hash, fee_recipient, gas_limit, builder_index, slot, value, el_payment + FROM block_bids + WHERE builder_index = $1 AND slot IN (`) + + args = append(args, builderIndex) + for i, slot := range slots { + if i > 0 { + fmt.Fprint(&sql, ", ") + } + fmt.Fprintf(&sql, "$%d", i+2) + args = append(args, slot) + } + fmt.Fprint(&sql, ") ORDER BY value DESC") + + bids := []*dbtypes.BlockBid{} + err := ReaderDb.SelectContext(ctx, &bids, sql.String(), args...) + if err != nil { + logger.Errorf("Error while fetching bids by slots: %v", err) + return result + } + + for _, bid := range bids { + // Keep only the highest-value bid per slot + if _, exists := result[bid.Slot]; !exists { + result[bid.Slot] = bid + } + } + + return result +} + // GetBidsByBuilderIndex returns bids submitted by a specific builder, ordered by slot descending func GetBidsByBuilderIndex(ctx context.Context, builderIndex uint64, offset uint64, limit uint32) ([]*dbtypes.BlockBid, uint64) { var sql strings.Builder diff --git a/handlers/blocks.go b/handlers/blocks.go index f6d998425..259de0204 100644 --- a/handlers/blocks.go +++ b/handlers/blocks.go @@ -114,6 +114,7 @@ func buildBlocksPageData(ctx context.Context, firstSlot uint64, pageSize uint64, 17: true, 18: false, 19: false, + 20: false, // Builder (hidden by default) } } @@ -148,6 +149,7 @@ func buildBlocksPageData(ctx context.Context, firstSlot uint64, pageSize uint64, pageData.DisplayBlockSize = displayMap[17] pageData.DisplayRecvDelay = displayMap[18] pageData.DisplayExecTime = displayMap[19] + pageData.DisplayBuilder = displayMap[20] pageData.DisplayColCount = uint64(len(displayMap)) chainState := services.GlobalBeaconService.GetChainState() @@ -298,6 +300,18 @@ func buildBlocksPageData(ctx context.Context, firstSlot uint64, pageSize uint64, } } + // Add builder info + if pageData.DisplayBuilder { + if dbSlot.BuilderIndex == -1 { + slotData.HasBuilder = true + slotData.BuilderIndex = math.MaxUint64 + } else if dbSlot.BuilderIndex >= 0 { + slotData.HasBuilder = true + slotData.BuilderIndex = uint64(dbSlot.BuilderIndex) + slotData.BuilderName = services.GlobalBeaconService.GetValidatorName(uint64(dbSlot.BuilderIndex) | services.BuilderIndexFlag) + } + } + // Add execution times if available if pageData.DisplayExecTime && dbSlot.MinExecTime > 0 && dbSlot.MaxExecTime > 0 { slotData.MinExecTime = dbSlot.MinExecTime diff --git a/handlers/builder.go b/handlers/builder.go index 6f9884fb5..ef8a14e72 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -33,6 +33,7 @@ func BuilderDetail(w http.ResponseWriter, r *http.Request) { "builder/recentBids.html", "builder/recentDeposits.html", "builder/withdrawals.html", + "_shared/txDetailsModal.html", "_svg/timeline.html", ) var notfoundTemplateFiles = append(layoutTemplateFiles, @@ -256,6 +257,9 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b switch tabView { case "blocks": pageData.RecentBlocks = buildBuilderRecentBlocks(ctx, builderIndex, chainState) + if len(pageData.RecentBlocks) >= 20 { + pageData.HasMoreBlocks = true + } case "bids": pageData.RecentBids = buildBuilderRecentBids(ctx, builderIndex, chainState) case "deposits": @@ -329,7 +333,7 @@ func buildBuilderPageData(ctx context.Context, builderIndex uint64, superseded b } func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBlock { - // Filter blocks by builder index using the new DB filter + // Filter blocks by builder index using the DB filter builderIndexInt64 := int64(builderIndex) filter := &dbtypes.BlockFilter{ BuilderIndex: &builderIndexInt64, @@ -337,13 +341,10 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta WithMissing: 0, // Exclude missing blocks } - // Get blocks built by this builder + // Get blocks built by this builder via chainservice (cache + DB) dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, filter, 0, 20, 0) - // Collect block hashes for batch bid lookup - blockHashes := make([][]byte, 0, len(dbBlocks)) validBlocks := make([]*dbtypes.Slot, 0, len(dbBlocks)) - for _, assignedSlot := range dbBlocks { if assignedSlot.Block == nil { continue @@ -356,13 +357,13 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta } if len(slot.EthBlockHash) > 0 { - blockHashes = append(blockHashes, slot.EthBlockHash) validBlocks = append(validBlocks, slot) } } - // Batch fetch all bids for these block hashes - bidsMap := db.GetBidsByBlockHashes(ctx, blockHashes, builderIndex) + // Look up bids via the indexer's bid accessor (checks in-memory cache first, then DB). + // Bids are keyed by parent block root, so we look up per block and match by block hash + builder. + indexer := services.GlobalBeaconService.GetBeaconIndexer() // Build result blocks := make([]*models.BuilderPageDataBlock, 0, len(validBlocks)) @@ -378,11 +379,16 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta GasLimit: slot.EthGasLimit, } - // Look up bid info for Value and ElPayment from the batch result - blockHashKey := fmt.Sprintf("%x", slot.EthBlockHash) - if bid, ok := bidsMap[blockHashKey]; ok { - block.Value = bid.Value - block.ElPayment = bid.ElPayment + // Look up bid by parent root, then match by block hash and builder index + var parentRoot phase0.Root + copy(parentRoot[:], slot.ParentRoot) + bids := indexer.GetBlockBids(parentRoot) + for _, bid := range bids { + if bid.BuilderIndex == builderIndexInt64 && fmt.Sprintf("%x", bid.BlockHash) == fmt.Sprintf("%x", slot.EthBlockHash) { + block.Value = bid.Value + block.ElPayment = bid.ElPayment + break + } } blocks = append(blocks, block) @@ -393,6 +399,43 @@ func buildBuilderRecentBlocks(ctx context.Context, builderIndex uint64, chainSta func buildBuilderRecentBids(ctx context.Context, builderIndex uint64, chainState *consensus.ChainState) []*models.BuilderPageDataBid { bids, _ := db.GetBidsByBuilderIndex(ctx, builderIndex, 0, 20) + if len(bids) == 0 { + return nil + } + + // Collect block hashes and determine slot range for batch lookup + bidBlockHashes := make(map[string]bool, len(bids)) + var minSlot, maxSlot uint64 + for i, bid := range bids { + bidBlockHashes[fmt.Sprintf("%x", bid.BlockHash)] = true + if i == 0 || bid.Slot > maxSlot { + maxSlot = bid.Slot + } + if i == 0 || bid.Slot < minSlot { + minSlot = bid.Slot + } + } + + // Batch fetch blocks for the slot range via chainservice (covers cache + DB) + canonicalBlockHashes := make(map[string]bool, len(bids)) + builderIndexInt64 := int64(builderIndex) + blockFilter := &dbtypes.BlockFilter{ + BuilderIndex: &builderIndexInt64, + MinSlot: &minSlot, + MaxSlot: &maxSlot, + WithOrphaned: 1, + WithMissing: 0, + } + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, 0, uint32(len(bids)*2), 0) + for _, assignedSlot := range dbBlocks { + if assignedSlot.Block == nil { + continue + } + hashKey := fmt.Sprintf("%x", assignedSlot.Block.EthBlockHash) + if bidBlockHashes[hashKey] && assignedSlot.Block.PayloadStatus == dbtypes.PayloadStatusCanonical { + canonicalBlockHashes[hashKey] = true + } + } result := make([]*models.BuilderPageDataBid, 0, len(bids)) for _, bid := range bids { @@ -406,16 +449,7 @@ func buildBuilderRecentBids(ctx context.Context, builderIndex uint64, chainState GasLimit: bid.GasLimit, Value: bid.Value, ElPayment: bid.ElPayment, - IsWinning: false, - } - - // Check if this bid won (payload was included) - slots := db.GetSlotsByBlockHash(ctx, bid.BlockHash) - for _, slot := range slots { - if slot.PayloadStatus == dbtypes.PayloadStatusCanonical { - bidData.IsWinning = true - break - } + IsWinning: canonicalBlockHashes[fmt.Sprintf("%x", bid.BlockHash)], } result = append(result, bidData) @@ -437,7 +471,9 @@ func buildBuilderRecentDeposits(ctx context.Context, pubkey []byte, chainState * deposits, _ := services.GlobalBeaconService.GetDepositRequestsByFilter(ctx, depositFilter, 0, 20) for _, deposit := range deposits { entry := &models.BuilderPageDataDeposit{ - Type: "deposit", + Type: "deposit", + Amount: deposit.Amount(), + DepositorAddress: deposit.SourceAddress(), } if deposit.Request != nil { entry.SlotNumber = deposit.Request.SlotNumber @@ -447,6 +483,21 @@ func buildBuilderRecentDeposits(ctx context.Context, pubkey []byte, chainState * } else if deposit.Transaction != nil { entry.Time = chainState.SlotToTime(phase0.Slot(deposit.Transaction.BlockTime)) } + + // Add transaction details if available + if deposit.Transaction != nil { + entry.HasTransaction = true + entry.TransactionHash = deposit.Transaction.TxHash + entry.TransactionDetails = &models.BuilderPageDataDepositTxDetails{ + BlockNumber: deposit.Transaction.BlockNumber, + BlockHash: fmt.Sprintf("%#x", deposit.Transaction.BlockRoot), + BlockTime: deposit.Transaction.BlockTime, + TxOrigin: common.Address(deposit.Transaction.TxSender).Hex(), + TxTarget: common.Address(deposit.Transaction.TxTarget).Hex(), + TxHash: fmt.Sprintf("%#x", deposit.Transaction.TxHash), + } + } + result = append(result, entry) } diff --git a/handlers/slots.go b/handlers/slots.go index 052400b3e..d7ab749f2 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -119,6 +119,7 @@ func buildSlotsPageData(ctx context.Context, firstSlot uint64, pageSize uint64, 17: false, 18: !hasSnooperClients, // Disable receive delay if snooper clients exist 19: hasSnooperClients, // Enable exec time if snooper clients exist + 20: false, // Builder (hidden by default) } } @@ -153,6 +154,7 @@ func buildSlotsPageData(ctx context.Context, firstSlot uint64, pageSize uint64, pageData.DisplayBlockSize = displayMap[17] pageData.DisplayRecvDelay = displayMap[18] pageData.DisplayExecTime = displayMap[19] + pageData.DisplayBuilder = displayMap[20] pageData.DisplayColCount = uint64(len(displayMap)) chainState := services.GlobalBeaconService.GetChainState() @@ -307,6 +309,18 @@ func buildSlotsPageData(ctx context.Context, firstSlot uint64, pageSize uint64, } } + // Add builder info + if pageData.DisplayBuilder { + if dbSlot.BuilderIndex == -1 { + slotData.HasBuilder = true + slotData.BuilderIndex = math.MaxUint64 + } else if dbSlot.BuilderIndex >= 0 { + slotData.HasBuilder = true + slotData.BuilderIndex = uint64(dbSlot.BuilderIndex) + slotData.BuilderName = services.GlobalBeaconService.GetValidatorName(uint64(dbSlot.BuilderIndex) | services.BuilderIndexFlag) + } + } + // Add execution times if available if pageData.DisplayExecTime && dbSlot.MinExecTime > 0 && dbSlot.MaxExecTime > 0 { slotData.MinExecTime = dbSlot.MinExecTime diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 83d3d042a..6fbd77994 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -3,6 +3,7 @@ package handlers import ( "context" "fmt" + "math" "net/http" "net/url" "strconv" @@ -63,6 +64,7 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { var forkIds string var minEpoch string var maxEpoch string + var builder string if urlArgs.Has("f") { if urlArgs.Has("f.graffiti") { @@ -125,11 +127,14 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { if urlArgs.Has("f.maxepoch") { maxEpoch = urlArgs.Get("f.maxepoch") } + if urlArgs.Has("f.builder") { + builder = urlArgs.Get("f.builder") + } } var pageError error pageError = services.GlobalCallRateLimiter.CheckCallLimit(r, 2) if pageError == nil { - data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, uint8(statusMask), uint8(payloadMask), minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, uint8(statusMask), uint8(payloadMask), minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, builder, displayColumns) } if pageError != nil { handlePageError(w, r, pageError) @@ -141,11 +146,11 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) { } } -func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) (*models.SlotsFilteredPageData, error) { +func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, builder string, displayColumns uint64) (*models.SlotsFilteredPageData, error) { pageData := &models.SlotsFilteredPageData{} - pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, builder, displayColumns) pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { - return buildFilteredSlotsPageData(pageCall.CallCtx, pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, displayColumns) + return buildFilteredSlotsPageData(pageCall.CallCtx, pageIdx, pageSize, graffiti, invertgraffiti, extradata, invertextradata, proposer, pname, invertproposer, statusMask, payloadMask, minSyncAgg, maxSyncAgg, minExecTime, maxExecTime, minTxCount, maxTxCount, minBlobCount, maxBlobCount, forkIds, minEpoch, maxEpoch, builder, displayColumns) }) if pageErr == nil && pageRes != nil { resData, resOk := pageRes.(*models.SlotsFilteredPageData) @@ -157,7 +162,7 @@ func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, return pageData, pageErr } -func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, displayColumns uint64) *models.SlotsFilteredPageData { +func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize uint64, graffiti string, invertgraffiti bool, extradata string, invertextradata bool, proposer string, pname string, invertproposer bool, statusMask uint8, payloadMask uint8, minSyncAgg string, maxSyncAgg string, minExecTime string, maxExecTime string, minTxCount string, maxTxCount string, minBlobCount string, maxBlobCount string, forkIds string, minEpoch string, maxEpoch string, builder string, displayColumns uint64) *models.SlotsFilteredPageData { chainState := services.GlobalBeaconService.GetChainState() filterArgs := url.Values{} if graffiti != "" { @@ -220,6 +225,9 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui if maxEpoch != "" { filterArgs.Add("f.maxepoch", maxEpoch) } + if builder != "" { + filterArgs.Add("f.builder", builder) + } // Check if snooper clients are configured hasSnooperClients := false @@ -313,6 +321,7 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui FilterForkIds: forkIds, FilterMinEpoch: minEpoch, FilterMaxEpoch: maxEpoch, + FilterBuilder: builder, DisplayEpoch: displayMap[1], DisplaySlot: displayMap[2], @@ -332,6 +341,7 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui DisplayBlockSize: displayMap[16], DisplayRecvDelay: displayMap[17], DisplayExecTime: displayMap[18], + DisplayBuilder: displayMap[19], DisplayColCount: uint64(len(displayMap)), HasSnooperClients: hasSnooperClients, @@ -458,6 +468,12 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui blockFilter.MaxEpoch = &maxEp } } + if builder != "" { + builderIdx, err := strconv.ParseInt(builder, 10, 64) + if err == nil { + blockFilter.BuilderIndex = &builderIdx + } + } dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(ctx, blockFilter, pageIdx, uint32(pageSize), 16) mevBlocksMap := make(map[string]*dbtypes.MevBlock) @@ -542,6 +558,18 @@ func buildFilteredSlotsPageData(ctx context.Context, pageIdx uint64, pageSize ui } } + // Add builder info + if pageData.DisplayBuilder { + if dbBlock.Block.BuilderIndex == -1 { + slotData.HasBuilder = true + slotData.BuilderIndex = math.MaxUint64 // Self-built sentinel + } else if dbBlock.Block.BuilderIndex >= 0 { + slotData.HasBuilder = true + slotData.BuilderIndex = uint64(dbBlock.Block.BuilderIndex) + slotData.BuilderName = services.GlobalBeaconService.GetValidatorName(uint64(dbBlock.Block.BuilderIndex) | services.BuilderIndexFlag) + } + } + // Add execution times if available if pageData.DisplayExecTime && dbBlock.Block.MinExecTime > 0 && dbBlock.Block.MaxExecTime > 0 { slotData.MinExecTime = dbBlock.Block.MinExecTime diff --git a/handlers/validator.go b/handlers/validator.go index a8663a2b2..a8825613d 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -35,7 +35,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { "validator/withdrawalRequests.html", "validator/consolidationRequests.html", "validator/withdrawals.html", - "validator/txDetails.html", + "_shared/txDetailsModal.html", "_svg/timeline.html", ) var notfoundTemplateFiles = append(layoutTemplateFiles, diff --git a/templates/_shared/txDetailsModal.html b/templates/_shared/txDetailsModal.html new file mode 100644 index 000000000..a283f9928 --- /dev/null +++ b/templates/_shared/txDetailsModal.html @@ -0,0 +1,167 @@ +{{ define "txDetailsModal" }} + +
    +
    +
    +
    + Tx Hash: +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + Block: +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + Block Time: +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + TX Origin: +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + TX Target: +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + +{{ end }} +{{ define "txDetailsModal-js" }} + +{{ end }} +{{ define "txDetailsModal-css" }} + +{{ end }} diff --git a/templates/blocks/blocks.html b/templates/blocks/blocks.html index e1e413b5d..0c3001e1f 100644 --- a/templates/blocks/blocks.html +++ b/templates/blocks/blocks.html @@ -48,6 +48,7 @@

    Blocks

    + @@ -88,6 +89,7 @@

    Blocks

    {{ if .DisplayTxCount }}Txs / Blobs{{ end }} {{ if .DisplayGasUsage }}Gas Usage{{ end }} {{ if .DisplaySyncAgg }}Sync Agg %{{ end }} + {{ if .DisplayBuilder }}Builder{{ end }} {{ if .DisplayMevBlock }}MEV Block{{ end }} {{ if .DisplayGasLimit }}Gas Limit{{ end }} {{ if .DisplayBlockSize }}Block Size{{ end }} @@ -162,6 +164,11 @@

    Blocks

    {{ end }}{{ end }} {{ if $g.DisplaySyncAgg }}{{ if not (eq $slot.Status 0) }}{{ formatPercentageAlert $slot.SyncParticipation 2 95 80 }}{{ end }}{{ end }} + {{ if $g.DisplayBuilder }}{{ if not (eq $slot.Status 0) }} + {{ if $slot.HasBuilder }} + {{ formatBuilder $slot.BuilderIndex $slot.BuilderName }} + {{ end }} + {{ end }}{{ end }} {{ if $g.DisplayMevBlock }}{{ if not (eq $slot.Status 0) }} {{ if $slot.IsMevBlock }} Yes @@ -190,6 +197,7 @@

    Blocks

    {{ if $g.DisplayTxCount }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayGasUsage }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplaySyncAgg }}{{ $colCount = add $colCount 1 }}{{ end }} + {{ if $g.DisplayBuilder }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayMevBlock }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayGasLimit }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayBlockSize }}{{ $colCount = add $colCount 1 }}{{ end }} diff --git a/templates/builder/builder.html b/templates/builder/builder.html index 940adfefa..dd6e44365 100644 --- a/templates/builder/builder.html +++ b/templates/builder/builder.html @@ -190,6 +190,7 @@

    Builder {{ form

    + {{ template "txDetailsModal" . }}
    • @@ -280,7 +281,9 @@

      Builder {{ form }); +{{ template "txDetailsModal-js" . }} {{ end }} {{ define "css" }} +{{ template "txDetailsModal-css" . }} {{ end }} diff --git a/templates/builder/recentBlocks.html b/templates/builder/recentBlocks.html index c03749e4a..128dde3f7 100644 --- a/templates/builder/recentBlocks.html +++ b/templates/builder/recentBlocks.html @@ -39,6 +39,13 @@ {{ formatEthFromGwei $block.Value }} {{ end }} + {{ if .HasMoreBlocks }} + + + View more blocks + + + {{ end }} {{ else }} diff --git a/templates/builder/recentDeposits.html b/templates/builder/recentDeposits.html index 5d676a8a1..2dbfd4d6a 100644 --- a/templates/builder/recentDeposits.html +++ b/templates/builder/recentDeposits.html @@ -8,6 +8,9 @@ Type Slot Time + Amount + Depositor + Transaction Status @@ -24,8 +27,36 @@ {{ $deposit.Type }} {{ end }} + {{ if $deposit.Orphaned }} + {{ formatAddCommas $deposit.SlotNumber }} + {{ else }} {{ formatAddCommas $deposit.SlotNumber }} + {{ end }} {{ formatRecentTimeShort $deposit.Time }} + {{ formatFullEthFromGwei $deposit.Amount }} + + {{ if $deposit.DepositorAddress }} + {{ ethAddressLink $deposit.DepositorAddress }} + + {{ else }} + ? + {{ end }} + + + {{- if $deposit.HasTransaction }} +
      + {{ ethTransactionLink $deposit.TransactionHash 0 }} +
      + +
      +
      + +
      +
      + {{- else }} + ? + {{- end }} + {{ if $deposit.Orphaned }} Orphaned @@ -38,7 +69,7 @@ {{ else }} - +
      {{ template "timeline_svg" }}
      diff --git a/templates/slots/slots.html b/templates/slots/slots.html index 6e97fbbfb..bb609d390 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -48,6 +48,7 @@

      Slots

      + @@ -88,6 +89,7 @@

      Slots

      {{ if .DisplayTxCount }}Txs / Blobs{{ end }} {{ if .DisplayGasUsage }}Gas Usage{{ end }} {{ if .DisplaySyncAgg }}Sync Agg %{{ end }} + {{ if .DisplayBuilder }}Builder{{ end }} {{ if .DisplayMevBlock }}MEV Block{{ end }} {{ if .DisplayGasLimit }}Gas Limit{{ end }} {{ if .DisplayBlockSize }}Block Size{{ end }} @@ -162,6 +164,11 @@

      Slots

    {{ end }}{{ end }} {{ if $g.DisplaySyncAgg }}{{ if not (eq $slot.Status 0) }}{{ formatPercentageAlert $slot.SyncParticipation 2 95 80 }}{{ end }}{{ end }} + {{ if $g.DisplayBuilder }}{{ if not (eq $slot.Status 0) }} + {{ if $slot.HasBuilder }} + {{ formatBuilder $slot.BuilderIndex $slot.BuilderName }} + {{ end }} + {{ end }}{{ end }} {{ if $g.DisplayMevBlock }}{{ if not (eq $slot.Status 0) }} {{ if $slot.IsMevBlock }} Yes @@ -190,6 +197,7 @@

    Slots

    {{ if $g.DisplayTxCount }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayGasUsage }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplaySyncAgg }}{{ $colCount = add $colCount 1 }}{{ end }} + {{ if $g.DisplayBuilder }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayMevBlock }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayGasLimit }}{{ $colCount = add $colCount 1 }}{{ end }} {{ if $g.DisplayBlockSize }}{{ $colCount = add $colCount 1 }}{{ end }} diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index a901e2eda..640e8431e 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -91,6 +91,14 @@

    Filtered Slots

    +
    +
    + Builder +
    +
    + +
    +
    @@ -236,6 +244,7 @@

    Filtered Slots

    + @@ -283,6 +292,7 @@

    Filtered Slots

    {{ if .DisplayTxCount }}Txs / Blobs{{ end }} {{ if .DisplayGasUsage }}Gas Usage{{ end }} {{ if .DisplaySyncAgg }}Sync Agg %{{ end }} + {{ if .DisplayBuilder }}Builder{{ end }} {{ if .DisplayMevBlock }}MEV Block{{ end }} {{ if .DisplayGasLimit }}Gas Limit{{ end }} {{ if .DisplayBlockSize }}Block Size{{ end }} @@ -357,6 +367,13 @@

    Filtered Slots

    {{- if $g.DisplaySyncAgg }} {{ if not (eq $slot.Status 0) }}{{ formatPercentageAlert $slot.SyncParticipation 2 95 80 }}{{ end }} {{- end }} + {{- if $g.DisplayBuilder }} + {{ if not (eq $slot.Status 0) }} + {{ if $slot.HasBuilder }} + {{ formatBuilder $slot.BuilderIndex $slot.BuilderName }} + {{ end }} + {{ end }} + {{- end }} {{- if $g.DisplayMevBlock }} {{ if not (eq $slot.Status 0) }} {{ if $slot.IsMevBlock }} diff --git a/templates/validator/validator.html b/templates/validator/validator.html index 62c3843d6..ea961bcaa 100644 --- a/templates/validator/validator.html +++ b/templates/validator/validator.html @@ -273,7 +273,7 @@

    Validator {{ forma {{ end }}

    - {{ template "txDetails" . }} + {{ template "txDetailsModal" . }} {{ end }} {{ define "lazyPage" }} @@ -322,9 +322,9 @@

    Validator {{ forma }); -{{ template "txDetails-js" . }} +{{ template "txDetailsModal-js" . }} {{ end }} {{ define "css" }} -{{ template "txDetails-css" . }} +{{ template "txDetailsModal-css" . }} {{ end }} \ No newline at end of file diff --git a/types/models/blocks.go b/types/models/blocks.go index 2937141d6..d828cc5f6 100644 --- a/types/models/blocks.go +++ b/types/models/blocks.go @@ -31,6 +31,7 @@ type BlocksPageData struct { DisplayBlockSize bool `json:"dp_blocksize"` DisplayRecvDelay bool `json:"dp_recvdelay"` DisplayExecTime bool `json:"dp_exectime"` + DisplayBuilder bool `json:"dp_builder"` DisplayColCount uint64 `json:"display_col_count"` IsDefaultPage bool `json:"default_page"` @@ -88,6 +89,9 @@ type BlocksPageDataSlot struct { ForkGraph []*BlocksPageDataForkGraph `json:"fork_graph"` IsMevBlock bool `json:"is_mev_block"` MevBlockRelays string `json:"mev_block_relays"` + HasBuilder bool `json:"has_builder"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` } type BlocksPageDataForkGraph struct { diff --git a/types/models/builders.go b/types/models/builders.go index 6ccc6b34c..89054e895 100644 --- a/types/models/builders.go +++ b/types/models/builders.go @@ -91,6 +91,7 @@ type BuilderPageData struct { Withdrawals []*BuilderPageDataWithdrawal `json:"withdrawals"` WithdrawalCount uint64 `json:"withdrawal_count"` AdditionalWithdrawalCount uint64 `json:"additional_withdrawal_count"` + HasMoreBlocks bool `json:"has_more_blocks"` } // BuilderPageDataBlock represents a block/payload built by this builder @@ -123,11 +124,26 @@ type BuilderPageDataBid struct { // BuilderPageDataDeposit represents a builder deposit transaction type BuilderPageDataDeposit struct { - Type string `json:"type"` // "deposit" - SlotNumber uint64 `json:"slot"` - SlotRoot []byte `json:"slot_root"` - Time time.Time `json:"time"` - Orphaned bool `json:"orphaned"` + Type string `json:"type"` // "deposit" + SlotNumber uint64 `json:"slot"` + SlotRoot []byte `json:"slot_root"` + Time time.Time `json:"time"` + Orphaned bool `json:"orphaned"` + Amount uint64 `json:"amount"` + DepositorAddress []byte `json:"depositor_address" ssz-size:"20"` + HasTransaction bool `json:"has_transaction"` + TransactionHash []byte `json:"tx_hash" ssz-size:"32"` + TransactionDetails *BuilderPageDataDepositTxDetails `json:"tx_details" ssz-type:"optional"` +} + +// BuilderPageDataDepositTxDetails contains transaction details for deposit transactions +type BuilderPageDataDepositTxDetails struct { + BlockNumber uint64 `json:"block"` + BlockHash string `json:"block_hash"` + BlockTime uint64 `json:"block_time"` + TxOrigin string `json:"tx_origin"` + TxTarget string `json:"tx_target"` + TxHash string `json:"tx_hash"` } // BuilderPageDataWithdrawal represents a withdrawal on the builder detail page diff --git a/types/models/slots.go b/types/models/slots.go index e3038321b..bd7cb04dd 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -31,6 +31,7 @@ type SlotsPageData struct { DisplayBlockSize bool `json:"dp_blocksize"` DisplayRecvDelay bool `json:"dp_recvdelay"` DisplayExecTime bool `json:"dp_exectime"` + DisplayBuilder bool `json:"dp_builder"` DisplayColCount uint64 `json:"display_col_count"` IsDefaultPage bool `json:"default_page"` @@ -89,6 +90,9 @@ type SlotsPageDataSlot struct { ForkGraph []*SlotsPageDataForkGraph `json:"fork_graph"` IsMevBlock bool `json:"is_mev_block"` MevBlockRelays string `json:"mev_block_relays"` + HasBuilder bool `json:"has_builder"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` } type SlotsPageDataForkGraph struct { diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index 0e8874aca..12263a745 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -30,6 +30,7 @@ type SlotsFilteredPageData struct { FilterForkIds string `json:"filter_fork_ids"` FilterMinEpoch string `json:"filter_min_epoch"` FilterMaxEpoch string `json:"filter_max_epoch"` + FilterBuilder string `json:"filter_builder"` DisplayEpoch bool `json:"dp_epoch"` DisplaySlot bool `json:"dp_slot"` @@ -49,6 +50,7 @@ type SlotsFilteredPageData struct { DisplayBlockSize bool `json:"dp_blocksize"` DisplayRecvDelay bool `json:"dp_recvdelay"` DisplayExecTime bool `json:"dp_exectime"` + DisplayBuilder bool `json:"dp_builder"` DisplayColCount uint64 `json:"display_col_count"` HasSnooperClients bool `json:"has_snooper_clients"` @@ -112,4 +114,7 @@ type SlotsFilteredPageDataSlot struct { ExecutionTimes []ExecutionTimeDetail `json:"execution_times"` IsMevBlock bool `json:"is_mev_block"` MevBlockRelays string `json:"mev_block_relays"` + HasBuilder bool `json:"has_builder"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` } From 317ce2858817958d763d4fc65cf85764799b37ec Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 12 Apr 2026 03:29:16 +0200 Subject: [PATCH 96/99] fix builder index rendering on address details page --- handlers/address.go | 17 ++-- indexer/beacon/indexer_getter.go | 118 ++++++++++++++++++++----- templates/address/system_deposits.html | 6 +- types/models/address.go | 1 + 4 files changed, 111 insertions(+), 31 deletions(-) diff --git a/handlers/address.go b/handlers/address.go index f5eded061..ac148ec74 100644 --- a/handlers/address.go +++ b/handlers/address.go @@ -878,12 +878,17 @@ func loadWithdrawalsTab(ctx context.Context, pageData *models.AddressPageData, a for _, w := range dbWithdrawals { slot := w.BlockUid >> 16 entry := &models.AddressPageDataWithdrawal{ - BlockUid: w.BlockUid, - BlockTime: chainState.SlotToTime(phase0.Slot(slot)), - Type: w.Type, - Amount: w.Amount, - ValidatorIndex: w.Validator, - ValidatorName: services.GlobalBeaconService.GetValidatorName(w.Validator), + BlockUid: w.BlockUid, + BlockTime: chainState.SlotToTime(phase0.Slot(slot)), + Type: w.Type, + Amount: w.Amount, + ValidatorName: services.GlobalBeaconService.GetValidatorName(w.Validator), + } + if w.Validator&services.BuilderIndexFlag != 0 { + entry.IsBuilder = true + entry.ValidatorIndex = w.Validator &^ services.BuilderIndexFlag + } else { + entry.ValidatorIndex = w.Validator } if blockInfo, ok := blockMap[w.BlockUid]; ok && blockInfo.Block != nil { diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 42876af77..7ec8491f3 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -545,8 +545,8 @@ func (indexer *Indexer) GetBuilderByIndex(index gloas.BuilderIndex, overrideFork } // GetRecentBuilderBalances returns the most recent builder balances for the given fork. -// Starts with epoch-boundary balances and deducts any builder withdrawals processed -// in blocks since the epoch start, reflecting the current head's state. +// Starts with epoch-boundary balances and replays in-epoch blocks to reflect live state: +// builder withdrawals/payments are deducted and builder deposits are credited. func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phase0.Gwei { chainState := indexer.consensusPool.GetChainState() @@ -590,33 +590,103 @@ func (indexer *Indexer) GetRecentBuilderBalances(overrideForkId *ForkKey) []phas balances := make([]phase0.Gwei, len(src)) copy(balances, src) - // Deduct builder withdrawals from blocks in the current epoch. - // Block UIDs encode the slot in the upper bits: uid = slot << 16. + // Walk the canonical chain from head back to epoch start, collecting blocks to replay. epochStartSlot := chainState.EpochToSlot(statsEpoch) - nextEpochSlot := chainState.EpochToSlot(statsEpoch + 1) - minBlockUid := uint64(epochStartSlot) << 16 - maxBlockUid := uint64(nextEpochSlot) << 16 - - withdrawals, err := db.GetWithdrawalsByBlockUidRange(indexer.ctx, minBlockUid, maxBlockUid, nil) - if err == nil { - for _, w := range withdrawals { - if w.Orphaned { - continue + head := indexer.GetCanonicalHead(overrideForkId) + + var epochBlocks []*Block + for block := head; block != nil && block.Slot >= epochStartSlot; { + epochBlocks = append(epochBlocks, block) + parentRoot := block.GetParentRoot() + if parentRoot == nil { + break + } + block = indexer.blockCache.getBlockByRoot(*parentRoot) + } + + // Replay in slot order (reverse the collected list). + isEip7732 := chainState.IsEip7732Enabled(chainState.EpochOfSlot(epochStartSlot)) + for i := len(epochBlocks) - 1; i >= 0; i-- { + block := epochBlocks[i] + indexer.applyBuilderBalanceChanges(block, balances, isEip7732) + } + + return balances +} + +// applyBuilderBalanceChanges extracts withdrawals and deposit requests from a +// cached block body and applies the corresponding builder balance changes. +func (indexer *Indexer) applyBuilderBalanceChanges(block *Block, balances []phase0.Gwei, isEip7732 bool) { + // Apply withdrawals (decrease builder balances). + if isEip7732 { + payload := block.GetExecutionPayload(indexer.ctx) + if payload != nil && payload.Message != nil && payload.Message.Payload != nil { + for _, w := range payload.Message.Payload.Withdrawals { + if uint64(w.ValidatorIndex)&BuilderIndexFlag == 0 { + continue + } + builderIdx := uint64(w.ValidatorIndex) &^ BuilderIndexFlag + if builderIdx < uint64(len(balances)) { + if balances[builderIdx] >= w.Amount { + balances[builderIdx] -= w.Amount + } else { + balances[builderIdx] = 0 + } + } } - // Builder withdrawals have the BuilderIndexFlag set in the Validator field. - if w.Validator&BuilderIndexFlag == 0 { - continue + + // Apply deposit requests (increase builder balances). + if payload.Message.ExecutionRequests != nil { + for _, deposit := range payload.Message.ExecutionRequests.Deposits { + if validatorIdx, found := indexer.pubkeyCache.Get(deposit.Pubkey); found { + idx := uint64(validatorIdx) + if idx&BuilderIndexFlag != 0 { + builderIdx := idx &^ BuilderIndexFlag + if builderIdx < uint64(len(balances)) { + balances[builderIdx] += phase0.Gwei(deposit.Amount) + } + } + } + } + } + } + } else { + blockBody := block.GetBlock(indexer.ctx) + if blockBody == nil { + return + } + + if execPayload, err := blockBody.ExecutionPayload(); err == nil && execPayload != nil { + if withdrawals, err := execPayload.Withdrawals(); err == nil { + for _, w := range withdrawals { + if uint64(w.ValidatorIndex)&BuilderIndexFlag == 0 { + continue + } + builderIdx := uint64(w.ValidatorIndex) &^ BuilderIndexFlag + if builderIdx < uint64(len(balances)) { + if balances[builderIdx] >= w.Amount { + balances[builderIdx] -= w.Amount + } else { + balances[builderIdx] = 0 + } + } + } } - builderIdx := w.Validator &^ BuilderIndexFlag - if builderIdx < uint64(len(balances)) { - if balances[builderIdx] >= phase0.Gwei(w.Amount) { - balances[builderIdx] -= phase0.Gwei(w.Amount) - } else { - balances[builderIdx] = 0 + } + + // Apply deposit requests (increase builder balances). + if requests, err := blockBody.ExecutionRequests(); err == nil && requests != nil { + for _, deposit := range requests.Deposits { + if validatorIdx, found := indexer.pubkeyCache.Get(deposit.Pubkey); found { + idx := uint64(validatorIdx) + if idx&BuilderIndexFlag != 0 { + builderIdx := idx &^ BuilderIndexFlag + if builderIdx < uint64(len(balances)) { + balances[builderIdx] += phase0.Gwei(deposit.Amount) + } + } } } } } - - return balances } diff --git a/templates/address/system_deposits.html b/templates/address/system_deposits.html index bcb9a055e..225cd3a8e 100644 --- a/templates/address/system_deposits.html +++ b/templates/address/system_deposits.html @@ -56,7 +56,11 @@ {{ end }} - {{ formatValidatorWithIndex .ValidatorIndex .ValidatorName }} + {{ if .IsBuilder }} + {{ formatBuilderWithIndex .ValidatorIndex .ValidatorName }} + {{ else }} + {{ formatValidatorWithIndex .ValidatorIndex .ValidatorName }} + {{ end }} {{ formatEthFromGweiP .Amount 6 }} diff --git a/types/models/address.go b/types/models/address.go index 55e34f61b..37130ca84 100644 --- a/types/models/address.go +++ b/types/models/address.go @@ -187,6 +187,7 @@ type AddressPageDataWithdrawal struct { Amount uint64 `json:"amount"` // Gwei ValidatorIndex uint64 `json:"validator_index"` ValidatorName string `json:"validator_name"` + IsBuilder bool `json:"is_builder"` } // AddressPageDataBlockFee represents a block fee reward on the address page. From 0fd4511e8ced7daa749c4d522f9005b6bd0dce44 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 14 Apr 2026 09:38:03 +0200 Subject: [PATCH 97/99] switch to ethpandaops/go-eth2-client --- blockdb/s3/format.go | 2 +- blockdb/s3/s3store.go | 2 +- clients/consensus/chainspec.go | 2 +- clients/consensus/chainstate.go | 4 ++-- clients/consensus/client.go | 6 +++--- clients/consensus/clientlogic.go | 6 +++--- clients/consensus/const.go | 2 +- clients/consensus/pool.go | 2 +- clients/consensus/rpc/beaconapi.go | 18 +++++++++--------- clients/consensus/rpc/beaconstream.go | 4 ++-- clients/consensus/rpc/syncstatus.go | 2 +- cmd/dora-utils/blockdb_sync.go | 2 +- db/slots.go | 2 +- db/validators.go | 2 +- dbtypes/other.go | 4 ++-- go.mod | 5 ++--- go.sum | 6 ++++-- handlers/address.go | 2 +- handlers/api/api_das_guardian.go | 2 +- handlers/api/api_das_guardian_mass.go | 2 +- handlers/api/consolidation_requests_v1.go | 2 +- handlers/api/deposits_included_v1.go | 2 +- handlers/api/deposits_queue_v1.go | 4 ++-- handlers/api/deposits_transactions_v1.go | 4 ++-- handlers/api/epoch_v1.go | 2 +- handlers/api/epochs_v1.go | 2 +- handlers/api/general.go | 4 ++-- handlers/api/mev_blocks_v1.go | 2 +- handlers/api/network_forks_v1.go | 2 +- handlers/api/network_overview_v1.go | 2 +- handlers/api/slashings_v1.go | 4 ++-- handlers/api/slots_v1.go | 2 +- handlers/api/validator_eth1_v1.go | 2 +- handlers/api/validator_names_v1.go | 2 +- handlers/api/validators_activity_v1.go | 2 +- handlers/api/validators_v1.go | 2 +- handlers/api/voluntary_exits_v1.go | 4 ++-- handlers/api/withdrawal_requests_v1.go | 2 +- handlers/blobs.go | 2 +- handlers/blocks.go | 2 +- handlers/blocks_filtered.go | 2 +- handlers/builder.go | 4 ++-- handlers/chain_forks.go | 2 +- handlers/clients_cl.go | 2 +- handlers/consolidations.go | 4 ++-- handlers/deposits.go | 6 +++--- handlers/el_consolidations.go | 2 +- handlers/el_withdrawals.go | 2 +- handlers/epoch.go | 2 +- handlers/epochs.go | 2 +- handlers/exits.go | 6 +++--- handlers/included_deposits.go | 4 ++-- handlers/index.go | 2 +- handlers/initiated_deposits.go | 4 ++-- handlers/mev_blocks.go | 2 +- handlers/queued_consolidations.go | 2 +- handlers/queued_deposits.go | 4 ++-- handlers/queued_withdrawals.go | 2 +- handlers/search.go | 2 +- handlers/slashings.go | 4 ++-- handlers/slot.go | 8 ++++---- handlers/slot_download.go | 4 ++-- handlers/slots.go | 2 +- handlers/slots_filtered.go | 2 +- handlers/submit_consolidation.go | 4 ++-- handlers/submit_deposit.go | 2 +- handlers/submit_withdrawal.go | 4 ++-- handlers/transaction.go | 2 +- handlers/validator.go | 4 ++-- handlers/validator_slots.go | 2 +- handlers/validators.go | 2 +- handlers/validators_activity.go | 2 +- handlers/validators_offline.go | 4 ++-- handlers/validators_summary.go | 2 +- handlers/voluntary_exits.go | 6 +++--- handlers/withdrawals.go | 4 ++-- handlers/withdrawals_list.go | 2 +- indexer/beacon/bidcache.go | 2 +- indexer/beacon/block.go | 6 +++--- indexer/beacon/block_helper.go | 16 ++++++++-------- indexer/beacon/blockcache.go | 2 +- indexer/beacon/buildercache.go | 4 ++-- indexer/beacon/canonical.go | 2 +- indexer/beacon/client.go | 8 ++++---- indexer/beacon/debug.go | 2 +- indexer/beacon/duties/duties.go | 2 +- indexer/beacon/epochcache.go | 2 +- indexer/beacon/epochstate.go | 8 ++++---- indexer/beacon/epochstats.go | 8 ++++---- indexer/beacon/epochstats_ssz.go | 2 +- indexer/beacon/epochstats_test.go | 2 +- indexer/beacon/epochvotes.go | 4 ++-- indexer/beacon/finalization.go | 8 ++++---- indexer/beacon/fork.go | 2 +- indexer/beacon/forkcache.go | 2 +- indexer/beacon/forkdetection.go | 2 +- indexer/beacon/indexer.go | 4 ++-- indexer/beacon/indexer_getter.go | 8 ++++---- indexer/beacon/precalc.go | 2 +- indexer/beacon/pruning.go | 2 +- indexer/beacon/pubkeycache.go | 2 +- indexer/beacon/requests.go | 6 +++--- indexer/beacon/state_sim.go | 6 +++--- indexer/beacon/statetransition/balance.go | 2 +- indexer/beacon/statetransition/block.go | 10 +++++----- indexer/beacon/statetransition/builder.go | 6 +++--- indexer/beacon/statetransition/committees.go | 4 ++-- indexer/beacon/statetransition/helpers.go | 2 +- .../beacon/statetransition/justification.go | 2 +- indexer/beacon/statetransition/lookahead.go | 2 +- indexer/beacon/statetransition/operations.go | 12 ++++++------ indexer/beacon/statetransition/pending.go | 4 ++-- indexer/beacon/statetransition/registry.go | 2 +- indexer/beacon/statetransition/resets.go | 4 ++-- indexer/beacon/statetransition/rewards.go | 2 +- indexer/beacon/statetransition/slashings.go | 2 +- indexer/beacon/statetransition/state.go | 14 +++++++------- .../beacon/statetransition/statetransition.go | 8 ++++---- indexer/beacon/statetransition/withdrawals.go | 8 ++++---- indexer/beacon/synchronizer.go | 6 +++--- indexer/beacon/validatoractivity.go | 2 +- indexer/beacon/validatorcache.go | 4 ++-- indexer/beacon/writedb.go | 10 +++++----- indexer/execution/indexerctx.go | 2 +- .../system_contracts/consolidation_indexer.go | 2 +- .../system_contracts/deposit_indexer.go | 2 +- .../system_contracts/withdrawal_indexer.go | 2 +- indexer/execution/txindexer/txindexer.go | 2 +- indexer/mevrelay/mevindexer.go | 2 +- indexer/snooper/snooper_manager.go | 2 +- services/chainservice.go | 4 ++-- services/chainservice_blocks.go | 8 ++++---- services/chainservice_builder.go | 4 ++-- services/chainservice_consolidations.go | 6 +++--- services/chainservice_deposits.go | 4 ++-- services/chainservice_epochs.go | 2 +- services/chainservice_objects.go | 2 +- services/chainservice_validators.go | 4 ++-- services/chainservice_withdrawals.go | 4 ++-- services/dasguardian.go | 4 ++-- services/validatornames.go | 2 +- statecache/statecache.go | 8 ++++---- types/models/search.go | 2 +- utils/beaconspec.go | 2 +- utils/blobs.go | 2 +- 145 files changed, 268 insertions(+), 267 deletions(-) diff --git a/blockdb/s3/format.go b/blockdb/s3/format.go index 79fa2c04a..378fc5e18 100644 --- a/blockdb/s3/format.go +++ b/blockdb/s3/format.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "fmt" - "github.com/attestantio/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec" "github.com/ethpandaops/dora/blockdb/types" ) diff --git a/blockdb/s3/s3store.go b/blockdb/s3/s3store.go index a3163a8b9..8fea67501 100644 --- a/blockdb/s3/s3store.go +++ b/blockdb/s3/s3store.go @@ -10,9 +10,9 @@ import ( "strings" "sync/atomic" - "github.com/attestantio/go-eth2-client/spec" "github.com/ethpandaops/dora/blockdb/types" dtypes "github.com/ethpandaops/dora/types" + "github.com/ethpandaops/go-eth2-client/spec" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 4713cc0b8..212b42cc4 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -8,7 +8,7 @@ import ( "sort" "sync" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "gopkg.in/Knetic/govaluate.v3" "gopkg.in/yaml.v2" ) diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index 63213c013..28295d3af 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -8,10 +8,10 @@ import ( "sync" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" "github.com/ethpandaops/ethwallclock" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) type ChainState struct { diff --git a/clients/consensus/client.go b/clients/consensus/client.go index ad514acdf..4b790aeb2 100644 --- a/clients/consensus/client.go +++ b/clients/consensus/client.go @@ -5,9 +5,9 @@ import ( "sync" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/consensus/rpc" diff --git a/clients/consensus/clientlogic.go b/clients/consensus/clientlogic.go index 84f63bb83..b65d219e5 100644 --- a/clients/consensus/clientlogic.go +++ b/clients/consensus/clientlogic.go @@ -7,9 +7,9 @@ import ( "runtime/debug" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/consensus/rpc" diff --git a/clients/consensus/const.go b/clients/consensus/const.go index b36b45ded..16f7b3966 100644 --- a/clients/consensus/const.go +++ b/clients/consensus/const.go @@ -1,5 +1,5 @@ package consensus -import "github.com/attestantio/go-eth2-client/spec/phase0" +import "github.com/ethpandaops/go-eth2-client/spec/phase0" var NullRoot phase0.Root = phase0.Root{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} diff --git a/clients/consensus/pool.go b/clients/consensus/pool.go index eb900ad79..a0616954e 100644 --- a/clients/consensus/pool.go +++ b/clients/consensus/pool.go @@ -6,9 +6,9 @@ import ( "math/rand/v2" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethpandaops/dora/utils" "github.com/ethpandaops/ethwallclock" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" ) diff --git a/clients/consensus/rpc/beaconapi.go b/clients/consensus/rpc/beaconapi.go index 7435764e1..e09a794d0 100644 --- a/clients/consensus/rpc/beaconapi.go +++ b/clients/consensus/rpc/beaconapi.go @@ -12,15 +12,15 @@ import ( "strings" "time" - eth2client "github.com/attestantio/go-eth2-client" - "github.com/attestantio/go-eth2-client/api" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/http" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + eth2client "github.com/ethpandaops/go-eth2-client" + "github.com/ethpandaops/go-eth2-client/api" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/http" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/deneb" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" diff --git a/clients/consensus/rpc/beaconstream.go b/clients/consensus/rpc/beaconstream.go index 6721c24dc..5bac7bf3f 100644 --- a/clients/consensus/rpc/beaconstream.go +++ b/clients/consensus/rpc/beaconstream.go @@ -9,8 +9,8 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/consensus/rpc/eventstream" diff --git a/clients/consensus/rpc/syncstatus.go b/clients/consensus/rpc/syncstatus.go index f52052674..9842166e7 100644 --- a/clients/consensus/rpc/syncstatus.go +++ b/clients/consensus/rpc/syncstatus.go @@ -1,6 +1,6 @@ package rpc -import v1 "github.com/attestantio/go-eth2-client/api/v1" +import v1 "github.com/ethpandaops/go-eth2-client/api/v1" type SyncStatus struct { IsSyncing bool diff --git a/cmd/dora-utils/blockdb_sync.go b/cmd/dora-utils/blockdb_sync.go index 7905d7e9f..d0a483685 100644 --- a/cmd/dora-utils/blockdb_sync.go +++ b/cmd/dora-utils/blockdb_sync.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" btypes "github.com/ethpandaops/dora/blockdb/types" "github.com/ethpandaops/dora/clients/consensus" @@ -13,6 +12,7 @@ import ( "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/types" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/db/slots.go b/db/slots.go index 4968f0802..574d3d56e 100644 --- a/db/slots.go +++ b/db/slots.go @@ -7,8 +7,8 @@ import ( "math" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/mitchellh/mapstructure" ) diff --git a/db/validators.go b/db/validators.go index 0c988b293..ade9bdc7b 100644 --- a/db/validators.go +++ b/db/validators.go @@ -6,8 +6,8 @@ import ( "math" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" ) diff --git a/dbtypes/other.go b/dbtypes/other.go index 32a410c53..b9f054e1a 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -1,8 +1,8 @@ package dbtypes import ( - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) type AssignedSlot struct { diff --git a/go.mod b/go.mod index d6d3618f4..8cf7ca837 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.25.1 require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/allegro/bigcache/v3 v3.1.0 - github.com/attestantio/go-eth2-client v0.28.0 github.com/cockroachdb/pebble v1.1.5 github.com/ethereum/go-ethereum v1.17.2 github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c github.com/ethpandaops/ethwallclock v0.4.0 + github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252 github.com/glebarez/go-sqlite v1.22.0 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v5 v5.3.1 @@ -52,6 +52,7 @@ require ( github.com/OffchainLabs/go-bitfield v0.0.0-20251031151322-f427d04d8506 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/VictoriaMetrics/fastcache v1.13.0 // indirect + github.com/attestantio/go-eth2-client v0.27.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/casbin/govaluate v1.10.0 // indirect github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 // indirect @@ -254,5 +255,3 @@ require ( modernc.org/memory v1.11.0 // indirect modernc.org/sqlite v1.46.1 // indirect ) - -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81 diff --git a/go.sum b/go.sum index ec1176d6b..0447f76c9 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk= github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/attestantio/go-eth2-client v0.27.1 h1:g7bm+gG/p+gfzYdEuxuAepVWYb8EO+2KojV5/Lo2BxM= +github.com/attestantio/go-eth2-client v0.27.1/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -118,6 +120,8 @@ github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c h1:uBRIitwcuCJ github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c/go.mod h1:QsmYTdesob+vQ6pW4KtRVvxLZUNop3cdtd/DgD30hJU= github.com/ethpandaops/ethwallclock v0.4.0 h1:+sgnhf4pk6hLPukP076VxkiLloE4L0Yk1yat+ZyHh1g= github.com/ethpandaops/ethwallclock v0.4.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24= +github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252 h1:cS54Q13E/dA6uFCgcuIufiYeU6ifZdtQPHMbqS3KuOU= +github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252/go.mod h1:9BBd/XIw1egZTkxtFGMvgXnsxX6ypKHKNKD7itqjmNQ= github.com/ferranbt/fastssz v1.0.0 h1:9EXXYsracSqQRBQiHeaVsG/KQeYblPf40hsQPb9Dzk8= github.com/ferranbt/fastssz v1.0.0/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= @@ -555,8 +559,6 @@ github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M= github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4= github.com/pk910/dynamic-ssz v1.3.1-0.20260407212738-e97de623fd84 h1:J3H3PiaO4+ej5HTK/nG/wnAj0jx+Ek2+0s8o+zujI4I= github.com/pk910/dynamic-ssz v1.3.1-0.20260407212738-e97de623fd84/go.mod h1:NmeFF4jxzVwWC8cnEhUB7xMI++8hd/0OZvZHFrUvFfs= -github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81 h1:mHgrerDiro/np9FCoJ19EYsCyl/CXMHFi34o+j3T+rE= -github.com/pk910/go-eth2-client v0.0.0-20260331085057-05aefaa5ce81/go.mod h1:lwj0l8l51hIjqdQpODPea01JfE33nyM++1VGjBZau08= github.com/pk910/hashtree-bindings v0.1.0 h1:w7NyRWFi2OaYEFvo9ADcE/QU6PMuVLl3hBgx92KiH9c= github.com/pk910/hashtree-bindings v0.1.0/go.mod h1:zrWt88783JmhBfcgni6kkIMYRdXTZi/FL//OyI5T/l4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/handlers/address.go b/handlers/address.go index ac148ec74..0992d597c 100644 --- a/handlers/address.go +++ b/handlers/address.go @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/api/api_das_guardian.go b/handlers/api/api_das_guardian.go index 8d85bfea1..0dde0f756 100644 --- a/handlers/api/api_das_guardian.go +++ b/handlers/api/api_das_guardian.go @@ -8,10 +8,10 @@ import ( "net/http" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dasguardian "github.com/probe-lab/eth-das-guardian" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/api_das_guardian_mass.go b/handlers/api/api_das_guardian_mass.go index d0afbd0b3..1c9e83b53 100644 --- a/handlers/api/api_das_guardian_mass.go +++ b/handlers/api/api_das_guardian_mass.go @@ -9,10 +9,10 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dasguardian "github.com/probe-lab/eth-das-guardian" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/consolidation_requests_v1.go b/handlers/api/consolidation_requests_v1.go index 607c3e2c8..bb0de78db 100644 --- a/handlers/api/consolidation_requests_v1.go +++ b/handlers/api/consolidation_requests_v1.go @@ -6,10 +6,10 @@ import ( "net/http" "strconv" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/deposits_included_v1.go b/handlers/api/deposits_included_v1.go index 7da4200a0..18c811d15 100644 --- a/handlers/api/deposits_included_v1.go +++ b/handlers/api/deposits_included_v1.go @@ -7,10 +7,10 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/deposits_queue_v1.go b/handlers/api/deposits_queue_v1.go index 2e7c5676d..754b29687 100644 --- a/handlers/api/deposits_queue_v1.go +++ b/handlers/api/deposits_queue_v1.go @@ -8,12 +8,12 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/deposits_transactions_v1.go b/handlers/api/deposits_transactions_v1.go index fc5bc3922..d147c4983 100644 --- a/handlers/api/deposits_transactions_v1.go +++ b/handlers/api/deposits_transactions_v1.go @@ -7,12 +7,12 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/epoch_v1.go b/handlers/api/epoch_v1.go index 4670be711..b1fe7e095 100644 --- a/handlers/api/epoch_v1.go +++ b/handlers/api/epoch_v1.go @@ -6,8 +6,8 @@ import ( "net/http" "strconv" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/epochs_v1.go b/handlers/api/epochs_v1.go index a0e6a7716..e02871cb5 100644 --- a/handlers/api/epochs_v1.go +++ b/handlers/api/epochs_v1.go @@ -5,9 +5,9 @@ import ( "net/http" "strconv" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/general.go b/handlers/api/general.go index b11b0be4c..2e9de798c 100644 --- a/handlers/api/general.go +++ b/handlers/api/general.go @@ -10,10 +10,10 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/mev_blocks_v1.go b/handlers/api/mev_blocks_v1.go index 5f6102fa4..3bc948e7c 100644 --- a/handlers/api/mev_blocks_v1.go +++ b/handlers/api/mev_blocks_v1.go @@ -7,11 +7,11 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/network_forks_v1.go b/handlers/api/network_forks_v1.go index 0ddfcce33..ebafd5ea7 100644 --- a/handlers/api/network_forks_v1.go +++ b/handlers/api/network_forks_v1.go @@ -6,9 +6,9 @@ import ( "net/http" "sort" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/network_overview_v1.go b/handlers/api/network_overview_v1.go index e26929e30..582ce75ef 100644 --- a/handlers/api/network_overview_v1.go +++ b/handlers/api/network_overview_v1.go @@ -7,7 +7,6 @@ import ( "net/http" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/clients/execution/rpc" @@ -15,6 +14,7 @@ import ( "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/slashings_v1.go b/handlers/api/slashings_v1.go index d613c3e7c..2b83f6ef9 100644 --- a/handlers/api/slashings_v1.go +++ b/handlers/api/slashings_v1.go @@ -7,10 +7,10 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/slots_v1.go b/handlers/api/slots_v1.go index 0945f0a8a..cc14abed9 100644 --- a/handlers/api/slots_v1.go +++ b/handlers/api/slots_v1.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/execution" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -16,6 +15,7 @@ import ( "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/validator_eth1_v1.go b/handlers/api/validator_eth1_v1.go index 100f460db..013ebd8e2 100644 --- a/handlers/api/validator_eth1_v1.go +++ b/handlers/api/validator_eth1_v1.go @@ -8,10 +8,10 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/validator_names_v1.go b/handlers/api/validator_names_v1.go index 0a09243ed..db09482e4 100644 --- a/handlers/api/validator_names_v1.go +++ b/handlers/api/validator_names_v1.go @@ -9,8 +9,8 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/validators_activity_v1.go b/handlers/api/validators_activity_v1.go index c6fd20be8..48d63370d 100644 --- a/handlers/api/validators_activity_v1.go +++ b/handlers/api/validators_activity_v1.go @@ -10,10 +10,10 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/validators_v1.go b/handlers/api/validators_v1.go index 78ed1e51e..abebf5240 100644 --- a/handlers/api/validators_v1.go +++ b/handlers/api/validators_v1.go @@ -8,9 +8,9 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/voluntary_exits_v1.go b/handlers/api/voluntary_exits_v1.go index 60e53a665..bff45babe 100644 --- a/handlers/api/voluntary_exits_v1.go +++ b/handlers/api/voluntary_exits_v1.go @@ -7,10 +7,10 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/api/withdrawal_requests_v1.go b/handlers/api/withdrawal_requests_v1.go index 6daed46fe..18e133972 100644 --- a/handlers/api/withdrawal_requests_v1.go +++ b/handlers/api/withdrawal_requests_v1.go @@ -6,11 +6,11 @@ import ( "net/http" "strconv" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/blobs.go b/handlers/blobs.go index aca3c16d7..a49d73074 100644 --- a/handlers/blobs.go +++ b/handlers/blobs.go @@ -5,12 +5,12 @@ import ( "net/http" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/blocks.go b/handlers/blocks.go index 259de0204..0fc8a2c54 100644 --- a/handlers/blocks.go +++ b/handlers/blocks.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/execution" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -20,6 +19,7 @@ import ( "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/blocks_filtered.go b/handlers/blocks_filtered.go index ddadef785..5838dbd25 100644 --- a/handlers/blocks_filtered.go +++ b/handlers/blocks_filtered.go @@ -8,13 +8,13 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/builder.go b/handlers/builder.go index ef8a14e72..2eec185a2 100644 --- a/handlers/builder.go +++ b/handlers/builder.go @@ -10,9 +10,9 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/chain_forks.go b/handlers/chain_forks.go index 6eae95cff..e218fb955 100644 --- a/handlers/chain_forks.go +++ b/handlers/chain_forks.go @@ -9,13 +9,13 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/clients_cl.go b/handlers/clients_cl.go index c4ec6acf7..f31773f7e 100644 --- a/handlers/clients_cl.go +++ b/handlers/clients_cl.go @@ -9,7 +9,6 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/clients/consensus/rpc" @@ -17,6 +16,7 @@ import ( "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" ) diff --git a/handlers/consolidations.go b/handlers/consolidations.go index 36b880569..56c20f358 100644 --- a/handlers/consolidations.go +++ b/handlers/consolidations.go @@ -9,12 +9,12 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/deposits.go b/handlers/deposits.go index bb83a0fa6..e662b3aa0 100644 --- a/handlers/deposits.go +++ b/handlers/deposits.go @@ -9,9 +9,6 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -19,6 +16,9 @@ import ( "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/el_consolidations.go b/handlers/el_consolidations.go index 85f7f7073..6c92d1336 100644 --- a/handlers/el_consolidations.go +++ b/handlers/el_consolidations.go @@ -8,7 +8,6 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/dbtypes" @@ -16,6 +15,7 @@ import ( "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/el_withdrawals.go b/handlers/el_withdrawals.go index 885287a93..42b130012 100644 --- a/handlers/el_withdrawals.go +++ b/handlers/el_withdrawals.go @@ -8,13 +8,13 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/epoch.go b/handlers/epoch.go index b9a1e2e90..9fb2e9b1d 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -7,7 +7,7 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/epochs.go b/handlers/epochs.go index af56d8ae2..61463bd4f 100644 --- a/handlers/epochs.go +++ b/handlers/epochs.go @@ -8,10 +8,10 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/exits.go b/handlers/exits.go index 2cff7d97f..a512da4a7 100644 --- a/handlers/exits.go +++ b/handlers/exits.go @@ -9,13 +9,13 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/included_deposits.go b/handlers/included_deposits.go index 18de1b461..afd89b5ca 100644 --- a/handlers/included_deposits.go +++ b/handlers/included_deposits.go @@ -10,13 +10,13 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/index.go b/handlers/index.go index f6902ee11..f84b28738 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -13,7 +13,6 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -22,6 +21,7 @@ import ( "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/initiated_deposits.go b/handlers/initiated_deposits.go index 09ed47b14..1e365f59a 100644 --- a/handlers/initiated_deposits.go +++ b/handlers/initiated_deposits.go @@ -9,14 +9,14 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/mev_blocks.go b/handlers/mev_blocks.go index 539356410..017d6fa05 100644 --- a/handlers/mev_blocks.go +++ b/handlers/mev_blocks.go @@ -8,13 +8,13 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/queued_consolidations.go b/handlers/queued_consolidations.go index a7c65ccab..9f03d51f3 100644 --- a/handlers/queued_consolidations.go +++ b/handlers/queued_consolidations.go @@ -10,11 +10,11 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" ) diff --git a/handlers/queued_deposits.go b/handlers/queued_deposits.go index dfe875a75..bd71ee482 100644 --- a/handlers/queued_deposits.go +++ b/handlers/queued_deposits.go @@ -10,14 +10,14 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // QueuedDeposits will return the "queued_deposits" page using a go template diff --git a/handlers/queued_withdrawals.go b/handlers/queued_withdrawals.go index 29c732712..cd6b5ceb8 100644 --- a/handlers/queued_withdrawals.go +++ b/handlers/queued_withdrawals.go @@ -9,11 +9,11 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" ) diff --git a/handlers/search.go b/handlers/search.go index 627d3a628..f941084d1 100644 --- a/handlers/search.go +++ b/handlers/search.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/slashings.go b/handlers/slashings.go index 48ab4e1c0..9e4fa322f 100644 --- a/handlers/slashings.go +++ b/handlers/slashings.go @@ -9,12 +9,12 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/slot.go b/handlers/slot.go index 615e63489..6fa6a38a4 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -14,11 +14,11 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/phase0" ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/bellatrix" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/slot_download.go b/handlers/slot_download.go index ec6b9288b..a905732e1 100644 --- a/handlers/slot_download.go +++ b/handlers/slot_download.go @@ -6,13 +6,13 @@ import ( "fmt" "net/http" - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/phase0" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethpandaops/dora/blockdb" bdbtypes "github.com/ethpandaops/dora/blockdb/types" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/services" + "github.com/ethpandaops/go-eth2-client/spec/bellatrix" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/golang/snappy" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/handlers/slots.go b/handlers/slots.go index d7ab749f2..155d27458 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" @@ -19,6 +18,7 @@ import ( "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 6fbd77994..a466c753e 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" @@ -17,6 +16,7 @@ import ( "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/submit_consolidation.go b/handlers/submit_consolidation.go index be07a297e..f5669465a 100644 --- a/handlers/submit_consolidation.go +++ b/handlers/submit_consolidation.go @@ -12,9 +12,9 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/consensus" diff --git a/handlers/submit_deposit.go b/handlers/submit_deposit.go index 72fc8eef4..7b2932b83 100644 --- a/handlers/submit_deposit.go +++ b/handlers/submit_deposit.go @@ -13,8 +13,8 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/ethereum/go-ethereum/common" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/db" diff --git a/handlers/submit_withdrawal.go b/handlers/submit_withdrawal.go index 8f958ddd1..9cbd473ff 100644 --- a/handlers/submit_withdrawal.go +++ b/handlers/submit_withdrawal.go @@ -9,9 +9,9 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/clients/execution/rpc" diff --git a/handlers/transaction.go b/handlers/transaction.go index 5865b69e1..eb5a86d58 100644 --- a/handlers/transaction.go +++ b/handlers/transaction.go @@ -12,9 +12,9 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/golang/snappy" "github.com/gorilla/mux" dynssz "github.com/pk910/dynamic-ssz" diff --git a/handlers/validator.go b/handlers/validator.go index a8825613d..0f0999c70 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -11,9 +11,9 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index d4cb6417d..19106ee5a 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -7,7 +7,7 @@ import ( "strconv" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/gorilla/mux" "github.com/sirupsen/logrus" diff --git a/handlers/validators.go b/handlers/validators.go index e23f94b41..c9c5c9e57 100644 --- a/handlers/validators.go +++ b/handlers/validators.go @@ -12,7 +12,7 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" diff --git a/handlers/validators_activity.go b/handlers/validators_activity.go index e58e65e4a..6487e536f 100644 --- a/handlers/validators_activity.go +++ b/handlers/validators_activity.go @@ -10,11 +10,11 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/validators_offline.go b/handlers/validators_offline.go index 94a8e2f0e..e058db87b 100644 --- a/handlers/validators_offline.go +++ b/handlers/validators_offline.go @@ -9,13 +9,13 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/validators_summary.go b/handlers/validators_summary.go index 21625694e..351735efe 100644 --- a/handlers/validators_summary.go +++ b/handlers/validators_summary.go @@ -9,7 +9,7 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/clients/execution" diff --git a/handlers/voluntary_exits.go b/handlers/voluntary_exits.go index 7c481dfc1..067249140 100644 --- a/handlers/voluntary_exits.go +++ b/handlers/voluntary_exits.go @@ -8,13 +8,13 @@ import ( "strconv" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/withdrawals.go b/handlers/withdrawals.go index 6a926e123..8937f61e1 100644 --- a/handlers/withdrawals.go +++ b/handlers/withdrawals.go @@ -9,13 +9,13 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/templates" "github.com/ethpandaops/dora/types/models" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/handlers/withdrawals_list.go b/handlers/withdrawals_list.go index 456445770..8860ac179 100644 --- a/handlers/withdrawals_list.go +++ b/handlers/withdrawals_list.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/db" diff --git a/indexer/beacon/bidcache.go b/indexer/beacon/bidcache.go index 3cdbe4bc5..d3e1c1329 100644 --- a/indexer/beacon/bidcache.go +++ b/indexer/beacon/bidcache.go @@ -3,9 +3,9 @@ package beacon import ( "sync" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" ) diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 1c47631f0..e66c40696 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -8,14 +8,14 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" btypes "github.com/ethpandaops/dora/blockdb/types" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 98ce2b7c8..24be5efec 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -4,15 +4,15 @@ import ( "errors" "fmt" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/altair" - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/altair" + "github.com/ethpandaops/go-eth2-client/spec/bellatrix" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/deneb" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/indexer/beacon/blockcache.go b/indexer/beacon/blockcache.go index f59cc4898..f5c79ac17 100644 --- a/indexer/beacon/blockcache.go +++ b/indexer/beacon/blockcache.go @@ -5,9 +5,9 @@ import ( "sort" "sync" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) var zeroHash = phase0.Hash32{} diff --git a/indexer/beacon/buildercache.go b/indexer/beacon/buildercache.go index 188c5aa6f..0a6bb62e9 100644 --- a/indexer/beacon/buildercache.go +++ b/indexer/beacon/buildercache.go @@ -9,8 +9,8 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/ethpandaops/dora/db" diff --git a/indexer/beacon/canonical.go b/indexer/beacon/canonical.go index 4467af3ad..aae2b11f8 100644 --- a/indexer/beacon/canonical.go +++ b/indexer/beacon/canonical.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) const FarFutureEpoch = phase0.Epoch(math.MaxUint64) diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 2d724c508..957e68b79 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -8,16 +8,16 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/clients/consensus/rpc" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/utils" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" ) diff --git a/indexer/beacon/debug.go b/indexer/beacon/debug.go index a23a306a9..7bfc5248c 100644 --- a/indexer/beacon/debug.go +++ b/indexer/beacon/debug.go @@ -4,7 +4,7 @@ import ( "reflect" "unsafe" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) type CacheDebugStats struct { diff --git a/indexer/beacon/duties/duties.go b/indexer/beacon/duties/duties.go index dd9647edc..ee313aeed 100644 --- a/indexer/beacon/duties/duties.go +++ b/indexer/beacon/duties/duties.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) const seedSize = int8(32) diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index 825c65b22..7e455e5f4 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/indexer/beacon/statetransition" diff --git a/indexer/beacon/epochstate.go b/indexer/beacon/epochstate.go index 51d98c951..1477f22eb 100644 --- a/indexer/beacon/epochstate.go +++ b/indexer/beacon/epochstate.go @@ -6,13 +6,13 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/indexer/beacon/statetransition" "github.com/ethpandaops/dora/statecache" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // epochState represents a beacon state which a epoch status depends on. diff --git a/indexer/beacon/epochstats.go b/indexer/beacon/epochstats.go index 7d82d6f1d..9ca945128 100644 --- a/indexer/beacon/epochstats.go +++ b/indexer/beacon/epochstats.go @@ -9,13 +9,13 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon/duties" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/mashingan/smapping" ) @@ -69,7 +69,7 @@ type EpochStatsValues struct { // EpochStatsPacked holds the packed values for the epoch-specific information. // // generate ssz: (this is really ugly, needs path patching and post-fixing to work) -// sszgen --suffix ssz --path . --include $GOPATH/pkg/mod/github.com/attestantio/go-eth2-client\@v0.26.0/spec/phase0,$GOPATH/pkg/mod/github.com/attestantio/go-eth2-client\@v0.26.0/spec/electra --objs EpochStatsPacked +// sszgen --suffix ssz --path . --include $GOPATH/pkg/mod/github.com/ethpandaops/go-eth2-client\@v0.26.0/spec/phase0,$GOPATH/pkg/mod/github.com/ethpandaops/go-eth2-client\@v0.26.0/spec/electra --objs EpochStatsPacked type EpochStatsPacked struct { ActiveValidators []EpochStatsPackedValidator `ssz-max:"10000000"` ProposerDuties []phase0.ValidatorIndex `ssz-max:"100"` diff --git a/indexer/beacon/epochstats_ssz.go b/indexer/beacon/epochstats_ssz.go index 93d393e0a..68e97cff5 100644 --- a/indexer/beacon/epochstats_ssz.go +++ b/indexer/beacon/epochstats_ssz.go @@ -6,7 +6,7 @@ package beacon import ( "encoding/binary" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" "github.com/pk910/dynamic-ssz/hasher" "github.com/pk910/dynamic-ssz/sszutils" diff --git a/indexer/beacon/epochstats_test.go b/indexer/beacon/epochstats_test.go index d5a38781b..53839c957 100644 --- a/indexer/beacon/epochstats_test.go +++ b/indexer/beacon/epochstats_test.go @@ -3,7 +3,7 @@ package beacon import ( "testing" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" ) diff --git a/indexer/beacon/epochvotes.go b/indexer/beacon/epochvotes.go index 4f4fe599c..73964f56a 100644 --- a/indexer/beacon/epochvotes.go +++ b/indexer/beacon/epochvotes.go @@ -5,9 +5,9 @@ import ( "encoding/binary" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/go-bitfield" ) diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 4cb65ddfb..734e279ff 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -7,13 +7,13 @@ import ( "sync" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/mashingan/smapping" ) diff --git a/indexer/beacon/fork.go b/indexer/beacon/fork.go index 5c3651f49..5f2a80b95 100644 --- a/indexer/beacon/fork.go +++ b/indexer/beacon/fork.go @@ -1,8 +1,8 @@ package beacon import ( - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // ForkKey represents a key used for indexing forks. diff --git a/indexer/beacon/forkcache.go b/indexer/beacon/forkcache.go index 93cebee11..70f551506 100644 --- a/indexer/beacon/forkcache.go +++ b/indexer/beacon/forkcache.go @@ -7,10 +7,10 @@ import ( "sort" "sync" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" ) diff --git a/indexer/beacon/forkdetection.go b/indexer/beacon/forkdetection.go index 49c98eb24..1001d25cc 100644 --- a/indexer/beacon/forkdetection.go +++ b/indexer/beacon/forkdetection.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/ethpandaops/dora/db" diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 085008367..3706e3619 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -7,8 +7,8 @@ import ( "sync" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" dynssz "github.com/pk910/dynamic-ssz" "github.com/sirupsen/logrus" diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 7ec8491f3..a8200d73a 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -7,13 +7,13 @@ import ( "slices" "sort" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/indexer/beacon/precalc.go b/indexer/beacon/precalc.go index ada710034..209b04409 100644 --- a/indexer/beacon/precalc.go +++ b/indexer/beacon/precalc.go @@ -3,7 +3,7 @@ package beacon import ( "fmt" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) func (indexer *Indexer) precalcNextEpochStats(epoch phase0.Epoch) error { diff --git a/indexer/beacon/pruning.go b/indexer/beacon/pruning.go index 92a02a010..d94fb9813 100644 --- a/indexer/beacon/pruning.go +++ b/indexer/beacon/pruning.go @@ -7,9 +7,9 @@ import ( "sort" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/mashingan/smapping" ) diff --git a/indexer/beacon/pubkeycache.go b/indexer/beacon/pubkeycache.go index 67cfc7ec0..573bed4af 100644 --- a/indexer/beacon/pubkeycache.go +++ b/indexer/beacon/pubkeycache.go @@ -4,7 +4,7 @@ import ( "strconv" "sync" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index cdbfe587d..97251a7be 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -5,9 +5,9 @@ import ( "fmt" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // BeaconHeaderRequestTimeout is the timeout duration for beacon header requests. diff --git a/indexer/beacon/state_sim.go b/indexer/beacon/state_sim.go index 3756e2fa1..8293f5c02 100644 --- a/indexer/beacon/state_sim.go +++ b/indexer/beacon/state_sim.go @@ -5,12 +5,12 @@ import ( "math" "slices" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) type stateSimulator struct { diff --git a/indexer/beacon/statetransition/balance.go b/indexer/beacon/statetransition/balance.go index 31cd98d74..dbb746f97 100644 --- a/indexer/beacon/statetransition/balance.go +++ b/indexer/beacon/statetransition/balance.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processEffectiveBalanceUpdates implements the Electra+ version of diff --git a/indexer/beacon/statetransition/block.go b/indexer/beacon/statetransition/block.go index 5f2787f8e..c7250640d 100644 --- a/indexer/beacon/statetransition/block.go +++ b/indexer/beacon/statetransition/block.go @@ -6,11 +6,11 @@ import ( "fmt" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/bellatrix" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/deneb" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/pk910/dynamic-ssz/sszutils" ) diff --git a/indexer/beacon/statetransition/builder.go b/indexer/beacon/statetransition/builder.go index 43e57f44f..6905504db 100644 --- a/indexer/beacon/statetransition/builder.go +++ b/indexer/beacon/statetransition/builder.go @@ -1,9 +1,9 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // Gloas-specific spec constants for builder payment quorum. diff --git a/indexer/beacon/statetransition/committees.go b/indexer/beacon/statetransition/committees.go index 4d46507a9..bb9ce6cdc 100644 --- a/indexer/beacon/statetransition/committees.go +++ b/indexer/beacon/statetransition/committees.go @@ -4,9 +4,9 @@ import ( "encoding/binary" "fmt" - "github.com/attestantio/go-eth2-client/spec/altair" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec/altair" + "github.com/ethpandaops/go-eth2-client/spec/phase0" blsu "github.com/protolambda/bls12-381-util" ) diff --git a/indexer/beacon/statetransition/helpers.go b/indexer/beacon/statetransition/helpers.go index 57275acd0..e3b4b64d0 100644 --- a/indexer/beacon/statetransition/helpers.go +++ b/indexer/beacon/statetransition/helpers.go @@ -4,8 +4,8 @@ import ( "crypto/sha256" "encoding/binary" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // hash256 returns the SHA-256 hash of data. diff --git a/indexer/beacon/statetransition/justification.go b/indexer/beacon/statetransition/justification.go index 350cca1ef..ff7fe0535 100644 --- a/indexer/beacon/statetransition/justification.go +++ b/indexer/beacon/statetransition/justification.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processJustificationAndFinalization implements the Altair+ version of diff --git a/indexer/beacon/statetransition/lookahead.go b/indexer/beacon/statetransition/lookahead.go index 778858c89..0e3217764 100644 --- a/indexer/beacon/statetransition/lookahead.go +++ b/indexer/beacon/statetransition/lookahead.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processProposerLookahead implements process_proposer_lookahead (Fulu+/EIP-7917). diff --git a/indexer/beacon/statetransition/operations.go b/indexer/beacon/statetransition/operations.go index 38372d40c..4760a5947 100644 --- a/indexer/beacon/statetransition/operations.go +++ b/indexer/beacon/statetransition/operations.go @@ -4,12 +4,12 @@ import ( "bytes" "slices" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/altair" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/altair" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processOperations implements process_operations. diff --git a/indexer/beacon/statetransition/pending.go b/indexer/beacon/statetransition/pending.go index bdc901756..e3f3a37a2 100644 --- a/indexer/beacon/statetransition/pending.go +++ b/indexer/beacon/statetransition/pending.go @@ -1,8 +1,8 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processPendingDeposits implements the Electra+ version of process_pending_deposits. diff --git a/indexer/beacon/statetransition/registry.go b/indexer/beacon/statetransition/registry.go index 5f5e618e7..28fe06f67 100644 --- a/indexer/beacon/statetransition/registry.go +++ b/indexer/beacon/statetransition/registry.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processRegistryUpdates implements the Electra+ version of process_registry_updates. diff --git a/indexer/beacon/statetransition/resets.go b/indexer/beacon/statetransition/resets.go index 132d13507..e50bd024b 100644 --- a/indexer/beacon/statetransition/resets.go +++ b/indexer/beacon/statetransition/resets.go @@ -1,8 +1,8 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/altair" - "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/altair" + "github.com/ethpandaops/go-eth2-client/spec/capella" ) // processEth1DataReset resets the ETH1 data votes at the start of a new voting period. diff --git a/indexer/beacon/statetransition/rewards.go b/indexer/beacon/statetransition/rewards.go index 06e3d7c31..e83261c4e 100644 --- a/indexer/beacon/statetransition/rewards.go +++ b/indexer/beacon/statetransition/rewards.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processInactivityUpdates implements process_inactivity_updates (Altair+). diff --git a/indexer/beacon/statetransition/slashings.go b/indexer/beacon/statetransition/slashings.go index 2c2f5f9d1..9e4bcd603 100644 --- a/indexer/beacon/statetransition/slashings.go +++ b/indexer/beacon/statetransition/slashings.go @@ -1,7 +1,7 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // processSlashings implements the Electra+ version of process_slashings. diff --git a/indexer/beacon/statetransition/state.go b/indexer/beacon/statetransition/state.go index 9d5ebf219..63900cd4d 100644 --- a/indexer/beacon/statetransition/state.go +++ b/indexer/beacon/statetransition/state.go @@ -4,14 +4,14 @@ import ( "fmt" "math" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/altair" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/altair" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/deneb" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/indexer/beacon/statetransition/statetransition.go b/indexer/beacon/statetransition/statetransition.go index 3c914fac6..07b526480 100644 --- a/indexer/beacon/statetransition/statetransition.go +++ b/indexer/beacon/statetransition/statetransition.go @@ -16,11 +16,11 @@ import ( "fmt" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/indexer/beacon/statetransition/withdrawals.go b/indexer/beacon/statetransition/withdrawals.go index 3c5f42022..b3b440a56 100644 --- a/indexer/beacon/statetransition/withdrawals.go +++ b/indexer/beacon/statetransition/withdrawals.go @@ -1,10 +1,10 @@ package statetransition import ( - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // expectedWithdrawals holds the result of get_expected_withdrawals. diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 68a12a434..9d49519a8 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -9,14 +9,14 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" ) diff --git a/indexer/beacon/validatoractivity.go b/indexer/beacon/validatoractivity.go index df56e13ff..c352fc517 100644 --- a/indexer/beacon/validatoractivity.go +++ b/indexer/beacon/validatoractivity.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) // validatorActivityCache is the cache for the validator activity. diff --git a/indexer/beacon/validatorcache.go b/indexer/beacon/validatorcache.go index 43f593136..d2df4908a 100644 --- a/indexer/beacon/validatorcache.go +++ b/indexer/beacon/validatorcache.go @@ -10,10 +10,10 @@ import ( "sync" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" ) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 91fcc3cff..1cc8fafda 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -4,15 +4,15 @@ import ( "fmt" "math" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/capella" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/bellatrix" + "github.com/ethpandaops/go-eth2-client/spec/capella" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" ) diff --git a/indexer/execution/indexerctx.go b/indexer/execution/indexerctx.go index 7d7195e1b..b56fe0e3d 100644 --- a/indexer/execution/indexerctx.go +++ b/indexer/execution/indexerctx.go @@ -6,11 +6,11 @@ import ( "slices" "sort" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/clients/execution" "github.com/ethpandaops/dora/indexer/beacon" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/indexer/execution/system_contracts/consolidation_indexer.go b/indexer/execution/system_contracts/consolidation_indexer.go index e25229b67..f442e2bea 100644 --- a/indexer/execution/system_contracts/consolidation_indexer.go +++ b/indexer/execution/system_contracts/consolidation_indexer.go @@ -4,9 +4,9 @@ import ( "fmt" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" diff --git a/indexer/execution/system_contracts/deposit_indexer.go b/indexer/execution/system_contracts/deposit_indexer.go index d16d94948..369bb8671 100644 --- a/indexer/execution/system_contracts/deposit_indexer.go +++ b/indexer/execution/system_contracts/deposit_indexer.go @@ -8,11 +8,11 @@ import ( "strings" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" blsu "github.com/protolambda/bls12-381-util" zrnt_common "github.com/protolambda/zrnt/eth2/beacon/common" diff --git a/indexer/execution/system_contracts/withdrawal_indexer.go b/indexer/execution/system_contracts/withdrawal_indexer.go index e2f22a351..48898e9b2 100644 --- a/indexer/execution/system_contracts/withdrawal_indexer.go +++ b/indexer/execution/system_contracts/withdrawal_indexer.go @@ -5,9 +5,9 @@ import ( "math/big" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" diff --git a/indexer/execution/txindexer/txindexer.go b/indexer/execution/txindexer/txindexer.go index e1b0a2952..b45afdecb 100644 --- a/indexer/execution/txindexer/txindexer.go +++ b/indexer/execution/txindexer/txindexer.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" diff --git a/indexer/mevrelay/mevindexer.go b/indexer/mevrelay/mevindexer.go index 059d0e180..a16de329d 100644 --- a/indexer/mevrelay/mevindexer.go +++ b/indexer/mevrelay/mevindexer.go @@ -13,8 +13,8 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" diff --git a/indexer/snooper/snooper_manager.go b/indexer/snooper/snooper_manager.go index 3004cf89d..f4aa2f638 100644 --- a/indexer/snooper/snooper_manager.go +++ b/indexer/snooper/snooper_manager.go @@ -7,12 +7,12 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/execution" "github.com/ethpandaops/dora/clients/execution/snooper" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" ) diff --git a/services/chainservice.go b/services/chainservice.go index 7f666078f..37aca1a08 100644 --- a/services/chainservice.go +++ b/services/chainservice.go @@ -9,9 +9,9 @@ import ( "strings" "time" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/ethpandaops/dora/blockdb" diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 8364c23c2..b006f7575 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -7,10 +7,10 @@ import ( "math" "strings" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/deneb" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" btypes "github.com/ethpandaops/dora/blockdb/types" diff --git a/services/chainservice_builder.go b/services/chainservice_builder.go index dcdfd6f34..d7c93f6af 100644 --- a/services/chainservice_builder.go +++ b/services/chainservice_builder.go @@ -6,8 +6,8 @@ import ( "slices" "sort" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" diff --git a/services/chainservice_consolidations.go b/services/chainservice_consolidations.go index f4114d27a..7f06f80be 100644 --- a/services/chainservice_consolidations.go +++ b/services/chainservice_consolidations.go @@ -5,12 +5,12 @@ import ( "context" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/prysm/v5/container/slice" "github.com/sirupsen/logrus" "golang.org/x/exp/maps" diff --git a/services/chainservice_deposits.go b/services/chainservice_deposits.go index 073039dc7..e951189a6 100644 --- a/services/chainservice_deposits.go +++ b/services/chainservice_deposits.go @@ -8,11 +8,11 @@ import ( "strings" "sync" - "github.com/attestantio/go-eth2-client/spec/electra" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" + "github.com/ethpandaops/go-eth2-client/spec/electra" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/prysm/v5/container/slice" "github.com/sirupsen/logrus" ) diff --git a/services/chainservice_epochs.go b/services/chainservice_epochs.go index 1d99b3c60..be21c224a 100644 --- a/services/chainservice_epochs.go +++ b/services/chainservice_epochs.go @@ -3,9 +3,9 @@ package services import ( "context" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) func (bs *ChainService) GetDbEpochs(ctx context.Context, firstEpoch uint64, limit uint32) []*dbtypes.Epoch { diff --git a/services/chainservice_objects.go b/services/chainservice_objects.go index e29e7042e..c32724328 100644 --- a/services/chainservice_objects.go +++ b/services/chainservice_objects.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/db" diff --git a/services/chainservice_validators.go b/services/chainservice_validators.go index b88338025..012cad3f9 100644 --- a/services/chainservice_validators.go +++ b/services/chainservice_validators.go @@ -7,11 +7,11 @@ import ( "sort" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) func (bs *ChainService) GetValidatorByIndex(index phase0.ValidatorIndex, withBalance bool) *v1.Validator { diff --git a/services/chainservice_withdrawals.go b/services/chainservice_withdrawals.go index ff516cd78..15a7645da 100644 --- a/services/chainservice_withdrawals.go +++ b/services/chainservice_withdrawals.go @@ -6,8 +6,8 @@ import ( "slices" "strings" - v1 "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" "github.com/ethpandaops/dora/db" diff --git a/services/dasguardian.go b/services/dasguardian.go index 40feb624b..e162cb6e1 100644 --- a/services/dasguardian.go +++ b/services/dasguardian.go @@ -7,9 +7,9 @@ import ( "reflect" "time" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dasguardian "github.com/probe-lab/eth-das-guardian" "github.com/probe-lab/eth-das-guardian/api" "github.com/sirupsen/logrus" diff --git a/services/validatornames.go b/services/validatornames.go index c32403050..53e4fab29 100644 --- a/services/validatornames.go +++ b/services/validatornames.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/config" @@ -21,6 +20,7 @@ import ( "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/indexer/beacon" "github.com/ethpandaops/dora/utils" + "github.com/ethpandaops/go-eth2-client/spec/phase0" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" "golang.org/x/exp/maps" diff --git a/statecache/statecache.go b/statecache/statecache.go index 23a8e0fd1..b553f205f 100644 --- a/statecache/statecache.go +++ b/statecache/statecache.go @@ -15,11 +15,11 @@ import ( "strings" "sync" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/fulu" - "github.com/attestantio/go-eth2-client/spec/gloas" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/types" + "github.com/ethpandaops/go-eth2-client/spec" + "github.com/ethpandaops/go-eth2-client/spec/fulu" + "github.com/ethpandaops/go-eth2-client/spec/gloas" + "github.com/ethpandaops/go-eth2-client/spec/phase0" dynssz "github.com/pk910/dynamic-ssz" ) diff --git a/types/models/search.go b/types/models/search.go index 468214870..f93fa0529 100644 --- a/types/models/search.go +++ b/types/models/search.go @@ -1,6 +1,6 @@ package models -import "github.com/attestantio/go-eth2-client/spec/phase0" +import "github.com/ethpandaops/go-eth2-client/spec/phase0" // SearchBlockResult is a struct to hold the search block result with a given graffiti type SearchBlockResult struct { diff --git a/utils/beaconspec.go b/utils/beaconspec.go index cb5ebe611..bbcdbc385 100644 --- a/utils/beaconspec.go +++ b/utils/beaconspec.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethpandaops/go-eth2-client/spec/phase0" ) func ParseSpecMap(data map[string]any) map[string]any { diff --git a/utils/blobs.go b/utils/blobs.go index c7bad1b50..6665b64e0 100644 --- a/utils/blobs.go +++ b/utils/blobs.go @@ -3,8 +3,8 @@ package utils import ( "crypto/sha256" - "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/ethereum/go-ethereum/common" + "github.com/ethpandaops/go-eth2-client/spec/deneb" ) // MatchBlobCommitments finds the KZG commitments from the block that match From 965c0646c5d007ef4bd5b075f5e17264c01ca531 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 14 Apr 2026 09:45:06 +0200 Subject: [PATCH 98/99] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8cf7ca837..a51931741 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/ethereum/go-ethereum v1.17.2 github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c github.com/ethpandaops/ethwallclock v0.4.0 - github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252 + github.com/ethpandaops/go-eth2-client v0.0.1 github.com/glebarez/go-sqlite v1.22.0 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v5 v5.3.1 diff --git a/go.sum b/go.sum index 0447f76c9..8c7bb6aef 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c h1:uBRIitwcuCJ github.com/ethpandaops/ethcore v0.0.0-20260320045412-9cdd5d70a29c/go.mod h1:QsmYTdesob+vQ6pW4KtRVvxLZUNop3cdtd/DgD30hJU= github.com/ethpandaops/ethwallclock v0.4.0 h1:+sgnhf4pk6hLPukP076VxkiLloE4L0Yk1yat+ZyHh1g= github.com/ethpandaops/ethwallclock v0.4.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24= -github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252 h1:cS54Q13E/dA6uFCgcuIufiYeU6ifZdtQPHMbqS3KuOU= -github.com/ethpandaops/go-eth2-client v0.0.0-20260414071626-8c1141f51252/go.mod h1:9BBd/XIw1egZTkxtFGMvgXnsxX6ypKHKNKD7itqjmNQ= +github.com/ethpandaops/go-eth2-client v0.0.1 h1:Xifvb7RF24tguA6HxEaE2vIN1BsY44SOSH/B+CBSFPk= +github.com/ethpandaops/go-eth2-client v0.0.1/go.mod h1:9BBd/XIw1egZTkxtFGMvgXnsxX6ypKHKNKD7itqjmNQ= github.com/ferranbt/fastssz v1.0.0 h1:9EXXYsracSqQRBQiHeaVsG/KQeYblPf40hsQPb9Dzk8= github.com/ferranbt/fastssz v1.0.0/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= From 3d8bbf9700d74c0fdb68c65837d220a888b6c0e1 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 14 Apr 2026 10:00:15 +0200 Subject: [PATCH 99/99] merge conflict & go-eth2-client bump fixes --- handlers/api/api_das_guardian_mass.go | 2 +- handlers/slot.go | 34 +++++++++++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/handlers/api/api_das_guardian_mass.go b/handlers/api/api_das_guardian_mass.go index 1c9e83b53..ec84bab86 100644 --- a/handlers/api/api_das_guardian_mass.go +++ b/handlers/api/api_das_guardian_mass.go @@ -12,8 +12,8 @@ import ( "github.com/ethpandaops/dora/dbtypes" "github.com/ethpandaops/dora/services" "github.com/ethpandaops/dora/utils" + dasguardian "github.com/ethpandaops/eth-das-guardian" "github.com/ethpandaops/go-eth2-client/spec/phase0" - dasguardian "github.com/probe-lab/eth-das-guardian" "github.com/sirupsen/logrus" ) diff --git a/handlers/slot.go b/handlers/slot.go index 6fa6a38a4..b12938eea 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -749,23 +749,33 @@ func getSlotPageBlockData(ctx context.Context, blockData *services.CombinedBlock } if payloadBid, err := blockData.Block.SignedExecutionPayloadBid(); err == nil { - commitments := make([][]byte, len(payloadBid.Message.BlobKZGCommitments)) - for i := range payloadBid.Message.BlobKZGCommitments { - commitments[i] = payloadBid.Message.BlobKZGCommitments[i][:] + blobKzgCommitments, _ := payloadBid.BlobKZGCommitments() + parentBlockHash, _ := payloadBid.ParentBlockHash() + parentBlockRoot, _ := payloadBid.ParentBlockRoot() + blockHash, _ := payloadBid.BlockHash() + gasLimit, _ := payloadBid.GasLimit() + builderIndex, _ := payloadBid.BuilderIndex() + slot, _ := payloadBid.Slot() + value, _ := payloadBid.Value() + signature, _ := payloadBid.Signature() + + commitments := make([][]byte, len(blobKzgCommitments)) + for i := range blobKzgCommitments { + commitments[i] = blobKzgCommitments[i][:] } pageData.PayloadHeader = &models.SlotPagePayloadHeader{ PayloadStatus: uint16(0), - ParentBlockHash: payloadBid.Message.ParentBlockHash[:], - ParentBlockRoot: payloadBid.Message.ParentBlockRoot[:], - BlockHash: payloadBid.Message.BlockHash[:], - GasLimit: uint64(payloadBid.Message.GasLimit), - BuilderIndex: uint64(payloadBid.Message.BuilderIndex), - BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadBid.Message.BuilderIndex) | services.BuilderIndexFlag), - Slot: uint64(payloadBid.Message.Slot), - Value: uint64(payloadBid.Message.Value), + ParentBlockHash: parentBlockHash[:], + ParentBlockRoot: parentBlockRoot[:], + BlockHash: blockHash[:], + GasLimit: uint64(gasLimit), + BuilderIndex: uint64(builderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(builderIndex) | services.BuilderIndexFlag), + Slot: uint64(slot), + Value: uint64(value), BlobKZGCommitments: commitments, - Signature: payloadBid.Signature[:], + Signature: signature[:], } }