diff --git a/common/constants.go b/common/constants.go index 37a94d27d3..aab3996520 100644 --- a/common/constants.go +++ b/common/constants.go @@ -259,6 +259,10 @@ const MetricNumShardHeadersProcessed = "erd_num_shard_headers_processed" // MetricNumTimesInForkChoice is the metric that counts how many times a node was in fork choice const MetricNumTimesInForkChoice = "erd_fork_choice_count" +// MetricNumEquivocationProofs is the metric that counts the equivocation events observed by the +// proofs pool (different-hash proofs received for the same header nonce) +const MetricNumEquivocationProofs = "erd_num_equivocation_proofs" + // MetricHighestFinalBlock is the metric for the nonce of the highest final block const MetricHighestFinalBlock = "erd_highest_final_nonce" diff --git a/dataRetriever/dataPool/proofsCache/export_test.go b/dataRetriever/dataPool/proofsCache/export_test.go index 6ca7c5344e..b73ab2f89a 100644 --- a/dataRetriever/dataPool/proofsCache/export_test.go +++ b/dataRetriever/dataPool/proofsCache/export_test.go @@ -2,6 +2,9 @@ package proofscache import "github.com/multiversx/mx-chain-core-go/data" +// MaxProofsPerNonce - +const MaxProofsPerNonce = maxProofsPerNonce + // NewProofsCache - func NewProofsCache(bucketSize int) *proofsCache { return newProofsCache(bucketSize) diff --git a/dataRetriever/dataPool/proofsCache/proofsBucket.go b/dataRetriever/dataPool/proofsCache/proofsBucket.go index f6321e1729..232650c47d 100644 --- a/dataRetriever/dataPool/proofsCache/proofsBucket.go +++ b/dataRetriever/dataPool/proofsCache/proofsBucket.go @@ -1,35 +1,50 @@ package proofscache -import "github.com/multiversx/mx-chain-core-go/data" - type proofNonceBucket struct { maxNonce uint64 - proofsByNonce map[uint64]string + proofsByNonce map[uint64][]string } func newProofBucket() *proofNonceBucket { return &proofNonceBucket{ - proofsByNonce: make(map[uint64]string), + proofsByNonce: make(map[uint64][]string), } } func (p *proofNonceBucket) size() int { - return len(p.proofsByNonce) + size := 0 + for _, hashes := range p.proofsByNonce { + size += len(hashes) + } + + return size } -func (p *proofNonceBucket) insert(proof data.HeaderProofHandler) string { - nonce := proof.GetHeaderNonce() - newHash := string(proof.GetHeaderHash()) +func (p *proofNonceBucket) hashesAt(nonce uint64) []string { + return p.proofsByNonce[nonce] +} - oldHash, existed := p.proofsByNonce[nonce] - p.proofsByNonce[nonce] = newHash +// insert adds the hash at the given nonce, keeping any different hashes already stored there +func (p *proofNonceBucket) insert(nonce uint64, hash string) { + for _, existingHash := range p.proofsByNonce[nonce] { + if existingHash == hash { + return + } + } + + p.proofsByNonce[nonce] = append(p.proofsByNonce[nonce], hash) if nonce > p.maxNonce { p.maxNonce = nonce } +} - if existed { - return oldHash +func (p *proofNonceBucket) remove(nonce uint64, hash string) { + hashes := p.proofsByNonce[nonce] + for i, existingHash := range hashes { + if existingHash == hash { + p.proofsByNonce[nonce] = append(hashes[:i], hashes[i+1:]...) + return + } } - return "" } diff --git a/dataRetriever/dataPool/proofsCache/proofsCache.go b/dataRetriever/dataPool/proofsCache/proofsCache.go index 1de9fcaa50..43d2ee7440 100644 --- a/dataRetriever/dataPool/proofsCache/proofsCache.go +++ b/dataRetriever/dataPool/proofsCache/proofsCache.go @@ -1,12 +1,18 @@ package proofscache import ( + "bytes" + "sort" "sync" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" ) +// maxProofsPerNonce bounds how many different-hash proofs are kept at one nonce (more than one is +// equivocation evidence); on overflow the highest (round, hash) proof is evicted +const maxProofsPerNonce = 4 + type proofsCache struct { mutProofsCache sync.RWMutex proofsByNonceBuckets map[uint64]*proofNonceBucket @@ -34,54 +40,125 @@ func (pc *proofsCache) getProofByHash(headerHash []byte) (data.HeaderProofHandle return proof, nil } +// getProofByNonce returns the canonical proof at the given nonce: the lowest round, with the +// lowest hash as tie-break func (pc *proofsCache) getProofByNonce(headerNonce uint64) (data.HeaderProofHandler, error) { pc.mutProofsCache.RLock() defer pc.mutProofsCache.RUnlock() - bucketKey := pc.getBucketKey(headerNonce) - bucket, ok := pc.proofsByNonceBuckets[bucketKey] - if !ok { + proof := pc.lowestProofAtNonce(headerNonce) + if proof == nil { return nil, ErrMissingProof } - proofHash, ok := bucket.proofsByNonce[headerNonce] + return proof, nil +} + +// lowestProofAtNonce must be called under mutex protection; allocation-free on purpose, it sits +// on the hot read path +func (pc *proofsCache) lowestProofAtNonce(headerNonce uint64) data.HeaderProofHandler { + bucket, ok := pc.proofsByNonceBuckets[pc.getBucketKey(headerNonce)] if !ok { - return nil, ErrMissingProof + return nil + } + + var lowest data.HeaderProofHandler + for _, headerHash := range bucket.hashesAt(headerNonce) { + proof, hasProof := pc.proofsByHash[headerHash] + if !hasProof { + continue + } + if lowest == nil || lessProof(proof, lowest) { + lowest = proof + } } - proof, ok := pc.proofsByHash[proofHash] + return lowest +} + +// lessProof reports whether a orders before b by (round, hash) ascending +func lessProof(a, b data.HeaderProofHandler) bool { + if a.GetHeaderRound() != b.GetHeaderRound() { + return a.GetHeaderRound() < b.GetHeaderRound() + } + return bytes.Compare(a.GetHeaderHash(), b.GetHeaderHash()) < 0 +} + +// getProofsByNonce returns all proofs at the given nonce, ordered by (round, hash) ascending +func (pc *proofsCache) getProofsByNonce(headerNonce uint64) []data.HeaderProofHandler { + pc.mutProofsCache.RLock() + defer pc.mutProofsCache.RUnlock() + + return pc.sortedProofsAtNonce(headerNonce) +} + +// sortedProofsAtNonce must be called under mutex protection +func (pc *proofsCache) sortedProofsAtNonce(headerNonce uint64) []data.HeaderProofHandler { + bucket, ok := pc.proofsByNonceBuckets[pc.getBucketKey(headerNonce)] if !ok { - return nil, ErrMissingProof + return nil } - return proof, nil + hashes := bucket.hashesAt(headerNonce) + proofs := make([]data.HeaderProofHandler, 0, len(hashes)) + for _, headerHash := range hashes { + proof, hasProof := pc.proofsByHash[headerHash] + if hasProof { + proofs = append(proofs, proof) + } + } + + if len(proofs) > 1 { + sort.Slice(proofs, func(i, j int) bool { + return lessProof(proofs[i], proofs[j]) + }) + } + + return proofs } -func (pc *proofsCache) addProof(proof data.HeaderProofHandler) { +// addProof stores the proof, keeping different-hash proofs at the same nonce as equivocation +// evidence; returns the competing proofs only on a newly stored hash, so equivocation reports once +func (pc *proofsCache) addProof(proof data.HeaderProofHandler) []data.HeaderProofHandler { if check.IfNil(proof) { - return + return nil } pc.mutProofsCache.Lock() defer pc.mutProofsCache.Unlock() - oldHash := pc.insertProofByNonce(proof) + nonce := proof.GetHeaderNonce() newHash := string(proof.GetHeaderHash()) + bucket := pc.getOrCreateBucket(nonce) + + alreadyStored := false + var competingProofs []data.HeaderProofHandler + for _, existingHash := range bucket.hashesAt(nonce) { + if existingHash == newHash { + alreadyStored = true + continue + } - // Delete the old hash from proofsByHash if it's different from the new hash - if len(oldHash) != 0 && oldHash != newHash { - log.Warn("proofsCache.addProof: overwrite proof by hash", - "oldHash", oldHash, - "newHash", newHash, - ) - delete(pc.proofsByHash, oldHash) + existingProof, hasProof := pc.proofsByHash[existingHash] + if hasProof { + competingProofs = append(competingProofs, existingProof) + } } + bucket.insert(nonce, newHash) pc.proofsByHash[newHash] = proof + + pc.evictExcessProofsAtNonce(bucket, nonce) + + if alreadyStored { + return nil + } + + return competingProofs } // addProofIfNoneAtNonce adds the proof only if its nonce slot is free; an occupied slot (same or -// different hash) rejects the add and returns the pre-existing proof, never overwriting it +// different hash) rejects the add and returns the pre-existing canonical proof, never overwriting it func (pc *proofsCache) addProofIfNoneAtNonce(proof data.HeaderProofHandler) (bool, data.HeaderProofHandler) { if check.IfNil(proof) { return false, nil @@ -90,31 +167,49 @@ func (pc *proofsCache) addProofIfNoneAtNonce(proof data.HeaderProofHandler) (boo pc.mutProofsCache.Lock() defer pc.mutProofsCache.Unlock() - bucketKey := pc.getBucketKey(proof.GetHeaderNonce()) - bucket, ok := pc.proofsByNonceBuckets[bucketKey] - if ok { - existingHash, hasNonce := bucket.proofsByNonce[proof.GetHeaderNonce()] - if hasNonce { - existingProof, hasProof := pc.proofsByHash[existingHash] - if hasProof { - return false, existingProof - } - } + nonce := proof.GetHeaderNonce() + existingProof := pc.lowestProofAtNonce(nonce) + if existingProof != nil { + return false, existingProof } - _ = pc.insertProofByNonce(proof) + bucket := pc.getOrCreateBucket(nonce) + bucket.insert(nonce, string(proof.GetHeaderHash())) pc.proofsByHash[string(proof.GetHeaderHash())] = proof return true, nil } +// evictExcessProofsAtNonce must be called under mutex protection +func (pc *proofsCache) evictExcessProofsAtNonce(bucket *proofNonceBucket, nonce uint64) { + if len(bucket.hashesAt(nonce)) <= maxProofsPerNonce { + return + } + + proofs := pc.sortedProofsAtNonce(nonce) + for len(proofs) > maxProofsPerNonce { + evictedProof := proofs[len(proofs)-1] + proofs = proofs[:len(proofs)-1] + evictedHash := string(evictedProof.GetHeaderHash()) + bucket.remove(nonce, evictedHash) + delete(pc.proofsByHash, evictedHash) + + log.Warn("proofsCache: too many proofs at the same nonce, evicted the highest round one", + "nonce", nonce, + "evicted hash", evictedProof.GetHeaderHash(), + "evicted round", evictedProof.GetHeaderRound(), + ) + } +} + // getBucketKey will return bucket key as lower bound window value func (pc *proofsCache) getBucketKey(index uint64) uint64 { return (index / pc.bucketSize) * pc.bucketSize } -func (pc *proofsCache) insertProofByNonce(proof data.HeaderProofHandler) string { - bucketKey := pc.getBucketKey(proof.GetHeaderNonce()) +// getOrCreateBucket must be called under mutex protection +func (pc *proofsCache) getOrCreateBucket(nonce uint64) *proofNonceBucket { + bucketKey := pc.getBucketKey(nonce) bucket, ok := pc.proofsByNonceBuckets[bucketKey] if !ok { @@ -122,7 +217,7 @@ func (pc *proofsCache) insertProofByNonce(proof data.HeaderProofHandler) string pc.proofsByNonceBuckets[bucketKey] = bucket } - return bucket.insert(proof) + return bucket } func (pc *proofsCache) cleanupProofsBehindNonce(nonce uint64) { @@ -142,7 +237,9 @@ func (pc *proofsCache) cleanupProofsBehindNonce(nonce uint64) { } func (pc *proofsCache) cleanupProofsInBucket(bucket *proofNonceBucket) { - for _, headerHash := range bucket.proofsByNonce { - delete(pc.proofsByHash, headerHash) + for _, headerHashes := range bucket.proofsByNonce { + for _, headerHash := range headerHashes { + delete(pc.proofsByHash, headerHash) + } } } diff --git a/dataRetriever/dataPool/proofsCache/proofsPool.go b/dataRetriever/dataPool/proofsCache/proofsPool.go index a963c71e82..58a32d4432 100644 --- a/dataRetriever/dataPool/proofsCache/proofsPool.go +++ b/dataRetriever/dataPool/proofsCache/proofsPool.go @@ -21,8 +21,12 @@ type proofsPool struct { mutAddedProofSubscribers sync.RWMutex addedProofSubscribers []func(headerProof data.HeaderProofHandler) - cleanupNonceDelta uint64 - bucketSize int + + mutEquivocationSubscribers sync.RWMutex + equivocationSubscribers []func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler) + + cleanupNonceDelta uint64 + bucketSize int } // NewProofsPool creates a new proofs pool component @@ -37,15 +41,16 @@ func NewProofsPool(cleanupNonceDelta uint64, bucketSize int) *proofsPool { } return &proofsPool{ - cache: make(map[uint32]*proofsCache), - addedProofSubscribers: make([]func(headerProof data.HeaderProofHandler), 0), - cleanupNonceDelta: cleanupNonceDelta, - bucketSize: bucketSize, + cache: make(map[uint32]*proofsCache), + addedProofSubscribers: make([]func(headerProof data.HeaderProofHandler), 0), + equivocationSubscribers: make([]func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler), 0), + cleanupNonceDelta: cleanupNonceDelta, + bucketSize: bucketSize, } } -// UpsertProof will add the provided proof to the pool. If there is already an existing proof, -// it will overwrite it. +// UpsertProof will add the provided proof to the pool. A proof with the same hash is overwritten; +// a different-hash proof at the same nonce is kept alongside the existing ones (see AddProof). func (pp *proofsPool) UpsertProof( headerProof data.HeaderProofHandler, ) bool { @@ -58,6 +63,7 @@ func (pp *proofsPool) UpsertProof( // AddProof will add the provided proof to the pool, if it's not already in the pool. // It will return true if the proof was added to the pool. +// A different-hash proof at the same nonce is kept alongside the existing ones and notifies the equivocation handlers. func (pp *proofsPool) AddProof( headerProof data.HeaderProofHandler, ) bool { @@ -129,7 +135,19 @@ func (pp *proofsPool) addProof( "isStartOfEpoch", headerProof.GetIsStartOfEpoch(), ) - proofsPerShard.addProof(headerProof) + competingProofs := proofsPerShard.addProof(headerProof) + if len(competingProofs) > 0 { + log.Error("proofsPool: equivocation - multiple proofs at the same nonce", + "shardID", headerProof.GetHeaderShardId(), + "nonce", headerProof.GetHeaderNonce(), + "new hash", headerProof.GetHeaderHash(), + "new round", headerProof.GetHeaderRound(), + "num competing proofs", len(competingProofs), + "first competing hash", competingProofs[0].GetHeaderHash(), + "first competing round", competingProofs[0].GetHeaderRound(), + ) + pp.callEquivocationSubscribers(headerProof, competingProofs) + } pp.callAddedProofSubscribers(headerProof) @@ -218,7 +236,8 @@ func (pp *proofsPool) GetProof( return proofsPerShard.getProofByHash(headerHash) } -// GetProofByNonce will get the proof from pool for the provided header nonce, searching through all shards +// GetProofByNonce will get the canonical proof from pool for the provided header nonce: the one +// with the lowest round, lowest hash as tie-break, among all proofs held at that nonce func (pp *proofsPool) GetProofByNonce(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { log.Trace("trying to get proof", "headerNonce", headerNonce, @@ -235,6 +254,24 @@ func (pp *proofsPool) GetProofByNonce(headerNonce uint64, shardID uint32) (data. return proofsPerShard.getProofByNonce(headerNonce) } +// GetProofsByNonce will get all the proofs held for the provided header nonce, ordered by +// (round, hash) ascending; more than one returned proof is evidence of equivocation +func (pp *proofsPool) GetProofsByNonce(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { + pp.mutCache.RLock() + proofsPerShard, ok := pp.cache[shardID] + pp.mutCache.RUnlock() + if !ok { + return nil, fmt.Errorf("%w: proofs cache per shard not found, shard ID: %d", ErrMissingProof, shardID) + } + + proofs := proofsPerShard.getProofsByNonce(headerNonce) + if len(proofs) == 0 { + return nil, ErrMissingProof + } + + return proofs, nil +} + // HasProof will check if there is a proof for the provided hash func (pp *proofsPool) HasProof( shardID uint32, @@ -256,6 +293,28 @@ func (pp *proofsPool) RegisterHandler(handler func(headerProof data.HeaderProofH pp.mutAddedProofSubscribers.Unlock() } +// RegisterEquivocationHandler registers a new handler to be called when a proof is added for a +// nonce that already holds one or more proofs with a different header hash +func (pp *proofsPool) RegisterEquivocationHandler(handler func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler)) { + if handler == nil { + log.Error("attempt to register a nil equivocation handler to proofs pool") + return + } + + pp.mutEquivocationSubscribers.Lock() + pp.equivocationSubscribers = append(pp.equivocationSubscribers, handler) + pp.mutEquivocationSubscribers.Unlock() +} + +func (pp *proofsPool) callEquivocationSubscribers(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler) { + pp.mutEquivocationSubscribers.RLock() + defer pp.mutEquivocationSubscribers.RUnlock() + + for _, handler := range pp.equivocationSubscribers { + go handler(headerProof, competingProofs) + } +} + // IsInterfaceNil returns true if there is no value under the interface func (pp *proofsPool) IsInterfaceNil() bool { return pp == nil diff --git a/dataRetriever/dataPool/proofsCache/proofsPool_test.go b/dataRetriever/dataPool/proofsCache/proofsPool_test.go index dd2871ea8b..b28cce842d 100644 --- a/dataRetriever/dataPool/proofsCache/proofsPool_test.go +++ b/dataRetriever/dataPool/proofsCache/proofsPool_test.go @@ -148,32 +148,37 @@ func TestProofsPool_UpsertMultipleHashes(t *testing.T) { pp := proofscache.NewProofsPool(3, 10) - // Upsert 10 different proofs for the same nonce - // Each upsert should clean up the previous hash + // Upsert 10 different-hash proofs for the same nonce, increasing rounds. + // They are kept as competing proofs, capped at the max per nonce (lowest rounds win). for i := 0; i < 10; i++ { proof := &block.HeaderProof{ HeaderHash: []byte{byte(i)}, // Different hash each time HeaderNonce: 5, // Same nonce + HeaderRound: uint64(10 + i), HeaderShardId: shardID, } ok := pp.UpsertProof(proof) require.True(t, ok, "upsert %d should succeed", i) } - // All previous hashes (hash[0] through hash[8]) should be cleaned up - for i := 0; i < 9; i++ { - hasOldHash := pp.HasProof(shardID, []byte{byte(i)}) - require.False(t, hasOldHash, "old hash[%d] should be cleaned up", i) + // The lowest-round proofs are retained up to the cap + for i := 0; i < proofscache.MaxProofsPerNonce; i++ { + require.True(t, pp.HasProof(shardID, []byte{byte(i)}), "low round hash[%d] should be retained", i) } - // Only the latest hash should be present - hasLatestHash := pp.HasProof(shardID, []byte{9}) - require.True(t, hasLatestHash, "latest hash[9] should be in pool") + // The higher-round proofs beyond the cap are evicted + for i := proofscache.MaxProofsPerNonce; i < 10; i++ { + require.False(t, pp.HasProof(shardID, []byte{byte(i)}), "high round hash[%d] should be evicted", i) + } - // Verify nonce 5 maps to the latest hash + // The canonical proof at nonce 5 is the lowest-round one proofByNonce, err := pp.GetProofByNonce(5, shardID) require.Nil(t, err) - require.Equal(t, []byte{9}, proofByNonce.GetHeaderHash(), "nonce 5 should map to latest hash") + require.Equal(t, []byte{0}, proofByNonce.GetHeaderHash(), "nonce 5 should map to the lowest round hash") + + proofs, err := pp.GetProofsByNonce(5, shardID) + require.Nil(t, err) + require.Equal(t, proofscache.MaxProofsPerNonce, len(proofs)) } func TestProofsPool_IsProofEqual(t *testing.T) { @@ -312,7 +317,7 @@ func TestProofsPool_Concurrency(t *testing.T) { for i := 0; i < numOperations; i++ { go func(idx int) { - switch idx % 7 { + switch idx % 9 { case 0, 1, 2: _ = pp.AddProof(generateProof()) case 3: @@ -328,6 +333,12 @@ func TestProofsPool_Concurrency(t *testing.T) { handler := func(proof data.HeaderProofHandler) { } pp.RegisterHandler(handler) + case 7: + _, _ = pp.GetProofsByNonce(generateRandomNonce(100), generateRandomShardID()) + case 8: + handler := func(proof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler) { + } + pp.RegisterEquivocationHandler(handler) default: assert.Fail(t, "should have not beed called") } @@ -371,6 +382,154 @@ func generateRandomInt(max int64) *big.Int { return rantInt } +func TestProofsPool_CompetingProofsAtSameNonce(t *testing.T) { + t.Parallel() + + proofRound6 := &block.HeaderProof{ + HeaderHash: []byte("hashA"), + HeaderNonce: 5, + HeaderRound: 6, + HeaderShardId: shardID, + } + proofRound7 := &block.HeaderProof{ + HeaderHash: []byte("hashB"), + HeaderNonce: 5, + HeaderRound: 7, + HeaderShardId: shardID, + } + + t.Run("both proofs retained, canonical is lowest round regardless of add order", func(t *testing.T) { + t.Parallel() + + for _, proofs := range [][]*block.HeaderProof{ + {proofRound6, proofRound7}, + {proofRound7, proofRound6}, + } { + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + require.True(t, pp.AddProof(proofs[0])) + require.True(t, pp.AddProof(proofs[1])) + + require.True(t, pp.HasProof(shardID, []byte("hashA"))) + require.True(t, pp.HasProof(shardID, []byte("hashB"))) + + proof, err := pp.GetProofByNonce(5, shardID) + require.Nil(t, err) + require.Equal(t, proofRound6, proof) + + allProofs, err := pp.GetProofsByNonce(5, shardID) + require.Nil(t, err) + require.Equal(t, 2, len(allProofs)) + require.Equal(t, proofRound6, allProofs[0]) + require.Equal(t, proofRound7, allProofs[1]) + } + }) + + t.Run("same round ties break on lowest hash", func(t *testing.T) { + t.Parallel() + + tieHigh := &block.HeaderProof{HeaderHash: []byte("hashZ"), HeaderNonce: 5, HeaderRound: 6, HeaderShardId: shardID} + + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + require.True(t, pp.AddProof(tieHigh)) + require.True(t, pp.AddProof(proofRound6)) + + proof, err := pp.GetProofByNonce(5, shardID) + require.Nil(t, err) + require.Equal(t, proofRound6, proof) + }) + + t.Run("equivocation handler notified with competing proofs", func(t *testing.T) { + t.Parallel() + + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + + type equivocationEvent struct { + newProof data.HeaderProofHandler + competing []data.HeaderProofHandler + } + eventChan := make(chan equivocationEvent, 2) + pp.RegisterEquivocationHandler(nil) + pp.RegisterEquivocationHandler(func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler) { + eventChan <- equivocationEvent{newProof: headerProof, competing: competingProofs} + }) + + require.True(t, pp.AddProof(proofRound6)) + select { + case <-eventChan: + require.Fail(t, "must not notify on the first proof at a nonce") + case <-time.After(50 * time.Millisecond): + } + + require.True(t, pp.AddProof(proofRound7)) + select { + case event := <-eventChan: + require.Equal(t, proofRound7, event.newProof) + require.Equal(t, []data.HeaderProofHandler{proofRound6}, event.competing) + case <-time.After(time.Second): + require.Fail(t, "equivocation handler was not notified") + } + + // re-adding an already stored proof at the equivocated nonce must not re-fire the event + require.True(t, pp.UpsertProof(proofRound7)) + select { + case <-eventChan: + require.Fail(t, "must not notify again on a same-hash re-add") + case <-time.After(50 * time.Millisecond): + } + }) + + t.Run("no notification for different nonces or same hash re-add", func(t *testing.T) { + t.Parallel() + + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + + notified := make(chan struct{}, 2) + pp.RegisterEquivocationHandler(func(_ data.HeaderProofHandler, _ []data.HeaderProofHandler) { + notified <- struct{}{} + }) + + require.True(t, pp.AddProof(proof1)) + require.True(t, pp.AddProof(proof2)) + require.False(t, pp.AddProof(proof1)) + require.True(t, pp.UpsertProof(proof1)) + + select { + case <-notified: + require.Fail(t, "must not notify without a different-hash proof at the same nonce") + case <-time.After(50 * time.Millisecond): + } + }) + + t.Run("cleanup removes all proofs at the nonce", func(t *testing.T) { + t.Parallel() + + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + require.True(t, pp.AddProof(proofRound6)) + require.True(t, pp.AddProof(proofRound7)) + + err := pp.CleanupProofsBehindNonce(shardID, 5+cleanupDelta+1) + require.Nil(t, err) + + require.False(t, pp.HasProof(shardID, []byte("hashA"))) + require.False(t, pp.HasProof(shardID, []byte("hashB"))) + _, err = pp.GetProofsByNonce(5, shardID) + require.NotNil(t, err) + }) +} + +func TestProofsPool_GetProofsByNonce_Missing(t *testing.T) { + t.Parallel() + + pp := proofscache.NewProofsPool(cleanupDelta, bucketSize) + + _, err := pp.GetProofsByNonce(5, shardID) + require.NotNil(t, err) + + _ = pp.AddProof(proof1) + _, err = pp.GetProofsByNonce(5, shardID) + require.NotNil(t, err) +} + func TestProofsPool_AddProofIfNoneAtNonce(t *testing.T) { t.Parallel() diff --git a/dataRetriever/interface.go b/dataRetriever/interface.go index 4cb4113d0e..af4facf795 100644 --- a/dataRetriever/interface.go +++ b/dataRetriever/interface.go @@ -393,9 +393,11 @@ type ProofsPool interface { AddProofIfNoneAtNonce(headerProof data.HeaderProofHandler) (bool, data.HeaderProofHandler) UpsertProof(headerProof data.HeaderProofHandler) bool RegisterHandler(handler func(headerProof data.HeaderProofHandler)) + RegisterEquivocationHandler(handler func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler)) CleanupProofsBehindNonce(shardID uint32, nonce uint64) error GetProof(shardID uint32, headerHash []byte) (data.HeaderProofHandler, error) GetProofByNonce(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) + GetProofsByNonce(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) HasProof(shardID uint32, headerHash []byte) bool IsProofInPoolEqualTo(headerProof data.HeaderProofHandler) bool IsInterfaceNil() bool diff --git a/dataRetriever/resolvers/equivalentProofsResolver.go b/dataRetriever/resolvers/equivalentProofsResolver.go index 06e0119ce5..b2991d50f1 100644 --- a/dataRetriever/resolvers/equivalentProofsResolver.go +++ b/dataRetriever/resolvers/equivalentProofsResolver.go @@ -8,12 +8,13 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/data/typeConverters" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process/interceptors/processor" "github.com/multiversx/mx-chain-go/storage" - logger "github.com/multiversx/mx-chain-logger-go" ) // maxBuffToSendEquivalentProofs represents max buffer size to send in bytes @@ -162,14 +163,22 @@ func (res *equivalentProofsResolver) resolveMultipleHashesRequest(hashShardKeysB return res.sendEquivalentProofsForHashes(equivalentProofsForHashes, pid, source) } -// resolveNonceRequest sends the response for a nonce request +// resolveNonceRequest sends the response for a nonce request; all proofs held at the nonce are +// sent, each as a separate message (the interceptor expects one proof per message) func (res *equivalentProofsResolver) resolveNonceRequest(nonceShardKey []byte, epoch uint32, pid core.PeerID, source p2p.MessageHandler) error { - data, err := res.fetchEquivalentProofFromNonceAsByteSlice(nonceShardKey, epoch) + proofsBuffs, err := res.fetchEquivalentProofsFromNonceAsByteSlices(nonceShardKey, epoch) if err != nil { - return fmt.Errorf("resolveNonceRequest.fetchEquivalentProofFromNonceAsByteSlice error %w", err) + return fmt.Errorf("resolveNonceRequest.fetchEquivalentProofsFromNonceAsByteSlices error %w", err) } - return res.Send(data, pid, source) + for _, buff := range proofsBuffs { + err = res.Send(buff, pid, source) + if err != nil { + return err + } + } + + return nil } // sendEquivalentProofsForHashes sends multiple equivalent proofs for specific hashes @@ -221,19 +230,35 @@ func (res *equivalentProofsResolver) fetchEquivalentProofAsByteSlice(headerHash return res.marshalizer.Marshal(proof) } -// fetchEquivalentProofFromNonceAsByteSlice returns the value from equivalent proofs pool or storage if exists -func (res *equivalentProofsResolver) fetchEquivalentProofFromNonceAsByteSlice(nonceShardKey []byte, epoch uint32) ([]byte, error) { +// fetchEquivalentProofsFromNonceAsByteSlices returns all proofs held at the nonce from the +// equivalent proofs pool, or the single stored one from storage if the pool has none +func (res *equivalentProofsResolver) fetchEquivalentProofsFromNonceAsByteSlices(nonceShardKey []byte, epoch uint32) ([][]byte, error) { headerNonce, shardID, err := common.GetNonceAndShardFromKey(nonceShardKey) if err != nil { - return nil, fmt.Errorf("fetchEquivalentProofFromNonceAsByteSlice.getNonceAndShard error %w", err) + return nil, fmt.Errorf("fetchEquivalentProofsFromNonceAsByteSlices.getNonceAndShard error %w", err) } - proof, err := res.equivalentProofsPool.GetProofByNonce(headerNonce, shardID) + proofs, err := res.equivalentProofsPool.GetProofsByNonce(headerNonce, shardID) if err != nil { - return res.getProofFromStorageByNonce(headerNonce, shardID, epoch) + buff, errStorage := res.getProofFromStorageByNonce(headerNonce, shardID, epoch) + if errStorage != nil { + return nil, errStorage + } + + return [][]byte{buff}, nil } - return res.marshalizer.Marshal(proof) + proofsBuffs := make([][]byte, 0, len(proofs)) + for _, proof := range proofs { + buff, errMarshal := res.marshalizer.Marshal(proof) + if errMarshal != nil { + return nil, errMarshal + } + + proofsBuffs = append(proofsBuffs, buff) + } + + return proofsBuffs, nil } // getProofFromStorageByNonce returns the value from equivalent storage if exists diff --git a/dataRetriever/resolvers/equivalentProofsResolver_test.go b/dataRetriever/resolvers/equivalentProofsResolver_test.go index 6539617067..5584913daa 100644 --- a/dataRetriever/resolvers/equivalentProofsResolver_test.go +++ b/dataRetriever/resolvers/equivalentProofsResolver_test.go @@ -12,6 +12,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers" @@ -21,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" - "github.com/stretchr/testify/require" ) var ( @@ -599,11 +600,11 @@ func TestEquivalentProofsResolver_ProcessReceivedMessage(t *testing.T) { args := createMockArgEquivalentProofsResolver() args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ - GetProofByNonceCalled: func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { require.Equal(t, uint64(1), headerNonce) require.Equal(t, uint32(1), shardID) - return &block.HeaderProof{}, nil + return []data.HeaderProofHandler{&block.HeaderProof{}}, nil }, } mockMarshaller := &marshallerMock.MarshalizerMock{} @@ -663,10 +664,10 @@ func TestEquivalentProofsResolver_ProcessReceivedMessage(t *testing.T) { providedMetaNonceKey := fmt.Sprintf("%d-%d", 1, core.MetachainShardId) // meta for coverage args := createMockArgEquivalentProofsResolver() - wasGetProofByNonceCalled := false + wasGetProofsByNonceCalled := false args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ - GetProofByNonceCalled: func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { - wasGetProofByNonceCalled = true + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { + wasGetProofsByNonceCalled = true require.Equal(t, uint64(1), headerNonce) require.Equal(t, core.MetachainShardId, shardID) @@ -695,16 +696,16 @@ func TestEquivalentProofsResolver_ProcessReceivedMessage(t *testing.T) { msgID, err := res.ProcessReceivedMessage(createRequestMsg(dataRetriever.NonceType, []byte(providedMetaNonceKey)), fromConnectedPeer, &p2pmocks.MessengerStub{}) require.True(t, errors.Is(err, expectedErr)) require.Nil(t, msgID) - require.True(t, wasGetProofByNonceCalled) + require.True(t, wasGetProofsByNonceCalled) }) t.Run("resolveNonceRequest: nonce not found anywhere should error", func(t *testing.T) { t.Parallel() args := createMockArgEquivalentProofsResolver() - wasGetProofByNonceCalled := false + wasGetProofsByNonceCalled := false args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ - GetProofByNonceCalled: func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { - wasGetProofByNonceCalled = true + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { + wasGetProofsByNonceCalled = true require.Equal(t, uint64(1), headerNonce) require.Equal(t, uint32(1), shardID) @@ -736,25 +737,28 @@ func TestEquivalentProofsResolver_ProcessReceivedMessage(t *testing.T) { msgID, err := res.ProcessReceivedMessage(createRequestMsg(dataRetriever.NonceType, providedNonceKey), fromConnectedPeer, &p2pmocks.MessengerStub{}) require.True(t, errors.Is(err, expectedErr)) require.Nil(t, msgID) - require.True(t, wasGetProofByNonceCalled) + require.True(t, wasGetProofsByNonceCalled) require.True(t, wasSearchFirstCalled) }) - t.Run("resolveNonceRequest: should work and return from pool", func(t *testing.T) { + t.Run("resolveNonceRequest: should work and return all proofs from pool", func(t *testing.T) { t.Parallel() + proofLowRound := &block.HeaderProof{HeaderHash: []byte("hashA"), HeaderNonce: 1, HeaderRound: 5, HeaderShardId: 1} + proofHighRound := &block.HeaderProof{HeaderHash: []byte("hashB"), HeaderNonce: 1, HeaderRound: 6, HeaderShardId: 1} + args := createMockArgEquivalentProofsResolver() args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ - GetProofByNonceCalled: func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { require.Equal(t, uint64(1), headerNonce) require.Equal(t, uint32(1), shardID) - return &block.HeaderProof{}, nil + return []data.HeaderProofHandler{proofLowRound, proofHighRound}, nil }, } - wasSendCalled := false + sentBuffs := make([][]byte, 0) args.SenderResolver = &mock.TopicResolverSenderStub{ SendCalled: func(buff []byte, peer core.PeerID, source p2p.MessageHandler) error { - wasSendCalled = true + sentBuffs = append(sentBuffs, buff) return nil }, @@ -765,14 +769,42 @@ func TestEquivalentProofsResolver_ProcessReceivedMessage(t *testing.T) { msgID, err := res.ProcessReceivedMessage(createRequestMsg(dataRetriever.NonceType, providedNonceKey), fromConnectedPeer, &p2pmocks.MessengerStub{}) require.NoError(t, err) require.Nil(t, msgID) - require.True(t, wasSendCalled) + require.Equal(t, 2, len(sentBuffs)) + + // each message carries exactly one proof, in the order returned by the pool + for i, expectedProof := range []*block.HeaderProof{proofLowRound, proofHighRound} { + sentProof := &block.HeaderProof{} + require.Nil(t, args.Marshaller.Unmarshal(sentProof, sentBuffs[i])) + require.Equal(t, expectedProof, sentProof) + } + }) + t.Run("resolveNonceRequest: send failure should error", func(t *testing.T) { + t.Parallel() + + args := createMockArgEquivalentProofsResolver() + args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { + return []data.HeaderProofHandler{&block.HeaderProof{}}, nil + }, + } + args.SenderResolver = &mock.TopicResolverSenderStub{ + SendCalled: func(buff []byte, peer core.PeerID, source p2p.MessageHandler) error { + return expectedErr + }, + } + res, err := resolvers.NewEquivalentProofsResolver(args) + require.Nil(t, err) + + msgID, err := res.ProcessReceivedMessage(createRequestMsg(dataRetriever.NonceType, providedNonceKey), fromConnectedPeer, &p2pmocks.MessengerStub{}) + require.True(t, errors.Is(err, expectedErr)) + require.Nil(t, msgID) }) t.Run("resolveNonceRequest: should work and return from storage", func(t *testing.T) { t.Parallel() args := createMockArgEquivalentProofsResolver() args.EquivalentProofsPool = &dataRetrieverMocks.ProofsPoolMock{ - GetProofByNonceCalled: func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) { + GetProofsByNonceCalled: func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { require.Equal(t, uint64(1), headerNonce) require.Equal(t, uint32(1), shardID) diff --git a/dataRetriever/resolvers/headerResolver_test.go b/dataRetriever/resolvers/headerResolver_test.go index f94acf36ac..47b894fd39 100644 --- a/dataRetriever/resolvers/headerResolver_test.go +++ b/dataRetriever/resolvers/headerResolver_test.go @@ -13,8 +13,10 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/multiversx/mx-chain-go/dataRetriever" + proofscache "github.com/multiversx/mx-chain-go/dataRetriever/dataPool/proofsCache" "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers" "github.com/multiversx/mx-chain-go/p2p" @@ -784,6 +786,53 @@ func TestHeaderResolver_ProcessReceivedMessageRequestNonceTypePrefersProvenHeade assert.Equal(t, expectedBuff, sentBuff) } +func TestHeaderResolver_ProcessReceivedMessageRequestNonceTypeCompetingProofsServesLowestRound(t *testing.T) { + t.Parallel() + + requestedNonce := uint64(67) + hashX := []byte("hashX-low-round") + hashY := []byte("hashY-high-round") + headerX := &block.Header{Nonce: requestedNonce, Round: 1} + headerY := &block.Header{Nonce: requestedNonce, Round: 2} + + headers := &mock.HeadersCacherStub{} + headers.GetHeaderByNonceAndShardIdCalled = func(hdrNonce uint64, shardId uint32) ([]data.HeaderHandler, [][]byte, error) { + return []data.HeaderHandler{headerX, headerY}, [][]byte{hashX, hashY}, nil + } + + arg := createMockArgHeaderResolver() + var sentBuff []byte + arg.SenderResolver = &mock.TopicResolverSenderStub{ + SendCalled: func(buff []byte, peer core.PeerID, source p2p.MessageHandler) error { + sentBuff = buff + return nil + }, + } + arg.Headers = headers + arg.HeadersNoncesStorage = &storageStubs.StorerStub{ + SearchFirstCalled: func(key []byte) ([]byte, error) { + return nil, errKeyNotFound + }, + } + // real proofs pool with two competing proofs; the higher-round one arrives first, yet the + // lowest-round one must be served (canonical selection is by round, not arrival order) + proofsPool := proofscache.NewProofsPool(3, 100) + require.True(t, proofsPool.AddProof(&block.HeaderProof{HeaderHash: hashY, HeaderNonce: requestedNonce, HeaderRound: 2})) + require.True(t, proofsPool.AddProof(&block.HeaderProof{HeaderHash: hashX, HeaderNonce: requestedNonce, HeaderRound: 1})) + arg.Proofs = proofsPool + hdrRes, _ := resolvers.NewHeaderResolver(arg) + + _, err := hdrRes.ProcessReceivedMessage( + createRequestMsg(dataRetriever.NonceType, arg.NonceConverter.ToByteSlice(requestedNonce)), + fromConnectedPeerId, + &p2pmocks.MessengerStub{}, + ) + + assert.Nil(t, err) + expectedBuff, _ := arg.Marshaller.Marshal(headerX) + assert.Equal(t, expectedBuff, sentBuff) +} + func TestHeaderResolver_ProcessReceivedMessageRequestNonceTypeNoProofServesLastSeenFromPool(t *testing.T) { t.Parallel() diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 6409711644..e5a9b73728 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -289,6 +289,11 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { pcf.epochNotifier.RegisterNotifyHandler(currentEpochProvider) + appStatusHandler := pcf.statusCoreComponents.AppStatusHandler() + pcf.data.Datapool().Proofs().RegisterEquivocationHandler(func(_ data.HeaderProofHandler, _ []data.HeaderProofHandler) { + appStatusHandler.Increment(common.MetricNumEquivocationProofs) + }) + fallbackHeaderValidator, err := fallback.NewFallbackHeaderValidator( pcf.data.Datapool().Headers(), pcf.coreData.InternalMarshalizer(), diff --git a/node/metrics/metrics.go b/node/metrics/metrics.go index dec3bf09ac..19d4d829b3 100644 --- a/node/metrics/metrics.go +++ b/node/metrics/metrics.go @@ -8,10 +8,11 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/sharding" - logger "github.com/multiversx/mx-chain-logger-go" ) const initUint = uint64(0) @@ -44,6 +45,7 @@ func InitBaseMetrics(appStatusHandler core.AppStatusHandler) error { appStatusHandler.SetUInt64Value(common.MetricNumShardHeadersFromPool, initUint) appStatusHandler.SetUInt64Value(common.MetricNumShardHeadersProcessed, initUint) appStatusHandler.SetUInt64Value(common.MetricNumTimesInForkChoice, initUint) + appStatusHandler.SetUInt64Value(common.MetricNumEquivocationProofs, initUint) appStatusHandler.SetUInt64Value(common.MetricHighestFinalBlock, initUint) appStatusHandler.SetUInt64Value(common.MetricCountConsensusAcceptedBlocks, initUint) appStatusHandler.SetUInt64Value(common.MetricRoundsPassedInCurrentEpoch, initUint) diff --git a/node/metrics/metrics_test.go b/node/metrics/metrics_test.go index b107e227a2..215356d638 100644 --- a/node/metrics/metrics_test.go +++ b/node/metrics/metrics_test.go @@ -6,6 +6,9 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" @@ -13,8 +16,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/genesisMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" "github.com/multiversx/mx-chain-go/testscommon/statusHandler" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestInitBaseMetrics(t *testing.T) { @@ -38,6 +39,7 @@ func TestInitBaseMetrics(t *testing.T) { common.MetricNumShardHeadersFromPool, common.MetricNumShardHeadersProcessed, common.MetricNumTimesInForkChoice, + common.MetricNumEquivocationProofs, common.MetricHighestFinalBlock, common.MetricCountConsensusAcceptedBlocks, common.MetricRoundsPassedInCurrentEpoch, diff --git a/process/interceptors/processor/interface.go b/process/interceptors/processor/interface.go index ab3baf43c5..7a7ef5adef 100644 --- a/process/interceptors/processor/interface.go +++ b/process/interceptors/processor/interface.go @@ -2,6 +2,7 @@ package processor import ( "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-go/state" ) @@ -32,6 +33,7 @@ type EquivalentProofsPool interface { CleanupProofsBehindNonce(shardID uint32, nonce uint64) error GetProof(shardID uint32, headerHash []byte) (data.HeaderProofHandler, error) GetProofByNonce(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) + GetProofsByNonce(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) HasProof(shardID uint32, headerHash []byte) bool IsInterfaceNil() bool } diff --git a/process/sync/export_test.go b/process/sync/export_test.go index 687bf4ae17..8fc4f46419 100644 --- a/process/sync/export_test.go +++ b/process/sync/export_test.go @@ -11,6 +11,11 @@ import ( "github.com/multiversx/mx-chain-go/process" ) +// GetNextHeaderRequestingIfMissing - +func (boot *baseBootstrap) GetNextHeaderRequestingIfMissing() (data.HeaderHandler, []byte, error) { + return boot.getNextHeaderRequestingIfMissing() +} + // RequestHeaderWithNonce - func (boot *baseBootstrap) RequestHeaderWithNonce(nonce uint64) { if boot.shardCoordinator.SelfId() == core.MetachainShardId { diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 251c395d74..f91884a271 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -29,6 +29,7 @@ import ( "github.com/multiversx/mx-chain-go/consensus/round" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/blockchain" + proofscache "github.com/multiversx/mx-chain-go/dataRetriever/dataPool/proofsCache" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/sync" @@ -3911,3 +3912,59 @@ func TestShardBootstrap_SyncBlockLegacy(t *testing.T) { assert.Equal(t, errExpected, err) }) } + +func TestShardBootstrap_GetNextHeaderWithCompetingProofsUsesLowestRound(t *testing.T) { + t.Parallel() + + args := CreateShardBootstrapMockArguments() + + currentHeader := &block.Header{Nonce: 1} + args.ChainHandler = &testscommon.ChainHandlerStub{ + GetGenesisHeaderCalled: func() data.HeaderHandler { + return &block.Header{} + }, + GetCurrentBlockHeaderCalled: func() data.HeaderHandler { + return currentHeader + }, + } + + forkDetector := &mock.ForkDetectorMock{} + forkDetector.GetNotarizedHeaderHashCalled = func(nonce uint64) []byte { + return nil + } + args.ForkDetector = forkDetector + + hashLowRound := []byte("hashA-low-round") + hashHighRound := []byte("hashB-high-round") + headerLowRound := &block.Header{Nonce: 2, Round: 2} + + // real proofs pool: the higher-round proof arrives first, yet fork-choice must fetch the + // lowest-round header (canonical selection by round, not arrival order) + proofsPool := proofscache.NewProofsPool(3, 100) + require.True(t, proofsPool.AddProof(&block.HeaderProof{HeaderHash: hashHighRound, HeaderNonce: 2, HeaderRound: 3})) + require.True(t, proofsPool.AddProof(&block.HeaderProof{HeaderHash: hashLowRound, HeaderNonce: 2, HeaderRound: 2})) + + pools := createMockPools() + pools.ProofsCalled = func() dataRetriever.ProofsPool { + return proofsPool + } + pools.HeadersCalled = func() dataRetriever.HeadersPool { + return &mock.HeadersCacherStub{ + GetHeaderByHashCalled: func(hash []byte) (data.HeaderHandler, error) { + if bytes.Equal(hash, hashLowRound) { + return headerLowRound, nil + } + return nil, errors.New("requested header for the wrong proof") + }, + } + } + args.PoolsHolder = pools + + bs, err := sync.NewShardBootstrap(args) + require.Nil(t, err) + + header, hash, err := bs.GetNextHeaderRequestingIfMissing() + require.Nil(t, err) + require.Equal(t, hashLowRound, hash) + require.Equal(t, headerLowRound, header) +} diff --git a/process/track/baseBlockTrack.go b/process/track/baseBlockTrack.go index dd6f221352..7e1621468f 100644 --- a/process/track/baseBlockTrack.go +++ b/process/track/baseBlockTrack.go @@ -72,6 +72,10 @@ type baseBlockTrack struct { headers map[uint32]map[uint64][]*HeaderInfo maxNumHeadersToKeepPerShard int + mutProofPull sync.Mutex + proofPullPerShard map[uint32]*proofPullState + lastProofPullRound int64 + cancelFunc context.CancelFunc } @@ -149,12 +153,15 @@ func createBaseBlockTrack(arguments ArgBaseTracker) (*baseBlockTrack, error) { processConfigsHandler: arguments.ProcessConfigsHandler, ownShardTracker: tracker, requestHandler: arguments.RequestHandler, + proofPullPerShard: make(map[uint32]*proofPullState), + lastProofPullRound: -1, } var ctx context.Context ctx, bbt.cancelFunc = context.WithCancel(context.Background()) go bbt.sweepQuarantinedHeaders(ctx) + go bbt.pullProofsForContendedHeadsLoop(ctx) return bbt, nil } diff --git a/process/track/contendedHeadersProofPuller.go b/process/track/contendedHeadersProofPuller.go new file mode 100644 index 0000000000..db65da34cb --- /dev/null +++ b/process/track/contendedHeadersProofPuller.go @@ -0,0 +1,179 @@ +package track + +import ( + "context" + "time" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data" + + "github.com/multiversx/mx-chain-go/common" +) + +// proofPullCheckInterval is shorter than any round duration; requests still go out at most once +// per round, gated by the round index +const proofPullCheckInterval = 200 * time.Millisecond + +const maxProofPullBackoffRounds = 8 + +type proofPullState struct { + nonce uint64 + nextPullRound int64 + backoffRounds int64 +} + +type headCandidate struct { + shardID uint32 + head data.HeaderHandler + parent data.HeaderHandler +} + +func (bbt *baseBlockTrack) pullProofsForContendedHeadsLoop(ctx context.Context) { + ticker := time.NewTicker(proofPullCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + bbt.pullProofsForContendedHeads() + } + } +} + +// pullProofsForContendedHeads actively discovers competing proofs by requesting all proofs at the +// nonce of each contended-unsettled tracked head, at most once per round with per-nonce backoff +func (bbt *baseBlockTrack) pullProofsForContendedHeads() { + if !bbt.enableEpochsHandler.IsFlagEnabled(common.SupernovaFlag) { + return + } + + currentRound := bbt.roundHandler.Index() + + bbt.mutProofPull.Lock() + defer bbt.mutProofPull.Unlock() + + if currentRound <= bbt.lastProofPullRound { + return + } + bbt.lastProofPullRound = currentRound + + contendedHeads := bbt.getContendedUnsettledHeads() + + for shardID := range bbt.proofPullPerShard { + _, stillContended := contendedHeads[shardID] + if !stillContended { + delete(bbt.proofPullPerShard, shardID) + } + } + + for shardID, head := range contendedHeads { + state := bbt.proofPullPerShard[shardID] + if state == nil || state.nonce != head.GetNonce() { + state = &proofPullState{nonce: head.GetNonce(), nextPullRound: currentRound, backoffRounds: 1} + bbt.proofPullPerShard[shardID] = state + } + + if currentRound < state.nextPullRound { + continue + } + + log.Debug("pulling proofs for contended head", + "shardID", shardID, + "nonce", head.GetNonce(), + "headRound", head.GetRound(), + "currentRound", currentRound, + ) + bbt.requestHandler.RequestEquivalentProofByNonce(shardID, head.GetNonce()) + + state.nextPullRound = currentRound + state.backoffRounds + state.backoffRounds = min(state.backoffRounds*2, maxProofPullBackoffRounds) + } +} + +// getContendedUnsettledHeads returns, per shard, the tracked head that skipped at least one round +// after its parent (a competing proof could exist in the skipped rounds) and has no proofed child +func (bbt *baseBlockTrack) getContendedUnsettledHeads() map[uint32]data.HeaderHandler { + candidates := bbt.getTrackedHeadCandidates() + + contendedHeads := make(map[uint32]data.HeaderHandler) + for _, candidate := range candidates { + parent := candidate.parent + if check.IfNil(parent) { + var err error + parent, err = bbt.headersPool.GetHeaderByHash(candidate.head.GetPrevHash()) + if err != nil { + continue + } + } + + isContended := candidate.head.GetRound() > parent.GetRound()+1 + if !isContended { + continue + } + + _, err := bbt.proofsPool.GetProofByNonce(candidate.head.GetNonce()+1, candidate.shardID) + isSettledByChildProof := err == nil + if isSettledByChildProof { + continue + } + + contendedHeads[candidate.shardID] = candidate.head + } + + return contendedHeads +} + +// getTrackedHeadCandidates returns, per shard, the highest-nonce tracked header (lowest round on +// ties) together with its tracked parent, if any +func (bbt *baseBlockTrack) getTrackedHeadCandidates() []headCandidate { + bbt.mutHeaders.RLock() + defer bbt.mutHeaders.RUnlock() + + candidates := make([]headCandidate, 0, len(bbt.headers)) + for shardID, headersForShard := range bbt.headers { + head := getHighestNonceLowestRoundHeader(headersForShard) + if check.IfNil(head) || head.GetNonce() == 0 { + continue + } + + var parent data.HeaderHandler + for _, headerInfo := range headersForShard[head.GetNonce()-1] { + if string(headerInfo.Hash) == string(head.GetPrevHash()) { + parent = headerInfo.Header + break + } + } + + candidates = append(candidates, headCandidate{shardID: shardID, head: head, parent: parent}) + } + + return candidates +} + +func getHighestNonceLowestRoundHeader(headersForShard map[uint64][]*HeaderInfo) data.HeaderHandler { + var head data.HeaderHandler + highestNonce := uint64(0) + for nonce, headersInfo := range headersForShard { + if nonce < highestNonce { + continue + } + + for _, headerInfo := range headersInfo { + if check.IfNil(headerInfo.Header) { + continue + } + + keepCurrentHead := nonce == highestNonce && !check.IfNil(head) && headerInfo.Header.GetRound() >= head.GetRound() + if keepCurrentHead { + continue + } + + head = headerInfo.Header + highestNonce = nonce + } + } + + return head +} diff --git a/process/track/contendedHeadersProofPuller_test.go b/process/track/contendedHeadersProofPuller_test.go new file mode 100644 index 0000000000..9c88c6beaf --- /dev/null +++ b/process/track/contendedHeadersProofPuller_test.go @@ -0,0 +1,174 @@ +package track_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/process/mock" + "github.com/multiversx/mx-chain-go/process/track" + "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" +) + +type pullRequest struct { + shardID uint32 + nonce uint64 +} + +type trackedHeaderAdder interface { + AddTrackedHeader(header data.HeaderHandler, hash []byte) +} + +func createProofPullTrackerScaffold(supernovaEnabled bool) (track.ArgShardTracker, *mock.RoundHandlerMock, *[]pullRequest) { + args := CreateShardTrackerMockArguments() + + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { + return supernovaEnabled && flag == common.SupernovaFlag + }, + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return false + }, + } + + roundHandler := &mock.RoundHandlerMock{RoundIndex: 10} + args.RoundHandler = roundHandler + + requests := &[]pullRequest{} + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestEquivalentProofByNonceCalled: func(headerShard uint32, headerNonce uint64) { + *requests = append(*requests, pullRequest{shardID: headerShard, nonce: headerNonce}) + }, + } + + return args, roundHandler, requests +} + +func addHeadWithParent(sbt trackedHeaderAdder, headRound uint64) { + parentHash := []byte("parentHash") + parent := &block.Header{Nonce: 1, Round: 1} + head := &block.Header{Nonce: 2, Round: headRound, PrevHash: parentHash} + + sbt.AddTrackedHeader(parent, parentHash) + sbt.AddTrackedHeader(head, []byte("headHash")) +} + +func TestBaseBlockTrack_PullProofsForContendedHeads(t *testing.T) { + t.Parallel() + + t.Run("non-contended head should not request", func(t *testing.T) { + t.Parallel() + + args, _, requests := createProofPullTrackerScaffold(true) + sbt, err := track.NewShardBlockTrack(args) + require.Nil(t, err) + _ = sbt.Close() // stop background loops; the test drives the pull explicitly + + addHeadWithParent(sbt, 2) // round 2 = parent round + 1, no skipped round + + sbt.PullProofsForContendedHeads() + require.Empty(t, *requests) + }) + + t.Run("contended head should request once per round with backoff until settled", func(t *testing.T) { + t.Parallel() + + args, roundHandler, requests := createProofPullTrackerScaffold(true) + sbt, err := track.NewShardBlockTrack(args) + require.Nil(t, err) + _ = sbt.Close() // stop background loops; the test drives the pull explicitly + + addHeadWithParent(sbt, 5) // rounds 2-4 skipped after parent round 1 + + sbt.PullProofsForContendedHeads() + require.Equal(t, []pullRequest{{shardID: 0, nonce: 2}}, *requests) + + // same round: no new request + sbt.PullProofsForContendedHeads() + require.Equal(t, 1, len(*requests)) + + // backoff 1: next round fires + roundHandler.RoundIndex = 11 + sbt.PullProofsForContendedHeads() + require.Equal(t, 2, len(*requests)) + + // backoff 2: round 12 skipped, round 13 fires + roundHandler.RoundIndex = 12 + sbt.PullProofsForContendedHeads() + require.Equal(t, 2, len(*requests)) + roundHandler.RoundIndex = 13 + sbt.PullProofsForContendedHeads() + require.Equal(t, 3, len(*requests)) + + // settled by child proof: no more requests; the round is advanced before AddProof, which + // dispatches the tracker's receivedProof subscriber on another goroutine + roundHandler.RoundIndex = 17 + _ = args.PoolsHolder.Proofs().AddProof(&block.HeaderProof{ + HeaderHash: []byte("childHash"), + HeaderNonce: 3, + HeaderRound: 6, + HeaderShardId: 0, + }) + sbt.PullProofsForContendedHeads() + require.Equal(t, 3, len(*requests)) + }) + + t.Run("supernova disabled should not request", func(t *testing.T) { + t.Parallel() + + args, _, requests := createProofPullTrackerScaffold(false) + sbt, err := track.NewShardBlockTrack(args) + require.Nil(t, err) + _ = sbt.Close() // stop background loops; the test drives the pull explicitly + + addHeadWithParent(sbt, 5) + + sbt.PullProofsForContendedHeads() + require.Empty(t, *requests) + }) + + t.Run("contended head with unknown parent should not request", func(t *testing.T) { + t.Parallel() + + args, _, requests := createProofPullTrackerScaffold(true) + sbt, err := track.NewShardBlockTrack(args) + require.Nil(t, err) + _ = sbt.Close() // stop background loops; the test drives the pull explicitly + + head := &block.Header{Nonce: 2, Round: 5, PrevHash: []byte("unknownParent")} + sbt.AddTrackedHeader(head, []byte("headHash")) + + sbt.PullProofsForContendedHeads() + require.Empty(t, *requests) + }) + + t.Run("new contended head resets the backoff", func(t *testing.T) { + t.Parallel() + + args, roundHandler, requests := createProofPullTrackerScaffold(true) + sbt, err := track.NewShardBlockTrack(args) + require.Nil(t, err) + _ = sbt.Close() // stop background loops; the test drives the pull explicitly + + addHeadWithParent(sbt, 5) + + sbt.PullProofsForContendedHeads() + roundHandler.RoundIndex = 11 + sbt.PullProofsForContendedHeads() + require.Equal(t, 2, len(*requests)) + + // a new contended head at the next nonce fires immediately, regardless of prior backoff + newHead := &block.Header{Nonce: 3, Round: 9, PrevHash: []byte("headHash")} + sbt.AddTrackedHeader(newHead, []byte("newHeadHash")) + + roundHandler.RoundIndex = 12 + sbt.PullProofsForContendedHeads() + require.Equal(t, 3, len(*requests)) + require.Equal(t, pullRequest{shardID: 0, nonce: 3}, (*requests)[2]) + }) +} diff --git a/process/track/export_test.go b/process/track/export_test.go index 45eb3b87d9..e0bc255b4b 100644 --- a/process/track/export_test.go +++ b/process/track/export_test.go @@ -335,3 +335,8 @@ func (mbt *miniBlockTrack) GetConfirmedMiniBlockInfo(miniBlockHash []byte) (cach } return info.cacheID, info.nonce, true } + +// PullProofsForContendedHeads - +func (bbt *baseBlockTrack) PullProofsForContendedHeads() { + bbt.pullProofsForContendedHeads() +} diff --git a/testscommon/dataRetriever/proofsPoolMock.go b/testscommon/dataRetriever/proofsPoolMock.go index 4d99fa0e74..119450fa9f 100644 --- a/testscommon/dataRetriever/proofsPoolMock.go +++ b/testscommon/dataRetriever/proofsPoolMock.go @@ -7,15 +7,17 @@ import ( // ProofsPoolMock - type ProofsPoolMock struct { - AddProofCalled func(headerProof data.HeaderProofHandler) bool - AddProofIfNoneAtNonceCalled func(headerProof data.HeaderProofHandler) (bool, data.HeaderProofHandler) - UpsertProofCalled func(headerProof data.HeaderProofHandler) bool - CleanupProofsBehindNonceCalled func(shardID uint32, nonce uint64) error - GetProofCalled func(shardID uint32, headerHash []byte) (data.HeaderProofHandler, error) - GetProofByNonceCalled func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) - HasProofCalled func(shardID uint32, headerHash []byte) bool - IsProofInPoolEqualToCalled func(headerProof data.HeaderProofHandler) bool - RegisterHandlerCalled func(handler func(headerProof data.HeaderProofHandler)) + AddProofCalled func(headerProof data.HeaderProofHandler) bool + AddProofIfNoneAtNonceCalled func(headerProof data.HeaderProofHandler) (bool, data.HeaderProofHandler) + UpsertProofCalled func(headerProof data.HeaderProofHandler) bool + CleanupProofsBehindNonceCalled func(shardID uint32, nonce uint64) error + GetProofCalled func(shardID uint32, headerHash []byte) (data.HeaderProofHandler, error) + GetProofByNonceCalled func(headerNonce uint64, shardID uint32) (data.HeaderProofHandler, error) + GetProofsByNonceCalled func(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) + HasProofCalled func(shardID uint32, headerHash []byte) bool + IsProofInPoolEqualToCalled func(headerProof data.HeaderProofHandler) bool + RegisterHandlerCalled func(handler func(headerProof data.HeaderProofHandler)) + RegisterEquivocationHandlerCalled func(handler func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler)) } // AddProof - @@ -72,6 +74,15 @@ func (p *ProofsPoolMock) GetProofByNonce(headerNonce uint64, shardID uint32) (da return &block.HeaderProof{}, nil } +// GetProofsByNonce - +func (p *ProofsPoolMock) GetProofsByNonce(headerNonce uint64, shardID uint32) ([]data.HeaderProofHandler, error) { + if p.GetProofsByNonceCalled != nil { + return p.GetProofsByNonceCalled(headerNonce, shardID) + } + + return []data.HeaderProofHandler{&block.HeaderProof{}}, nil +} + // HasProof - func (p *ProofsPoolMock) HasProof(shardID uint32, headerHash []byte) bool { if p.HasProofCalled != nil { @@ -97,6 +108,13 @@ func (p *ProofsPoolMock) RegisterHandler(handler func(headerProof data.HeaderPro } } +// RegisterEquivocationHandler - +func (p *ProofsPoolMock) RegisterEquivocationHandler(handler func(headerProof data.HeaderProofHandler, competingProofs []data.HeaderProofHandler)) { + if p.RegisterEquivocationHandlerCalled != nil { + p.RegisterEquivocationHandlerCalled(handler) + } +} + // IsInterfaceNil - func (p *ProofsPoolMock) IsInterfaceNil() bool { return p == nil