diff --git a/eth/downloader/bor_downloader.go b/eth/downloader/bor_downloader.go index ae8b52714a..5481896aa3 100644 --- a/eth/downloader/bor_downloader.go +++ b/eth/downloader/bor_downloader.go @@ -76,7 +76,7 @@ var ( errNoPeers = errors.New("no peers to keep download active") errTimeout = errors.New("timeout") errEmptyHeaderSet = errors.New("empty header set by peer") - errPeersUnavailable = errors.New("no peers available or all tried for download") + ErrPeersUnavailable = errors.New("no peers available or all tried for download") errInvalidAncestor = errors.New("retrieved ancestor is invalid") errInvalidChain = errors.New("retrieved hash chain is invalid") errInvalidBody = errors.New("retrieved block body is invalid") @@ -87,7 +87,9 @@ var ( errTooOld = errors.New("peer's protocol version too old") errNoAncestorFound = errors.New("no common ancestor found") errNoPivotHeader = errors.New("pivot header is not found") - ErrMergeTransition = errors.New("legacy sync reached the merge") + // ErrPeerBackedOff is returned when a sync is attempted against a peer that is currently benched. + ErrPeerBackedOff = errors.New("peer is temporarily backed off") + ErrMergeTransition = errors.New("legacy sync reached the merge") ) // peerDropFn is a callback type for dropping a peer detected as malicious. @@ -375,29 +377,57 @@ func (d *Downloader) UnregisterPeer(id string) error { return nil } +func (d *Downloader) PeerBackoff(id string) time.Duration { + persistent := d.peers.persistentBackoff(id) + var live time.Duration + if peer := d.peers.Peer(id); peer != nil { + live = peer.backoffRemaining() + } + if persistent > live { + return persistent + } + return live +} + +func (d *Downloader) SetPeerBackoffForTesting(id string, duration time.Duration) bool { + peer := d.peers.Peer(id) + if peer == nil { + return false + } + if duration <= 0 { + peer.setBackoffForTesting(time.Time{}) + d.peers.clearJailForTesting(id) + return true + } + until := time.Now().Add(duration) + peer.setBackoffForTesting(until) + d.peers.recordJail(peer, until) + return true +} + // LegacySync tries to sync up our local block chain with a remote peer, both // adding various sanity checks as well as wrapping it with various log entries. func (d *Downloader) LegacySync(id string, head common.Hash, td, ttd *big.Int, mode SyncMode) error { + masterPeer := d.peers.Peer(id) + err := d.synchronise(id, head, td, ttd, mode, false, nil) switch err { - case nil, errBusy, errCanceled: + case nil, errBusy, errCanceled, errCancelContentProcessing, errTerminated: return err } - if errors.Is(err, errInvalidChain) || errors.Is(err, errBadPeer) || errors.Is(err, errTimeout) || - errors.Is(err, errStallingPeer) || errors.Is(err, errUnsyncedPeer) || errors.Is(err, errEmptyHeaderSet) || - errors.Is(err, errPeersUnavailable) || errors.Is(err, errTooOld) || errors.Is(err, errInvalidAncestor) { - log.Warn("Synchronisation failed, dropping peer", "peer", id, "err", err, "mode", d.getMode()) + if errors.Is(err, ErrPeerBackedOff) || isSyncCancellation(err) { + return err + } - if d.dropPeer == nil { - // The dropPeer method is nil when `--copydb` is used for a local copy. - // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored - log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", id) - } else { - d.dropPeer(id) - } + d.syncStatsLock.RLock() + origin, height := d.syncStatsChainOrigin, d.syncStatsChainHeight + d.syncStatsLock.RUnlock() + + log.Debug("Synchronisation failed, evaluating peer response", "peer", id, "target", head, "td", td, "ttd", ttd, "mode", mode, "origin", origin, "height", height, "err", err) + if d.handleSyncFailure(masterPeer, id, err) { return err } @@ -405,8 +435,7 @@ func (d *Downloader) LegacySync(id string, head common.Hash, td, ttd *big.Int, m return err // This is an expected fault, don't keep printing it in a spin-loop } - // Warn in case of any error thrown by whitelisting module - if errors.Is(err, whitelist.ErrNoRemote) || errors.Is(err, whitelist.ErrMismatch) { + if errors.Is(err, whitelist.ErrLongFutureChain) { log.Warn("Synchronisation failed due to whitelist validation", "peer", id, "err", err) return err } @@ -502,6 +531,9 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, if p == nil { return errUnknownPeer } + if p.backedOff() { + return ErrPeerBackedOff + } } if beaconPing != nil { @@ -1382,10 +1414,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, head uint64) e return err default: - // Header retrieval either timed out, or the peer failed in some strange way - // (e.g. disconnect). Consider the master peer bad and drop - p.log.Debug("Downloader: header fetch failed, dropping master peer", "peer", p.id, "err", err) - d.dropPeer(p.id) + p.log.Debug("Downloader: master header fetch failed", "peer", p.id, "from", from, "head", head, "ancestor", ancestor, "skeleton", skeleton, "pivoting", pivoting, "err", err) // Finish the sync gracefully instead of dumping the gathered data though for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { @@ -1405,7 +1434,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, head uint64) e case <-d.cancelCh: } - return fmt.Errorf("%w: header request failed: %v", errBadPeer, err) + return fmt.Errorf("%w: header request failed: %w", errBadPeer, err) } // If the pivot is being checked, move if it became stale and run the real retrieval var pivot uint64 @@ -1486,8 +1515,11 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, head uint64) e if skeleton { filled, hashset, proced, err := d.fillHeaderSkeleton(from, headers) if err != nil { - p.log.Debug("Skeleton chain invalid", "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) + p.log.Debug("Skeleton header fill failed", "from", from, "head", head, "processed", proced, "headers", len(headers), "err", err) + if isSyncCancellation(err) || errors.Is(err, ErrPeersUnavailable) || errors.Is(err, errNoPeers) || errors.Is(err, ErrPeerBackedOff) || errors.Is(err, errTimeout) { + return err + } + return fmt.Errorf("%w: %w", errInvalidChain, err) } headers = filled[proced:] @@ -1844,6 +1876,18 @@ func (d *Downloader) processFullSyncContent(ttd *big.Int, beaconMode bool) error } } +func (d *Downloader) reportBadBlock(block *types.Block) { + if d.badBlock == nil { + return + } + head, _, _, err := d.skeleton.Bounds() + if err != nil { + log.Error("Failed to retrieve beacon bounds for bad block reporting", "err", err) + return + } + d.badBlock(block.Header(), head) +} + func (d *Downloader) importBlockResults(results []*fetchResult) error { // Check for any early termination requests if len(results) == 0 { @@ -1874,23 +1918,16 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { // consensus-layer. if index, err := d.blockchain.InsertChain(blocks, d.syncAndProduceWitnesses); err != nil { if index < len(results) { - log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "firstnum", first.Number, "firsthash", first.Hash(), "lastnum", last.Number, "lasthash", last.Hash(), "err", err) // In post-merge, notify the engine API of encountered bad chains - if d.badBlock != nil { - head, _, _, err := d.skeleton.Bounds() - if err != nil { - log.Error("Failed to retrieve beacon bounds for bad block reporting", "err", err) - } else { - d.badBlock(blocks[index].Header(), head) - } - } + d.reportBadBlock(blocks[index]) } else { // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, // when it needs to preprocess blocks to import a sidechain. // The importer will put together a new list of blocks to import, which is a superset // of the blocks delivered from the downloader, and the indexing will be off. - log.Debug("Downloaded item processing failed on sidechain import", "index", index, "err", err) + log.Debug("Downloaded item processing failed on sidechain import", "index", index, "firstnum", first.Number, "firsthash", first.Hash(), "lastnum", last.Number, "lasthash", last.Hash(), "err", err) } // If we've received too long future chain error (from whitelisting service), @@ -1899,7 +1936,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { return fmt.Errorf("%v: %w", errInvalidChain, err) } - return fmt.Errorf("%w: %v", errInvalidChain, err) + return fmt.Errorf("%w: %w", errInvalidChain, err) } return nil @@ -2352,8 +2389,8 @@ func (d *Downloader) importBlockResultsStateless(results []*fetchResult) error { } // Import the batch of blocks if index, err := d.blockchain.InsertChainStateless(blocks, witnesses); err != nil { - log.Warn("Invalid block body", "index", index, "hash", blocks[index].Hash(), "err", err) - return errInvalidBody // Or a more specific error if possible + log.Warn("Stateless block import failed", "index", index, "hash", blocks[index].Hash(), "err", err) + return errInvalidBody } return nil diff --git a/eth/downloader/bor_downloader_test.go b/eth/downloader/bor_downloader_test.go index adddfdfdb5..3f83dd40e5 100644 --- a/eth/downloader/bor_downloader_test.go +++ b/eth/downloader/bor_downloader_test.go @@ -532,7 +532,7 @@ func TestSnapSyncFailsOverEth68(t *testing.T) { if err == nil { t.Fatalf("snap sync succeeded instead of failing on eth/68 peer") } - if !errors.Is(err, errPeersUnavailable) { + if !errors.Is(err, ErrPeersUnavailable) { t.Fatalf("unexpected error while snap syncing with eth/68 peer: %v", err) } } @@ -1058,26 +1058,33 @@ func TestBlockHeaderAttackerDropping68(t *testing.T) { testBlockHeaderAttackerDr func TestBlockHeaderAttackerDropping69(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH69) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { - // Define the disconnection requirement for individual hash fetch errors tests := []struct { - result error - drop bool + name string + result error + drop bool + backoff bool }{ - {nil, false}, // Sync succeeded, all is well - {errBusy, false}, // Sync is already in progress, no problem - {errUnknownPeer, false}, // Peer is unknown, was already dropped, don't double drop - {errBadPeer, true}, // Peer was deemed bad for some reason, drop it - {errStallingPeer, true}, // Peer was detected to be stalling, drop it - {errUnsyncedPeer, true}, // Peer was detected to be unsynced, drop it - {errNoPeers, false}, // No peers to download from, soft race, no issue - {errTimeout, true}, // No hashes received in due time, drop the peer - {errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end - {errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser - {errInvalidAncestor, true}, // Agreed upon ancestor is not acceptable, drop the chain rewriter - {errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop - {errInvalidBody, false}, // A bad peer was detected, but not the sync origin - {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin - {errCancelContentProcessing, false}, // Synchronisation was canceled, origin may be innocent, don't drop + {name: "success"}, + {name: "busy", result: errBusy}, + {name: "unknown peer", result: errUnknownPeer}, + {name: "bad peer", result: errBadPeer, drop: true}, + {name: "stalling peer", result: errStallingPeer, backoff: true}, + {name: "unsynced peer", result: errUnsyncedPeer, backoff: true}, + {name: "no peers", result: errNoPeers}, + {name: "timeout", result: errTimeout, backoff: true}, + {name: "empty header set", result: errEmptyHeaderSet, backoff: true}, + {name: "peers unavailable", result: ErrPeersUnavailable}, + {name: "invalid ancestor", result: errInvalidAncestor, drop: true}, + {name: "invalid chain", result: errInvalidChain, drop: true}, + { + name: "pruned sidechain mismatch", + result: fmt.Errorf("%w: %w", errInvalidChain, errors.New(sidechainGhostStateMsg)), + backoff: true, + }, + {name: "invalid body", result: errInvalidBody}, + {name: "invalid receipt", result: errInvalidReceipt}, + {name: "no ancestor found", result: errNoAncestorFound, backoff: true}, + {name: "content processing canceled", result: errCancelContentProcessing}, } // Run the tests and check disconnection status tester := newTester(t) @@ -1099,8 +1106,161 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { tester.downloader.LegacySync(id, tester.chain.Genesis().Hash(), big.NewInt(1000), nil, FullSync) if _, ok := tester.peers[id]; !ok != tt.drop { - t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) + t.Errorf("test %q: peer drop mismatch for %v: have %v, want %v", tt.name, tt.result, !ok, tt.drop) + } + if tt.drop { + continue } + assertPeerBackoffState(t, tt.name, tt.result, tt.backoff, tester.downloader.peers.Peer(id)) + } +} + +func assertPeerBackoffState(t *testing.T, name string, result error, wantBackoff bool, peer *peerConnection) { + t.Helper() + if wantBackoff { + if peer == nil || !peer.backedOff() { + t.Errorf("test %q: peer backoff mismatch for %v: have false, want true", name, result) + } + return + } + if peer != nil && peer.backedOff() { + t.Errorf("test %q: unexpected peer backoff for %v", name, result) + } +} + +func TestDownloaderPeerBackoff(t *testing.T) { + tester := newTester(t) + defer tester.terminate() + + chain := testChainBase.shorten(1) + tester.newPeer("peer", eth.ETH69, chain.blocks[1:]) + + peer := tester.downloader.peers.Peer("peer") + if peer == nil { + t.Fatal("registered peer not found") + } + peer.backoffFor(30 * time.Second) + + remaining := tester.downloader.PeerBackoff("peer") + if remaining <= 0 || remaining > 30*time.Second { + t.Fatalf("peer backoff mismatch: have %v, want up to %v", remaining, 30*time.Second) + } + if remaining := tester.downloader.PeerBackoff("unknown"); remaining != 0 { + t.Fatalf("unknown peer backoff mismatch: have %v, want 0", remaining) + } +} + +func TestDownloaderPeerBackoffConsultsPersistedPenalties(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet()} + live := newPeerConnection("dup", eth.ETH69, nil, log.New("id", "dup")) + if err := d.peers.Register(live); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + if live.backedOff() { + t.Fatal("freshly registered peer should not be backed off") + } + + d.peers.lock.Lock() + d.peers.jailed["dup"] = time.Now().Add(peerJailBackoff) + d.peers.lock.Unlock() + if d.PeerBackoff("dup") <= 0 { + t.Fatal("PeerBackoff must consult the persisted jail map, not just the live peer") + } + + if remaining := d.PeerBackoff("unknown"); remaining != 0 { + t.Fatalf("unknown peer backoff mismatch: have %v, want 0", remaining) + } +} + +func TestSetPeerBackoffForTestingPersistsAndClearsJail(t *testing.T) { + d := &Downloader{peers: newPeerSet()} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New("id", "peer")) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + if !d.SetPeerBackoffForTesting("peer", time.Minute) { + t.Fatal("SetPeerBackoffForTesting must succeed for a registered peer") + } + if remaining := d.peers.persistentBackoff("peer"); remaining <= 0 { + t.Fatalf("SetPeerBackoffForTesting must persist the bench in the jail map: have %v, want > 0", remaining) + } + + if !d.SetPeerBackoffForTesting("peer", 0) { + t.Fatal("SetPeerBackoffForTesting with zero duration must succeed") + } + if remaining := d.peers.persistentBackoff("peer"); remaining != 0 { + t.Fatalf("SetPeerBackoffForTesting with zero duration must clear the persisted jail: have %v, want 0", remaining) + } + if peer.backedOff() { + t.Fatal("SetPeerBackoffForTesting with zero duration must clear the live backoff") + } +} + +func TestPeerBackoffExpiry(t *testing.T) { + peer := newPeerConnection("peer", eth.ETH69, nil, log.New("id", "peer")) + + if exp := peer.backoffExpiry(); !exp.IsZero() { + t.Fatalf("fresh peer expiry mismatch: have %v, want zero", exp) + } + + peer.backoffFor(30 * time.Second) + if exp := peer.backoffExpiry(); exp.IsZero() { + t.Fatal("backed-off peer should report a non-zero expiry") + } + if !peer.backedOff() { + t.Fatal("peer should be backed off") + } + + peer.backoffFor(-time.Second) + peer.lock.Lock() + peer.backoff = time.Now().Add(-time.Second) + peer.lock.Unlock() + if remaining := peer.backoffRemaining(); remaining != 0 { + t.Fatalf("expired backoff mismatch: have %v, want 0", remaining) + } + if exp := peer.backoffExpiry(); !exp.IsZero() { + t.Fatalf("expired backoff should reset to zero, have %v", exp) + } +} + +func TestLegacySyncReturnsPeerBackedOff(t *testing.T) { + tester := newTester(t) + defer tester.terminate() + + chain := testChainBase.shorten(1) + tester.newPeer("peer", eth.ETH69, chain.blocks[1:]) + + peer := tester.downloader.peers.Peer("peer") + if peer == nil { + t.Fatal("registered peer not found") + } + peer.backoffFor(30 * time.Second) + + err := tester.downloader.LegacySync("peer", tester.chain.Genesis().Hash(), big.NewInt(1000), nil, FullSync) + if err != ErrPeerBackedOff { + t.Fatalf("sync error mismatch: have %v, want %v", err, ErrPeerBackedOff) + } + if _, ok := tester.peers["peer"]; !ok { + t.Fatal("backed-off peer was dropped") + } +} + +func TestImportBlockResultsWrapsInvalidChain(t *testing.T) { + tester := newTester(t) + defer tester.terminate() + + header := &types.Header{ + ParentHash: common.Hash{0x01}, + Number: big.NewInt(1), + Difficulty: big.NewInt(1), + GasLimit: params.GenesisGasLimit, + } + err := tester.downloader.importBlockResults([]*fetchResult{{Header: header}}) + if !errors.Is(err, errInvalidChain) { + t.Fatalf("import error mismatch: have %v, want %v", err, errInvalidChain) } } diff --git a/eth/downloader/bor_fetch_helpers_test.go b/eth/downloader/bor_fetch_helpers_test.go new file mode 100644 index 0000000000..8c9e5e4aae --- /dev/null +++ b/eth/downloader/bor_fetch_helpers_test.go @@ -0,0 +1,427 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/log" +) + +type fakeTypedQueue struct { + k queueKind +} + +type witnessSupportPeer bool + +func (w witnessSupportPeer) Head() (common.Hash, *big.Int) { return common.Hash{}, nil } +func (w witnessSupportPeer) RequestHeadersByHash(common.Hash, int, int, bool, chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (w witnessSupportPeer) RequestHeadersByNumber(uint64, int, int, bool, chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (w witnessSupportPeer) RequestBodies([]common.Hash, chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (w witnessSupportPeer) RequestReceipts([]common.Hash, chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (w witnessSupportPeer) RequestWitnesses([]common.Hash, chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (w witnessSupportPeer) SupportsWitness() bool { return bool(w) } + +func (f fakeTypedQueue) kind() queueKind { return f.k } +func (f fakeTypedQueue) waker() chan bool { return nil } +func (f fakeTypedQueue) pending() int { return 0 } +func (f fakeTypedQueue) capacity(peer *peerConnection, rtt time.Duration) int { return 0 } +func (f fakeTypedQueue) updateCapacity(peer *peerConnection, items int, elapsed time.Duration) { +} +func (f fakeTypedQueue) reserve(peer *peerConnection, items int) (*fetchRequest, bool, bool) { + return nil, false, false +} +func (f fakeTypedQueue) unreserve(peer string) int { return 0 } +func (f fakeTypedQueue) request(peer *peerConnection, req *fetchRequest, resCh chan *eth.Response) (*eth.Request, error) { + return nil, nil +} +func (f fakeTypedQueue) deliver(peer *peerConnection, packet *eth.Response) (int, error) { + return 0, nil +} + +func TestQueueAcceptsPeer(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + kind queueKind + version uint + witness bool + peer Peer + accept bool + }{ + {name: "header accepts any version", kind: headerQueueKind, version: eth.ETH68, accept: true}, + {name: "receipt rejects below eth/69", kind: receiptQueueKind, version: eth.ETH68, accept: false}, + {name: "receipt accepts eth/69", kind: receiptQueueKind, version: eth.ETH69, accept: true}, + {name: "witness rejects peer without support", kind: witnessQueueKind, version: eth.ETH69, peer: witnessSupportPeer(false), accept: false}, + {name: "witness accepts peer with support", kind: witnessQueueKind, version: eth.ETH69, peer: witnessSupportPeer(true), accept: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + peer := newPeerConnection("peer", tt.version, tt.peer, log.New()) + if got := queueAcceptsPeer(fakeTypedQueue{k: tt.kind}, peer); got != tt.accept { + t.Fatalf("acceptance mismatch: have %v, want %v", got, tt.accept) + } + }) + } +} + +func TestCollectIdlePeersSurfacesBackoff(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet()} + + idle := newPeerConnection("idle", eth.ETH69, nil, log.New()) + benched := newPeerConnection("benched", eth.ETH69, nil, log.New()) + if err := d.peers.Register(idle); err != nil { + t.Fatalf("register idle peer: %v", err) + } + if err := d.peers.Register(benched); err != nil { + t.Fatalf("register benched peer: %v", err) + } + benched.backoffFor(30 * time.Second) + + queue := fakeTypedQueue{k: headerQueueKind} + idles, _, hasBackedOff, awaitingStale, nextBackoff := d.collectIdlePeers(queue, nil, nil) + + if len(idles) != 1 || idles[0].id != "idle" { + t.Fatalf("expected only the non-benched peer idle, got %v", idles) + } + if !hasBackedOff { + t.Fatal("expected hasBackedOff to be reported") + } + if awaitingStale != 0 { + t.Fatalf("no stale requests outstanding, want awaitingStale 0, have %d", awaitingStale) + } + if nextBackoff.IsZero() { + t.Fatal("expected a backoff wake instant for the benched peer") + } + if want := benched.backoffExpiry(); !nextBackoff.Equal(want) { + t.Fatalf("backoff wake mismatch: have %v, want %v", nextBackoff, want) + } +} + +func TestArmBackoffTimer(t *testing.T) { + t.Parallel() + + created, ch := armBackoffTimer(nil, time.Now().Add(20*time.Millisecond)) + if created == nil || ch == nil { + t.Fatal("expected an armed timer for a future wake instant") + } + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("armed timer did not fire") + } + + reused, ch := armBackoffTimer(created, time.Now().Add(20*time.Millisecond)) + if reused != created { + t.Fatal("expected the existing timer to be reused, not reallocated, for a future wake instant") + } + if ch == nil { + t.Fatal("expected an active channel when reusing a timer") + } + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("reused timer did not fire") + } + + retained, ch := armBackoffTimer(reused, time.Time{}) + if retained != reused { + t.Fatal("expected the timer to be retained for reuse on a zero wake instant") + } + if ch != nil { + t.Fatal("expected no active channel for a zero wake instant") + } + if retained.Stop() { + t.Fatal("a retained timer for a zero wake instant should already be stopped") + } + + fresh, ch := armBackoffTimer(nil, time.Now().Add(-time.Second)) + if fresh == nil || ch == nil { + t.Fatal("expected an armed timer for a past wake instant") + } + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("timer for a past wake instant did not fire promptly") + } +} + +func TestRespondToPeerDropRoutesToDropPeer(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + + d.respondToPeer(peer, peerFailureInvalidChain, errInvalidChain) + + select { + case id := <-dropped: + if id != "peer" { + t.Fatalf("dropped wrong peer: have %q, want %q", id, "peer") + } + default: + t.Fatal("drop reason should route to dropPeer") + } +} + +func TestClassifyPeerStallerBacksOff(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + queue := fakeTypedQueue{k: headerQueueKind} + + sent := time.Now() + fresh := map[string]*eth.Request{peer.id: {Sent: sent}} + idle, _, backedOff, awaiting, wake := d.classifyPeer(queue, peer, nil, fresh) + if idle || backedOff { + t.Fatalf("stale peer within grace period mismatch: idle=%v backedOff=%v", idle, backedOff) + } + if !awaiting { + t.Fatal("a stale peer within the grace period should be awaited for a late delivery") + } + if want := sent.Add(timeoutGracePeriod); !wake.Equal(want) { + t.Fatalf("within-grace wake mismatch: have %v, want %v", wake, want) + } + if peer.backedOff() { + t.Fatal("a peer within the grace period must not be backed off") + } + select { + case id := <-dropped: + t.Fatalf("peer within grace period was dropped: %q", id) + default: + } + + stale := map[string]*eth.Request{peer.id: {Sent: time.Now().Add(-2 * timeoutGracePeriod)}} + idle, _, backedOff, awaiting, wake = d.classifyPeer(queue, peer, nil, stale) + if idle { + t.Fatal("a stalling peer must not be offered as idle") + } + if !backedOff { + t.Fatal("a stalling peer past the grace period should be backed off") + } + if awaiting { + t.Fatal("a stalling peer past the grace period is no longer awaited for a late delivery") + } + if wake.IsZero() { + t.Fatal("a backed-off staller should schedule a wake instant") + } + if !peer.backedOff() { + t.Fatal("a stalling peer past the grace period should be benched") + } + select { + case id := <-dropped: + t.Fatalf("a stalling peer should back off, not drop: %q", id) + default: + } +} + +func TestClassifyPeerBusyPeerNotIdle(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet()} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + queue := fakeTypedQueue{k: headerQueueKind} + + pending := map[string]*eth.Request{peer.id: {Sent: time.Now()}} + idle, capacity, backedOff, awaiting, wake := d.classifyPeer(queue, peer, pending, nil) + if idle { + t.Fatal("a peer with an in-flight request must not be offered as idle") + } + if backedOff { + t.Fatal("a peer with an in-flight request must not be reported as backed off") + } + if awaiting { + t.Fatal("a peer with a live in-flight request is not an awaited staller") + } + if capacity != 0 { + t.Fatalf("busy peer capacity mismatch: have %d, want 0", capacity) + } + if !wake.IsZero() { + t.Fatalf("busy peer should not schedule a backoff wake: have %v", wake) + } +} + +func TestFetchStallError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stalled bool + idles int + capablePeers int + awaitingStale int + hasBackedOff bool + want error + }{ + {name: "not stalled never errors", stalled: false, idles: 0, capablePeers: 2, awaitingStale: 0, hasBackedOff: true, want: nil}, + {name: "all capable peers idle but unusable", stalled: true, idles: 2, capablePeers: 2, awaitingStale: 0, hasBackedOff: false, want: ErrPeersUnavailable}, + {name: "all capable peers backed off past grace", stalled: true, idles: 0, capablePeers: 2, awaitingStale: 0, hasBackedOff: true, want: ErrPeerBackedOff}, + {name: "awaiting a late delivery keeps waiting", stalled: true, idles: 0, capablePeers: 2, awaitingStale: 1, hasBackedOff: true, want: nil}, + {name: "no backed-off peer keeps waiting", stalled: true, idles: 1, capablePeers: 2, awaitingStale: 0, hasBackedOff: false, want: nil}, + {name: "idle-equals-capable wins over backed-off", stalled: true, idles: 2, capablePeers: 2, awaitingStale: 0, hasBackedOff: true, want: ErrPeersUnavailable}, + {name: "more idles than capable does not over-trigger", stalled: true, idles: 3, capablePeers: 2, awaitingStale: 0, hasBackedOff: true, want: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := fetchStallError(tt.stalled, tt.idles, tt.capablePeers, tt.awaitingStale, tt.hasBackedOff); got != tt.want { + t.Fatalf("fetchStallError mismatch: have %v, want %v", got, tt.want) + } + }) + } +} + +func TestCollectIdlePeersAllStalePastGraceUnblocksExit(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 4) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + queue := fakeTypedQueue{k: headerQueueKind} + + ids := []string{"peer-a", "peer-b"} + for _, id := range ids { + pc := newPeerConnection(id, eth.ETH69, nil, log.New("peer", id)) + if err := d.peers.Register(pc); err != nil { + t.Fatalf("register %s: %v", id, err) + } + } + + stales := make(map[string]*eth.Request) + for _, id := range ids { + stales[id] = ð.Request{Peer: id, Sent: time.Now().Add(-2 * timeoutGracePeriod)} + } + + idles, _, hasBackedOff, awaitingStale, nextBackoff := d.collectIdlePeers(queue, nil, stales) + if len(idles) != 0 { + t.Fatalf("stalling peers must not be offered as idle, got %v", idles) + } + if !hasBackedOff { + t.Fatal("stalling peers past grace should be reported as backed off") + } + if awaitingStale != 0 { + t.Fatalf("no stale peer is within grace, want awaitingStale 0, have %d", awaitingStale) + } + if nextBackoff.IsZero() { + t.Fatal("a backed-off staller should schedule a wake instant") + } + for _, id := range ids { + if !d.peers.Peer(id).backedOff() { + t.Fatalf("peer %s should be benched, not dropped", id) + } + } + select { + case id := <-dropped: + t.Fatalf("stalling peers should back off, not drop: %q", id) + default: + } + + if err := fetchStallError(true, len(idles), len(ids), awaitingStale, hasBackedOff); err != ErrPeerBackedOff { + t.Fatalf("all-stale-past-grace round should return ErrPeerBackedOff, have %v", err) + } +} + +func TestCollectIdlePeersWithinGraceKeepsWaiting(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet(), dropPeer: func(string) {}} + queue := fakeTypedQueue{k: headerQueueKind} + + pc := newPeerConnection("peer", eth.ETH69, nil, log.New("peer", "peer")) + if err := d.peers.Register(pc); err != nil { + t.Fatalf("register peer: %v", err) + } + + stales := map[string]*eth.Request{"peer": {Peer: "peer", Sent: time.Now()}} + idles, _, hasBackedOff, awaitingStale, nextBackoff := d.collectIdlePeers(queue, nil, stales) + if len(idles) != 0 { + t.Fatalf("a stalling peer must not be idle, got %v", idles) + } + if hasBackedOff { + t.Fatal("a peer within the grace period must not be backed off") + } + if awaitingStale != 1 { + t.Fatalf("the within-grace staller should be awaited, want 1, have %d", awaitingStale) + } + if nextBackoff.IsZero() { + t.Fatal("a within-grace staller should still schedule a grace-expiry wake") + } + if err := fetchStallError(true, len(idles), 1, awaitingStale, hasBackedOff); err != nil { + t.Fatalf("a round still awaiting a late delivery must keep waiting, have %v", err) + } +} + +func TestGradeStalePeerClearsEntryAndStrikesOnce(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet(), dropPeer: func(string) {}} + queue := fakeTypedQueue{k: headerQueueKind} + + pc := newPeerConnection("peer", eth.ETH69, nil, log.New("peer", "peer")) + if err := d.peers.Register(pc); err != nil { + t.Fatalf("register peer: %v", err) + } + + stales := map[string]*eth.Request{"peer": {Peer: "peer", Sent: time.Now().Add(-2 * timeoutGracePeriod)}} + + d.classifyPeer(queue, pc, nil, stales) + if _, ok := stales["peer"]; ok { + t.Fatal("a graded stale entry must be removed so an expired backoff does not re-grade the same timeout") + } + if got := softStrikeTally(d.peers, "peer"); got != 1 { + t.Fatalf("grading a stale entry once should record exactly one soft strike, have %d", got) + } + if !pc.backedOff() { + t.Fatal("a graded staller should be benched") + } + + pc.lock.Lock() + pc.backoff = time.Time{} + pc.lock.Unlock() + + d.classifyPeer(queue, pc, nil, stales) + if got := softStrikeTally(d.peers, "peer"); got != 1 { + t.Fatalf("a peer with no outstanding stale entry must not accrue a second strike, have %d", got) + } +} + +func softStrikeTally(ps *peerSet, id string) int { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.softStrikes[id].count +} diff --git a/eth/downloader/bor_fetchers.go b/eth/downloader/bor_fetchers.go index 29107804c3..f81b9ff9cd 100644 --- a/eth/downloader/bor_fetchers.go +++ b/eth/downloader/bor_fetchers.go @@ -46,6 +46,9 @@ func (d *Downloader) fetchHeadersByHash(p *peerConnection, hash common.Hash, amo defer timeoutTimer.Stop() select { + case <-d.cancelCh: + return nil, nil, errCanceled + case <-timeoutTimer.C: // Header retrieval timed out, update the metrics p.log.Debug("Header request timed out", "elapsed", ttl) diff --git a/eth/downloader/bor_fetchers_concurrent.go b/eth/downloader/bor_fetchers_concurrent.go index 387c3b46f3..822d116003 100644 --- a/eth/downloader/bor_fetchers_concurrent.go +++ b/eth/downloader/bor_fetchers_concurrent.go @@ -18,6 +18,7 @@ package downloader import ( "errors" + "fmt" "sort" "time" @@ -103,6 +104,103 @@ type typedQueue interface { deliver(peer *peerConnection, packet *eth.Response) (int, error) } +func (d *Downloader) collectIdlePeers(queue typedQueue, pending, stales map[string]*eth.Request) (idles []*peerConnection, caps []int, hasBackedOff bool, awaitingStale int, nextBackoff time.Time) { + for _, peer := range d.peers.AllPeers() { + idle, capacity, backedOff, awaiting, wake := d.classifyPeer(queue, peer, pending, stales) + if idle { + idles = append(idles, peer) + caps = append(caps, capacity) + } + if backedOff { + hasBackedOff = true + } + if awaiting { + awaitingStale++ + } + nextBackoff = earlierBackoff(nextBackoff, wake) + } + + return idles, caps, hasBackedOff, awaitingStale, nextBackoff +} + +func (d *Downloader) classifyPeer(queue typedQueue, peer *peerConnection, pending, stales map[string]*eth.Request) (idle bool, capacity int, backedOff bool, awaitingStale bool, wake time.Time) { + req, stale := pending[peer.id], stales[peer.id] + switch { + case req == nil && stale == nil: + free, capacity, backoff := d.freePeerStatus(queue, peer) + return free, capacity, !backoff.IsZero(), false, backoff + case stale != nil: + if d.gradeStalePeer(queue, peer, stale) { + delete(stales, peer.id) + stale.Close() + } + if peer.backedOff() { + return false, 0, true, false, peer.backoffExpiry() + } + return false, 0, false, true, stale.Sent.Add(timeoutGracePeriod) + default: + return false, 0, false, false, time.Time{} + } +} + +func (d *Downloader) freePeerStatus(queue typedQueue, peer *peerConnection) (idle bool, capacity int, backoff time.Time) { + if !queueAcceptsPeer(queue, peer) { + return false, 0, time.Time{} + } + if peer.backedOff() { + return false, 0, peer.backoffExpiry() + } + + return true, queue.capacity(peer, time.Second), time.Time{} +} + +func queueAcceptsPeer(queue typedQueue, peer *peerConnection) bool { + switch queue.kind() { + case witnessQueueKind: + if !peer.peer.SupportsWitness() { + peer.log.Trace("Skipping peer for witness fetch - no witness support", "peer", peer.id) + return false + } + case receiptQueueKind: + if peer.version < eth.ETH69 { + peer.log.Trace("Skipping peer for fetching receipts - version below eth/69", "peer", peer.id) + return false + } + } + + return true +} + +func (d *Downloader) gradeStalePeer(queue typedQueue, peer *peerConnection, stale *eth.Request) bool { + if stale.Sent.IsZero() { + stale.Sent = time.Now() + } + waited := time.Since(stale.Sent) + if waited <= timeoutGracePeriod { + return false + } + if peer.backedOff() { + return false + } + peer.log.Warn("Peer stalling, backing off", "waited", common.PrettyDuration(waited), "graceperiod", timeoutGracePeriod, "queueKind", queue.kind()) + d.respondToPeer(peer, peerFailureStalling, fmt.Errorf("%w: stalled for %s", errStallingPeer, common.PrettyDuration(waited))) + return true +} + +func fetchStallError(stalled bool, idles, capablePeers, awaitingStale int, hasBackedOff bool) error { + if !stalled { + return nil + } + if idles == capablePeers { + return ErrPeersUnavailable + } + if awaitingStale == 0 && idles < capablePeers && hasBackedOff { + return ErrPeerBackedOff + } + + return nil +} + // concurrentFetch iteratively downloads scheduled block parts, taking available // peers, reserving a chunk of fetch requests for each and waiting for delivery // or timeouts. @@ -154,6 +252,16 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { peeringSub := d.peers.SubscribeEvents(peering) defer peeringSub.Unsubscribe() + var ( + backoffTimer *time.Timer + backoffCh <-chan time.Time + ) + defer func() { + if backoffTimer != nil { + backoffTimer.Stop() + } + }() + // Prepare the queue and fetch block parts until the block header fetcher's done finished := false @@ -162,6 +270,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { if d.peers.Len() == 0 && !beaconMode { return errNoPeers } + var wakeBackoff time.Time // If there's nothing more to fetch, wait or terminate if queue.pending() == 0 { if len(pending) == 0 && finished { @@ -169,42 +278,11 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { } } else { // Send a download request to all idle peers, until throttled - var ( - idles []*peerConnection - caps []int - ) - isWitnessQueue := queue.kind() == witnessQueueKind isReceiptQueue := queue.kind() == receiptQueueKind - for _, peer := range d.peers.AllPeers() { - pending, stale := pending[peer.id], stales[peer.id] - if pending == nil && stale == nil { - // For witness fetching, skip peers that don't support the witness protocol - if isWitnessQueue && !peer.peer.SupportsWitness() { - peer.log.Trace("Skipping peer for witness fetch - no witness support", "peer", peer.id) - continue - } - - // eth/69 handlers also sends bor receipts via p2p. Skip peers - // below that to avoid missing bor receipts. - if isReceiptQueue && peer.version < eth.ETH69 { - peer.log.Trace("Skipping peer for fetching receipts - version below eth/69", "peer", peer.id) - continue - } - - idles = append(idles, peer) - caps = append(caps, queue.capacity(peer, time.Second)) - } else if stale != nil { - if waited := time.Since(stale.Sent); waited > timeoutGracePeriod { - // Request has been in flight longer than the grace period - // permitted it, consider the peer malicious attempting to - // stall the sync. - peer.log.Warn("Peer stalling, dropping", "waited", common.PrettyDuration(waited), "graceperiod", timeoutGracePeriod, "queueKind", queue.kind()) - d.dropPeer(peer.id) - } - } - } + idles, caps, hasBackedOffPeer, awaitingStale, nextBackoff := d.collectIdlePeers(queue, pending, stales) + wakeBackoff = nextBackoff sort.Sort(&peerCapacitySort{idles, caps}) @@ -286,10 +364,12 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { capablePeers = d.peers.Len() } - if !progressed && !throttled && len(pending) == 0 && len(idles) == capablePeers && queued > 0 && !beaconMode { - return errPeersUnavailable + stalledNoProgress := !progressed && !throttled && len(pending) == 0 && queued > 0 && !beaconMode + if err := fetchStallError(stalledNoProgress, len(idles), capablePeers, awaitingStale, hasBackedOffPeer); err != nil { + return err } } + backoffTimer, backoffCh = armBackoffTimer(backoffTimer, wakeBackoff) // Wait for something to happen select { case <-d.cancelCh: @@ -341,6 +421,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { } if req, ok := stales[peerid]; ok { + d.respondToPeer(event.peer, peerFailureStalling, fmt.Errorf("%w: peer left with a stale request", errStallingPeer)) delete(stales, peerid) req.Close() } @@ -398,23 +479,21 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { continue } + d.cancelLock.RLock() + master := peer.id == d.cancelPeer + d.cancelLock.RUnlock() + + if master { + peer.log.Debug("Downloader: master peer timed out, aborting sync", "queueKind", queue.kind(), "fails", fails) + d.cancel() + return errTimeout + } + if fails > 2 { peer.log.Debug("Downloader: peer exceeded fail threshold, zeroing capacity", "queueKind", queue.kind(), "fails", fails) queue.updateCapacity(peer, 0, 0) } else { - d.dropPeer(peer.id) - - // If this peer was the master peer, abort sync immediately - d.cancelLock.RLock() - master := peer.id == d.cancelPeer - d.cancelLock.RUnlock() - - peer.log.Debug("Downloader: dropping peer on timeout", "queueKind", queue.kind(), "fails", fails, "master", master) - - if master { - d.cancel() - return errTimeout - } + peer.log.Debug("Downloader: peer timed out, awaiting stale grace before grading", "queueKind", queue.kind(), "fails", fails) } case res := <-responses: @@ -447,14 +526,6 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { res.Done <- nil res.Req.Close() - if queue.kind() == witnessQueueKind { - for _, peer := range d.peers.AllPeers() { - log.Debug("Peer", "peer", peer.id, "peer", peer.peer, "queue kind", "witness") - } - - log.Debug("Peer", "peer1", res.Req.Peer, "peer2", d.peers.Peer(res.Req.Peer), "queue kind", "witness") - } - // If the peer was previously banned and failed to deliver its pack // in a reasonable time frame, ignore its message. if peer := d.peers.Peer(res.Req.Peer); peer != nil { @@ -479,6 +550,8 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { if !cont { finished = true } + + case <-backoffCh: } } } diff --git a/eth/downloader/bor_fetchers_concurrent_test.go b/eth/downloader/bor_fetchers_concurrent_test.go index d269c5f463..bbc19fd8c5 100644 --- a/eth/downloader/bor_fetchers_concurrent_test.go +++ b/eth/downloader/bor_fetchers_concurrent_test.go @@ -34,7 +34,7 @@ func (m *mockPeer) RequestReceipts([]common.Hash, chan *eth.Response) (*eth.Requ m.receiptRequested.Store(true) // Return a valid request so concurrentFetch can track it. // peer field is nil so Close() is a no-op (test-safe per dispatcher.go). - return ð.Request{Peer: m.id}, nil + return ð.Request{Peer: m.id, Sent: time.Now()}, nil } func (m *mockPeer) RequestWitnesses([]common.Hash, chan *eth.Response) (*eth.Request, error) { return nil, nil @@ -56,17 +56,29 @@ func newTestDownloader() *Downloader { // schedules a header with non-empty receipts so concurrentFetch enters the // receipt peer selection path. func scheduleReceiptTask(d *Downloader) { + scheduleReceiptTasks(d, 1) +} + +func scheduleReceiptTasks(d *Downloader, count int) { d.queue.Prepare(1, SnapSync) - header := &types.Header{ - Number: big.NewInt(1), - ReceiptHash: common.HexToHash("0x1"), // non-empty so receipts get scheduled + var ( + headers []*types.Header + hashes []common.Hash + ) + for i := 1; i <= count; i++ { + header := &types.Header{ + Number: big.NewInt(int64(i)), + ReceiptHash: common.Hash{byte(i)}, + } + headers = append(headers, header) + hashes = append(hashes, header.Hash()) } - d.queue.Schedule([]*types.Header{header}, []common.Hash{header.Hash()}, 1) + d.queue.Schedule(headers, hashes, 1) } // TestConcurrentFetchReceipts_OnlyEth68Peers verifies that concurrentFetch -// returns errPeersUnavailable when all peers are below eth/69, since only +// returns ErrPeersUnavailable when all peers are below eth/69, since only // eth/69 peers include bor receipts in responses. func TestConcurrentFetchReceipts_OnlyEth68Peers(t *testing.T) { d := newTestDownloader() @@ -88,8 +100,8 @@ func TestConcurrentFetchReceipts_OnlyEth68Peers(t *testing.T) { } err := d.concurrentFetch((*receiptQueue)(d), false) - if err != errPeersUnavailable { - t.Fatalf("expected errPeersUnavailable, got %v", err) + if err != ErrPeersUnavailable { + t.Fatalf("expected ErrPeersUnavailable, got %v", err) } for _, peer := range mockPeers { @@ -136,3 +148,93 @@ func TestConcurrentFetchReceipts_MixedPeers(t *testing.T) { t.Error("eth/69 peer should have received a receipt request") } } + +func TestConcurrentFetchReceipts_BackedOffPeer(t *testing.T) { + d := newTestDownloader() + scheduleReceiptTask(d) + + peer := &mockPeer{id: "peer-eth69", protocol: eth.ETH69} + pc := newPeerConnection(peer.id, peer.protocol, peer, log.New("peer", peer.id)) + pc.backoffFor(time.Minute) + if err := d.peers.Register(pc); err != nil { + t.Fatal(err) + } + + err := d.concurrentFetch((*receiptQueue)(d), false) + if err != ErrPeerBackedOff { + t.Fatalf("expected ErrPeerBackedOff, got %v", err) + } + if peer.receiptRequested.Load() { + t.Fatal("backed-off peer should not receive a receipt request") + } +} + +func TestConcurrentFetchGradesDepartedStalePeer(t *testing.T) { + d := newTestDownloader() + d.peers.rates.OverrideTTLLimit = 50 * time.Millisecond + scheduleReceiptTask(d) + + peer := &mockPeer{id: "peer-eth69", protocol: eth.ETH69} + pc := newPeerConnection(peer.id, peer.protocol, peer, log.New("peer", peer.id)) + if err := d.peers.Register(pc); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + done <- d.concurrentFetch((*receiptQueue)(d), false) + }() + + waitFor := func(cond func() bool) bool { + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return true + } + time.Sleep(10 * time.Millisecond) + } + return cond() + } + + if !waitFor(func() bool { return peer.receiptRequested.Load() }) { + close(d.cancelCh) + <-done + t.Fatal("receipt request was never dispatched") + } + + time.Sleep(200 * time.Millisecond) + if err := d.peers.Unregister(pc.id); err != nil { + t.Fatalf("failed to unregister peer: %v", err) + } + + if !waitFor(func() bool { return softStrikeTally(d.peers, peer.id) >= 1 }) { + close(d.cancelCh) + <-done + t.Fatal("a peer that left with a stale request must be graded, not silently dropped") + } + + close(d.cancelCh) + <-done +} + +func TestConcurrentFetchMasterTimeoutAborts(t *testing.T) { + d := newTestDownloader() + d.peers.rates.OverrideTTLLimit = 50 * time.Millisecond + scheduleReceiptTask(d) + + peer := &mockPeer{id: "master-eth69", protocol: eth.ETH69} + pc := newPeerConnection(peer.id, peer.protocol, peer, log.New("peer", peer.id)) + if err := d.peers.Register(pc); err != nil { + t.Fatal(err) + } + d.cancelPeer = peer.id + + if err := d.concurrentFetch((*receiptQueue)(d), false); err != errTimeout { + t.Fatalf("expected errTimeout when the master peer times out, got %v", err) + } + select { + case <-d.cancelCh: + default: + t.Fatal("a master timeout must cancel the sync cycle") + } +} diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index e847d1703e..31aa4654de 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -45,4 +45,10 @@ var ( bodyItemDownloadTimer = metrics.NewRegisteredTimer("eth/downloader/bodies/item_download_duration", nil) receiptItemDownloadTimer = metrics.NewRegisteredTimer("eth/downloader/receipts/item_download_duration", nil) witnessItemDownloadTimer = metrics.NewRegisteredTimer("eth/downloader/witnesses/item_download_duration", nil) + + peerJailMeter = metrics.NewRegisteredMeter("eth/downloader/peer/response/jail", nil) + peerDropResponseMeter = metrics.NewRegisteredMeter("eth/downloader/peer/response/drop", nil) + peerSoftBackoffMeter = metrics.NewRegisteredMeter("eth/downloader/peer/response/backoff", nil) + peerMismatchMeter = metrics.NewRegisteredMeter("eth/downloader/peer/response/mismatch", nil) + peerGhostStateMeter = metrics.NewRegisteredMeter("eth/downloader/peer/response/ghoststate", nil) ) diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 15a6a59760..aef25ecfa4 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -54,6 +54,7 @@ type peerConnection struct { version uint // Eth protocol version number to switch strategies log log.Logger // Contextual logger to add extra infos to peer logs lock sync.RWMutex + backoff time.Time } // Peer encapsulates the methods required to synchronise with a remote full peer. @@ -89,6 +90,78 @@ func (p *peerConnection) Reset() { p.lacking = make(map[common.Hash]struct{}) } +func (p *peerConnection) backoffFor(duration time.Duration) { + if duration <= 0 { + return + } + until := time.Now().Add(duration) + + p.lock.Lock() + defer p.lock.Unlock() + + if until.After(p.backoff) { + p.backoff = until + } +} + +func (p *peerConnection) extendBackoff(until time.Time) { + p.lock.Lock() + defer p.lock.Unlock() + + if until.After(p.backoff) { + p.backoff = until + } +} + +func (p *peerConnection) backoffForClaim(duration time.Duration) bool { + p.lock.Lock() + defer p.lock.Unlock() + + now := time.Now() + claimed := p.backoff.IsZero() || !p.backoff.After(now) + if until := now.Add(duration); until.After(p.backoff) { + p.backoff = until + } + return claimed +} + +func (p *peerConnection) setBackoffForTesting(until time.Time) { + p.lock.Lock() + defer p.lock.Unlock() + + p.backoff = until +} + +func (p *peerConnection) backedOff() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return !p.backoff.IsZero() && p.backoff.After(time.Now()) +} + +func (p *peerConnection) backoffExpiry() time.Time { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.backoff +} + +func (p *peerConnection) backoffRemaining() time.Duration { + p.lock.Lock() + defer p.lock.Unlock() + + if p.backoff.IsZero() { + return 0 + } + now := time.Now() + if p.backoff.After(now) { + return p.backoff.Sub(now) + } + p.backoff = time.Time{} + + return 0 +} + // UpdateHeaderRate updates the peer's estimated header retrieval throughput with // the current measurement. func (p *peerConnection) UpdateHeaderRate(delivered int, elapsed time.Duration) { @@ -195,19 +268,238 @@ type peeringEvent struct { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peerConnection - rates *msgrate.Trackers // Set of rate trackers to give the sync a common beat - events event.Feed // Feed to publish peer lifecycle events on + peers map[string]*peerConnection + jailed map[string]time.Time // Backoff expiry per peer ID, surviving reconnects + softStrikes map[string]softStrikeRecord + mismatchStrikes map[string]softStrikeRecord + ghostStrikes map[string]softStrikeRecord + lastJailPrune time.Time + lastSoftPrune time.Time + lastMismatchPrune time.Time + lastGhostPrune time.Time + rates *msgrate.Trackers // Set of rate trackers to give the sync a common beat + events event.Feed // Feed to publish peer lifecycle events on lock sync.RWMutex } +type softStrikeRecord struct { + count int + windowStart time.Time +} + // newPeerSet creates a new peer set top track the active download sources. func newPeerSet() *peerSet { return &peerSet{ - peers: make(map[string]*peerConnection), - rates: msgrate.NewTrackers(log.New("proto", "eth")), + peers: make(map[string]*peerConnection), + jailed: make(map[string]time.Time), + softStrikes: make(map[string]softStrikeRecord), + mismatchStrikes: make(map[string]softStrikeRecord), + ghostStrikes: make(map[string]softStrikeRecord), + rates: msgrate.NewTrackers(log.New("proto", "eth")), + } +} + +func evictOldestStrike(m map[string]softStrikeRecord) { + var ( + oldestID string + oldest time.Time + ) + for id, record := range m { + if oldestID == "" || record.windowStart.Before(oldest) { + oldestID, oldest = id, record.windowStart + } + } + if oldestID != "" { + delete(m, oldestID) + } +} + +func evictSoonestJail(m map[string]time.Time) { + var ( + evictID string + soonest time.Time + ) + for id, expiry := range m { + if evictID == "" || expiry.Before(soonest) { + evictID, soonest = id, expiry + } + } + if evictID != "" { + delete(m, evictID) + } +} + +func (ps *peerSet) backoffSoftFailure(peer *peerConnection) (penalized, jailed bool) { + if !peer.backoffForClaim(peerSoftBackoff) { + return false, false + } + if ps.recordSoftFailure(peer.id, time.Now()) >= softFailureJailThreshold { + ps.clearSoftFailures(peer.id) + peer.backoffFor(peerJailBackoff) + jailed = true + } + ps.recordJail(peer, peer.backoffExpiry()) + return true, jailed +} + +func (ps *peerSet) recordSoftFailure(id string, now time.Time) int { + ps.lock.Lock() + defer ps.lock.Unlock() + + if now.Sub(ps.lastSoftPrune) >= backoffPruneInterval { + for peer, record := range ps.softStrikes { + if now.Sub(record.windowStart) > softFailureWindow { + delete(ps.softStrikes, peer) + } + } + ps.lastSoftPrune = now + } + + if _, ok := ps.softStrikes[id]; !ok && len(ps.softStrikes) >= maxStrikeEntries { + evictOldestStrike(ps.softStrikes) + } + + record := ps.softStrikes[id] + if record.count == 0 || now.Sub(record.windowStart) > softFailureWindow { + record = softStrikeRecord{windowStart: now} + } + record.count += 1 + ps.softStrikes[id] = record + + return record.count +} + +func (ps *peerSet) clearSoftFailures(id string) { + ps.lock.Lock() + defer ps.lock.Unlock() + + delete(ps.softStrikes, id) +} + +func (ps *peerSet) recordMismatch(id string, now time.Time) int { + ps.lock.Lock() + defer ps.lock.Unlock() + + if now.Sub(ps.lastMismatchPrune) >= backoffPruneInterval { + for peer, record := range ps.mismatchStrikes { + if now.Sub(record.windowStart) > whitelistMismatchWindow { + delete(ps.mismatchStrikes, peer) + } + } + ps.lastMismatchPrune = now + } + + if _, ok := ps.mismatchStrikes[id]; !ok && len(ps.mismatchStrikes) >= maxStrikeEntries { + evictOldestStrike(ps.mismatchStrikes) + } + + record := ps.mismatchStrikes[id] + if record.count == 0 || now.Sub(record.windowStart) > whitelistMismatchWindow { + record = softStrikeRecord{windowStart: now} + } + record.count += 1 + ps.mismatchStrikes[id] = record + + return record.count +} + +func (ps *peerSet) clearMismatches(id string) { + ps.lock.Lock() + defer ps.lock.Unlock() + + delete(ps.mismatchStrikes, id) +} + +func (ps *peerSet) recordGhostState(id string, now time.Time) int { + ps.lock.Lock() + defer ps.lock.Unlock() + + if now.Sub(ps.lastGhostPrune) >= backoffPruneInterval { + for peer, record := range ps.ghostStrikes { + if now.Sub(record.windowStart) > prunedSidechainWindow { + delete(ps.ghostStrikes, peer) + } + } + ps.lastGhostPrune = now + } + + if _, ok := ps.ghostStrikes[id]; !ok && len(ps.ghostStrikes) >= maxStrikeEntries { + evictOldestStrike(ps.ghostStrikes) + } + + record := ps.ghostStrikes[id] + if record.count == 0 || now.Sub(record.windowStart) > prunedSidechainWindow { + record = softStrikeRecord{windowStart: now} } + record.count += 1 + ps.ghostStrikes[id] = record + + return record.count +} + +func (ps *peerSet) clearGhostStates(id string) { + ps.lock.Lock() + defer ps.lock.Unlock() + + delete(ps.ghostStrikes, id) +} + +func (ps *peerSet) recordJail(peer *peerConnection, until time.Time) { + ps.recordJailByID(peer.id, until) +} + +func (ps *peerSet) recordJailByID(id string, until time.Time) { + ps.lock.Lock() + defer ps.lock.Unlock() + + now := time.Now() + if now.Sub(ps.lastJailPrune) >= backoffPruneInterval { + for jid, expiry := range ps.jailed { + if !expiry.After(now) { + delete(ps.jailed, jid) + } + } + ps.lastJailPrune = now + } + if until.After(now) { + if _, ok := ps.jailed[id]; !ok && len(ps.jailed) >= maxStrikeEntries { + evictSoonestJail(ps.jailed) + } + if existing, ok := ps.jailed[id]; !ok || until.After(existing) { + ps.jailed[id] = until + } + if live := ps.peers[id]; live != nil { + live.extendBackoff(until) + } + } +} + +func (ps *peerSet) clearJailForTesting(id string) { + ps.lock.Lock() + defer ps.lock.Unlock() + + delete(ps.jailed, id) +} + +func (ps *peerSet) persistentBackoff(id string) time.Duration { + ps.lock.Lock() + defer ps.lock.Unlock() + + now := time.Now() + + var until time.Time + if exp, ok := ps.jailed[id]; ok { + if exp.After(now) { + until = exp + } else { + delete(ps.jailed, id) + } + } + if remaining := time.Until(until); remaining > 0 { + return remaining + } + return 0 } // SubscribeEvents subscribes to peer arrival and departure events. @@ -246,6 +538,8 @@ func (ps *peerSet) Register(p *peerConnection) error { return err } + ps.restoreReconnectPenalties(p) + ps.peers[p.id] = p ps.lock.Unlock() @@ -254,6 +548,17 @@ func (ps *peerSet) Register(p *peerConnection) error { return nil } +// Callers must hold ps.lock. +func (ps *peerSet) restoreReconnectPenalties(p *peerConnection) { + if until, ok := ps.jailed[p.id]; ok { + if time.Until(until) > 0 { + p.extendBackoff(until) + } else { + delete(ps.jailed, p.id) + } + } +} + // Unregister removes a remote peer from the active set, disabling any further // actions to/from that particular entity. func (ps *peerSet) Unregister(id string) error { diff --git a/eth/downloader/peer_response.go b/eth/downloader/peer_response.go new file mode 100644 index 0000000000..20d75bf087 --- /dev/null +++ b/eth/downloader/peer_response.go @@ -0,0 +1,272 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/downloader/whitelist" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/log" +) + +const ( + peerJailBackoff = 5 * time.Minute + peerSoftBackoff = 30 * time.Second + peerDropBackoff = 30 * time.Minute + + softFailureWindow = 10 * time.Minute + softFailureJailThreshold = 4 + + whitelistMismatchWindow = 30 * time.Minute + whitelistMismatchJailThreshold = 2 + whitelistMismatchDropThreshold = 4 + + prunedSidechainWindow = 30 * time.Minute + prunedSidechainDropThreshold = 2 + + backoffPruneInterval = time.Minute + + maxStrikeEntries = 4096 +) + +type peerFailureReason string + +const ( + peerFailureInvalidChain peerFailureReason = "invalid-chain" + peerFailurePrunedSidechain peerFailureReason = "pruned-sidechain" + peerFailureBadPeer peerFailureReason = "bad-peer" + peerFailureTimeout peerFailureReason = "timeout" + peerFailureStalling peerFailureReason = "stalling" + peerFailureUnsynced peerFailureReason = "unsynced" + peerFailureEmptyHeaderSet peerFailureReason = "empty-header-set" + peerFailurePeersUnavailable peerFailureReason = "peers-unavailable" + peerFailureTooOld peerFailureReason = "too-old" + peerFailureInvalidAncestor peerFailureReason = "invalid-ancestor" + peerFailureWhitelistMismatch peerFailureReason = "whitelist-mismatch" + peerFailureDisconnected peerFailureReason = "disconnected" + peerFailureNoRemote peerFailureReason = "whitelist-no-remote" +) + +type peerResponseAction uint8 + +const ( + peerResponseNone peerResponseAction = iota + peerResponseDrop + peerResponseBackoff + peerResponseMismatch + peerResponseGhostState +) + +type peerResponseDecision struct { + action peerResponseAction + backoff time.Duration + reason peerFailureReason +} + +func (d *Downloader) liveOrCaptured(captured *peerConnection, id string) *peerConnection { + if live := d.peers.Peer(id); live != nil { + return live + } + return captured +} + +func (d *Downloader) handleSyncFailure(peer *peerConnection, id string, err error) bool { + reason, ok := classifySyncFailure(err) + if !ok { + return false + } + peer = d.liveOrCaptured(peer, id) + if peer == nil { + log.Debug("Downloader peer response skipped for unknown peer", "peer", id, "reason", reason, "err", err) + return true + } + d.respondToPeer(peer, reason, err) + return true +} + +func classifySyncFailure(err error) (peerFailureReason, bool) { + switch { + case isPrunedSidechainMismatch(err): + return peerFailurePrunedSidechain, true + case isTransientFailure(err): + return peerFailureTimeout, true + case errors.Is(err, eth.ErrDisconnected): + return peerFailureDisconnected, true + case isWhitelistMismatch(err): + return peerFailureWhitelistMismatch, true + case errors.Is(err, whitelist.ErrNoRemote): + return peerFailureNoRemote, true + case errors.Is(err, ErrPeersUnavailable), errors.Is(err, errNoPeers): + return peerFailurePeersUnavailable, true + case errors.Is(err, errInvalidChain): + return peerFailureInvalidChain, true + case errors.Is(err, errBadPeer): + return peerFailureBadPeer, true + case errors.Is(err, errStallingPeer): + return peerFailureStalling, true + case errors.Is(err, errUnsyncedPeer), errors.Is(err, errNoAncestorFound): + return peerFailureUnsynced, true + case errors.Is(err, errEmptyHeaderSet): + return peerFailureEmptyHeaderSet, true + case errors.Is(err, errTooOld): + return peerFailureTooOld, true + case errors.Is(err, errInvalidAncestor): + return peerFailureInvalidAncestor, true + default: + return "", false + } +} + +func isWhitelistMismatch(err error) bool { + return errors.Is(err, whitelist.ErrMismatch) +} + +func isSyncCancellation(err error) bool { + return errors.Is(err, errCanceled) || errors.Is(err, errCancelContentProcessing) || errors.Is(err, errTerminated) || errors.Is(err, errCancelStateFetch) || errors.Is(err, snap.ErrCancelled) +} + +func isTransientFailure(err error) bool { + if err == nil { + return false + } + if errors.Is(err, errTimeout) || errors.Is(err, context.DeadlineExceeded) { + return true + } + if errors.Is(err, errInvalidChain) || errors.Is(err, errBadPeer) || errors.Is(err, errInvalidAncestor) { + return false + } + return strings.Contains(err.Error(), context.DeadlineExceeded.Error()) +} + +func (d *Downloader) respondToPeer(peer *peerConnection, reason peerFailureReason, err error) { + decision := peer.responseDecision(reason) + + switch decision.action { + case peerResponseBackoff: + d.backoffPeer(peer, decision, err) + case peerResponseMismatch: + d.escalateMismatch(peer, decision, err) + case peerResponseGhostState: + d.escalateGhostState(peer, decision, err) + case peerResponseDrop: + d.dropPeerForResponse(peer, decision.reason, err) + case peerResponseNone: + peer.log.Warn("Synchronisation stalled, no peer action taken", "reason", reason, "err", err) + } +} + +func (d *Downloader) benchPeer(peer *peerConnection, backoff time.Duration) { + peer.backoffFor(backoff) + d.peers.recordJail(peer, peer.backoffExpiry()) +} + +func (d *Downloader) jailPeer(peer *peerConnection, backoff time.Duration, reason peerFailureReason, err error) { + d.benchPeer(peer, backoff) + peerJailMeter.Mark(1) + peer.log.Warn("Downloader: locally jailing peer", "reason", reason, "err", err, "requested", common.PrettyDuration(backoff), "effective", common.PrettyDuration(peer.backoffRemaining())) +} + +func (d *Downloader) backoffPeer(peer *peerConnection, decision peerResponseDecision, err error) { + penalized, jailed := d.peers.backoffSoftFailure(peer) + if !penalized { + return + } + if jailed { + peerJailMeter.Mark(1) + peer.log.Warn("Downloader: escalating repeated soft failures to local jail", "reason", decision.reason, "err", err, "effective", common.PrettyDuration(peer.backoffRemaining())) + return + } + peerSoftBackoffMeter.Mark(1) + peer.log.Warn("Downloader: backing off peer", "reason", decision.reason, "err", err, "requested", common.PrettyDuration(decision.backoff), "effective", common.PrettyDuration(peer.backoffRemaining())) +} + +func (d *Downloader) escalateMismatch(peer *peerConnection, decision peerResponseDecision, err error) { + peerMismatchMeter.Mark(1) + strikes := d.peers.recordMismatch(peer.id, time.Now()) + switch { + case strikes >= whitelistMismatchDropThreshold: + d.peers.clearMismatches(peer.id) + peer.log.Warn("Downloader: dropping peer after persistent whitelist mismatch", "reason", decision.reason, "strikes", strikes, "err", err) + d.dropPeerForResponse(peer, decision.reason, err) + case strikes >= whitelistMismatchJailThreshold: + peer.log.Warn("Downloader: escalating repeated whitelist mismatch to local jail", "reason", decision.reason, "strikes", strikes, "err", err) + d.jailPeer(peer, peerJailBackoff, decision.reason, err) + default: + d.benchPeer(peer, decision.backoff) + peer.log.Warn("Downloader: backing off peer after whitelist mismatch", "reason", decision.reason, "err", err, "strikes", strikes, "requested", common.PrettyDuration(decision.backoff), "effective", common.PrettyDuration(peer.backoffRemaining())) + } +} + +func (d *Downloader) escalateGhostState(peer *peerConnection, decision peerResponseDecision, err error) { + peerGhostStateMeter.Mark(1) + strikes := d.peers.recordGhostState(peer.id, time.Now()) + if strikes >= prunedSidechainDropThreshold { + d.peers.clearGhostStates(peer.id) + peer.log.Warn("Downloader: dropping peer after repeated sidechain ghost-state attacks", "reason", decision.reason, "strikes", strikes, "err", err) + d.dropPeerForResponse(peer, decision.reason, err) + return + } + peer.log.Warn("Downloader: jailing peer for sidechain ghost-state attack", "reason", decision.reason, "strikes", strikes, "err", err) + d.jailPeer(peer, decision.backoff, decision.reason, err) +} + +func (p *peerConnection) responseDecision(reason peerFailureReason) peerResponseDecision { + decision := peerResponseDecision{ + reason: reason, + } + + switch reason { + case peerFailurePrunedSidechain: + decision.action = peerResponseGhostState + decision.backoff = peerJailBackoff + case peerFailureInvalidChain, peerFailureBadPeer, peerFailureInvalidAncestor: + decision.action = peerResponseDrop + case peerFailureWhitelistMismatch: + decision.action = peerResponseMismatch + decision.backoff = peerSoftBackoff + case peerFailureTimeout, peerFailureStalling, peerFailureUnsynced, peerFailureEmptyHeaderSet, peerFailureTooOld, peerFailureDisconnected: + decision.action = peerResponseBackoff + decision.backoff = peerSoftBackoff + default: + decision.action = peerResponseNone + } + return decision +} + +func (d *Downloader) dropPeerForResponse(peer *peerConnection, reason peerFailureReason, err error) { + d.benchPeer(peer, peerDropBackoff) + peer.log.Warn("Synchronisation failed, dropping peer", "reason", reason, "err", err, "mode", d.getMode()) + + if d.dropPeer == nil { + log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", peer.id) + return + } + peerDropResponseMeter.Mark(1) + d.dropPeer(peer.id) +} + +const sidechainGhostStateMsg = "sidechain ghost-state attack" + +func isPrunedSidechainMismatch(err error) bool { + return err != nil && strings.Contains(err.Error(), sidechainGhostStateMsg) +} diff --git a/eth/downloader/peer_response_test.go b/eth/downloader/peer_response_test.go new file mode 100644 index 0000000000..b962754c95 --- /dev/null +++ b/eth/downloader/peer_response_test.go @@ -0,0 +1,884 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "context" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/downloader/whitelist" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/log" +) + +func TestClassifySyncFailureReasons(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + reason peerFailureReason + ok bool + }{ + {name: "invalid chain", err: errInvalidChain, reason: peerFailureInvalidChain, ok: true}, + {name: "pruned sidechain", err: fmt.Errorf("%w: %w", errInvalidChain, errors.New(sidechainGhostStateMsg)), reason: peerFailurePrunedSidechain, ok: true}, + {name: "bad peer", err: errBadPeer, reason: peerFailureBadPeer, ok: true}, + {name: "timeout", err: errTimeout, reason: peerFailureTimeout, ok: true}, + {name: "stalling", err: errStallingPeer, reason: peerFailureStalling, ok: true}, + {name: "unsynced", err: errUnsyncedPeer, reason: peerFailureUnsynced, ok: true}, + {name: "empty header set", err: errEmptyHeaderSet, reason: peerFailureEmptyHeaderSet, ok: true}, + {name: "peers unavailable", err: ErrPeersUnavailable, reason: peerFailurePeersUnavailable, ok: true}, + {name: "too old", err: errTooOld, reason: peerFailureTooOld, ok: true}, + {name: "invalid ancestor", err: errInvalidAncestor, reason: peerFailureInvalidAncestor, ok: true}, + {name: "wrapped timeout overrides bad peer", err: fmt.Errorf("%w: header request failed: %w", errBadPeer, errTimeout), reason: peerFailureTimeout, ok: true}, + {name: "wrapped timeout overrides invalid chain", err: fmt.Errorf("%w: %w", errInvalidChain, errTimeout), reason: peerFailureTimeout, ok: true}, + {name: "wrapped context deadline overrides invalid chain", err: fmt.Errorf("%w: %w", errInvalidChain, context.DeadlineExceeded), reason: peerFailureTimeout, ok: true}, + {name: "flattened context deadline overrides invalid chain", err: fmt.Errorf("%v: %s", errInvalidChain, context.DeadlineExceeded), reason: peerFailureTimeout, ok: true}, + {name: "pruned sidechain wins over wrapped deadline", err: fmt.Errorf("%w: %s: %w", errInvalidChain, sidechainGhostStateMsg, context.DeadlineExceeded), reason: peerFailurePrunedSidechain, ok: true}, + {name: "whitelist mismatch wrapped in invalid chain", err: fmt.Errorf("%w: %w", errInvalidChain, whitelist.ErrMismatch), reason: peerFailureWhitelistMismatch, ok: true}, + {name: "bare whitelist mismatch", err: whitelist.ErrMismatch, reason: peerFailureWhitelistMismatch, ok: true}, + {name: "whitelist no remote classified", err: whitelist.ErrNoRemote, reason: peerFailureNoRemote, ok: true}, + {name: "whitelist mismatch wins over invalid chain", err: fmt.Errorf("retrieved hash chain is invalid: %w: %w", errInvalidChain, whitelist.ErrMismatch), reason: peerFailureWhitelistMismatch, ok: true}, + {name: "wrapped peers unavailable is not a chain fault", err: fmt.Errorf("%w: %w", errInvalidChain, ErrPeersUnavailable), reason: peerFailurePeersUnavailable, ok: true}, + {name: "wrapped no peers is not a chain fault", err: fmt.Errorf("%w: %w", errInvalidChain, errNoPeers), reason: peerFailurePeersUnavailable, ok: true}, + {name: "bare no peers", err: errNoPeers, reason: peerFailurePeersUnavailable, ok: true}, + {name: "wrapped disconnect backs off instead of dropping", err: fmt.Errorf("%w: header request failed: %w", errBadPeer, eth.ErrDisconnected), reason: peerFailureDisconnected, ok: true}, + {name: "bare disconnect backs off instead of dropping", err: eth.ErrDisconnected, reason: peerFailureDisconnected, ok: true}, + {name: "invalid chain wins over wrapped stalling", err: fmt.Errorf("%w: %w", errInvalidChain, errStallingPeer), reason: peerFailureInvalidChain, ok: true}, + {name: "invalid chain with deadline text is not downgraded", err: fmt.Errorf("%w: %v", errInvalidChain, errors.New("invalid merkle root: "+context.DeadlineExceeded.Error())), reason: peerFailureInvalidChain, ok: true}, + {name: "bad peer with deadline text is not downgraded", err: fmt.Errorf("%w: %v", errBadPeer, errors.New("served garbage: "+context.DeadlineExceeded.Error())), reason: peerFailureBadPeer, ok: true}, + {name: "invalid ancestor with deadline text is not downgraded", err: fmt.Errorf("%w: %v", errInvalidAncestor, errors.New(context.DeadlineExceeded.Error())), reason: peerFailureInvalidAncestor, ok: true}, + {name: "ghost-state text alone is pruned sidechain", err: errors.New(sidechainGhostStateMsg), reason: peerFailurePrunedSidechain, ok: true}, + {name: "no ancestor backs off", err: errNoAncestorFound, reason: peerFailureUnsynced, ok: true}, + {name: "invalid body handled at delivery is unclassified", err: errInvalidBody}, + {name: "unclassified generic", err: errors.New("some unrelated failure")}, + {name: "nil"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reason, ok := classifySyncFailure(tt.err) + if ok != tt.ok { + t.Fatalf("classification mismatch: have %v, want %v", ok, tt.ok) + } + if reason != tt.reason { + t.Fatalf("reason mismatch: have %q, want %q", reason, tt.reason) + } + }) + } +} + +func TestPeerResponseDecisionActions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + reason peerFailureReason + action peerResponseAction + backoff time.Duration + }{ + {name: "ghost-state escalation", reason: peerFailurePrunedSidechain, action: peerResponseGhostState, backoff: peerJailBackoff}, + {name: "drop invalid chain", reason: peerFailureInvalidChain, action: peerResponseDrop}, + {name: "drop bad peer", reason: peerFailureBadPeer, action: peerResponseDrop}, + {name: "drop invalid ancestor", reason: peerFailureInvalidAncestor, action: peerResponseDrop}, + {name: "mismatch escalation", reason: peerFailureWhitelistMismatch, action: peerResponseMismatch, backoff: peerSoftBackoff}, + {name: "backoff old peer", reason: peerFailureTooOld, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "backoff timeout", reason: peerFailureTimeout, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "backoff stalling", reason: peerFailureStalling, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "backoff unsynced", reason: peerFailureUnsynced, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "backoff empty header set", reason: peerFailureEmptyHeaderSet, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "backoff disconnected", reason: peerFailureDisconnected, action: peerResponseBackoff, backoff: peerSoftBackoff}, + {name: "no remote is no-op", reason: peerFailureNoRemote, action: peerResponseNone}, + {name: "ignore peers unavailable", reason: peerFailurePeersUnavailable, action: peerResponseNone}, + {name: "ignore unknown reason", reason: peerFailureReason("unclassified"), action: peerResponseNone}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + decision := peer.responseDecision(tt.reason) + if decision.action != tt.action { + t.Fatalf("action mismatch: have %v, want %v", decision.action, tt.action) + } + if decision.backoff != tt.backoff { + t.Fatalf("backoff mismatch: have %v, want %v", decision.backoff, tt.backoff) + } + if decision.reason != tt.reason { + t.Fatalf("reason mismatch: have %q, want %q", decision.reason, tt.reason) + } + }) + } +} + +func TestHandleSyncFailureSkipsUnknownPeer(t *testing.T) { + t.Parallel() + + downloader := &Downloader{peers: newPeerSet()} + if !downloader.handleSyncFailure(nil, "missing", errTimeout) { + t.Fatal("a classified failure for an unknown peer should still be handled") + } +} + +func TestIsSyncCancellation(t *testing.T) { + t.Parallel() + + for _, base := range []error{errCanceled, errCancelContentProcessing, errTerminated, errCancelStateFetch, snap.ErrCancelled} { + if !isSyncCancellation(base) { + t.Fatalf("bare %v should be a cancellation", base) + } + if wrapped := fmt.Errorf("%w: %w", errInvalidChain, base); !isSyncCancellation(wrapped) { + t.Fatalf("wrapped %v should be a cancellation", base) + } + } + if isSyncCancellation(errInvalidChain) { + t.Fatal("an invalid chain is not a cancellation") + } + if isSyncCancellation(errTimeout) { + t.Fatal("a timeout is not a cancellation") + } + if isSyncCancellation(nil) { + t.Fatal("a nil error is not a cancellation") + } +} + +func TestDropPeerForResponseWithoutDropper(t *testing.T) { + t.Parallel() + + downloader := &Downloader{peers: newPeerSet()} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + downloader.dropPeerForResponse(peer, peerFailureTooOld, errTooOld) +} + +func TestJailSurvivesReconnect(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + + peer := newPeerConnection("jailed-peer", eth.ETH69, nil, log.New()) + if err := ps.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + peer.backoffFor(peerJailBackoff) + ps.recordJail(peer, peer.backoffExpiry()) + + if err := ps.Unregister(peer.id); err != nil { + t.Fatalf("failed to unregister peer: %v", err) + } + + reconnected := newPeerConnection("jailed-peer", eth.ETH69, nil, log.New()) + if err := ps.Register(reconnected); err != nil { + t.Fatalf("failed to re-register peer: %v", err) + } + if !reconnected.backedOff() { + t.Fatal("reconnecting peer should remain jailed for the remaining duration") + } +} + +func TestRecordJailPrunesExpiredEntries(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + expired := newPeerConnection("expired", eth.ETH69, nil, log.New()) + ps.recordJail(expired, time.Now().Add(-time.Second)) + if _, ok := ps.jailed["expired"]; ok { + t.Fatal("already-expired jail entry should not be stored") + } + + active := newPeerConnection("active", eth.ETH69, nil, log.New()) + stale := newPeerConnection("stale", eth.ETH69, nil, log.New()) + ps.recordJail(active, time.Now().Add(peerJailBackoff)) + ps.recordJail(stale, time.Now().Add(-time.Second)) + if _, ok := ps.jailed["stale"]; ok { + t.Fatal("expired entry should be pruned on access") + } + if _, ok := ps.jailed["active"]; !ok { + t.Fatal("active jail entry should be retained") + } +} + +func TestRecordJailPropagatesToLivePeer(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + live := newPeerConnection("dup", eth.ETH69, nil, log.New()) + if err := ps.Register(live); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + if live.backedOff() { + t.Fatal("freshly registered peer should not be backed off") + } + + until := time.Now().Add(peerJailBackoff) + ps.recordJail(live, until) + + if !live.backedOff() { + t.Fatal("recordJail must push the jail backoff onto the already-registered peer") + } + if got := live.backoffExpiry(); got.Before(until) { + t.Fatalf("live peer backoff not lifted to jail expiry: have %v, want >= %v", got, until) + } +} + +func TestPersistentBackoffPrunesExpiredEntries(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + ps.jailed["expired-jail"] = now.Add(-time.Second) + if remaining := ps.persistentBackoff("expired-jail"); remaining != 0 { + t.Fatalf("expired jail should report no backoff: have %v, want 0", remaining) + } + if _, ok := ps.jailed["expired-jail"]; ok { + t.Fatal("expired jail entry should be pruned on read") + } + + ps.jailed["jailed"] = now.Add(peerJailBackoff) + if remaining := ps.persistentBackoff("jailed"); remaining <= 0 { + t.Fatalf("active jail should report remaining backoff: have %v, want > 0", remaining) + } + + if remaining := ps.persistentBackoff("unknown"); remaining != 0 { + t.Fatalf("unknown peer should report no backoff: have %v, want 0", remaining) + } +} + +func TestLiveOrCapturedPrefersRegisteredPeer(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet()} + captured := newPeerConnection("peer", eth.ETH69, nil, log.New()) + + if d.liveOrCaptured(captured, captured.id) != captured { + t.Fatal("with no registered peer the captured handle must be used") + } + + live := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(live); err != nil { + t.Fatalf("failed to register live peer: %v", err) + } + if d.liveOrCaptured(captured, captured.id) != live { + t.Fatal("a registered peer must supersede the captured handle") + } + + if d.liveOrCaptured(nil, "absent") != nil { + t.Fatal("an absent peer with no captured handle must be nil") + } +} + +func TestPeerBackoffStateTransitions(t *testing.T) { + t.Parallel() + + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + peer.backoffFor(0) + if !peer.backoffExpiry().IsZero() { + t.Fatalf("zero-duration backoff should leave expiry unset: have %v", peer.backoffExpiry()) + } + if peer.backedOff() { + t.Fatal("peer should not be backed off for zero duration") + } + + peer.backoffFor(30 * time.Second) + if !peer.backedOff() { + t.Fatal("peer should be backed off for positive duration") + } + + peer.lock.Lock() + peer.backoff = time.Now().Add(-time.Second) + peer.lock.Unlock() + + if remaining := peer.backoffRemaining(); remaining != 0 { + t.Fatalf("expired backoff mismatch: have %v, want 0", remaining) + } + + peer.lock.RLock() + expired := peer.backoff.IsZero() + peer.lock.RUnlock() + if !expired { + t.Fatal("expired backoff was not cleared") + } +} + +func TestPrunedSidechainJailsPeer(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet()} + peer := newPeerConnection("ghost", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + ghostErr := fmt.Errorf("%w: %s", errInvalidChain, sidechainGhostStateMsg) + d.respondToPeer(peer, peerFailurePrunedSidechain, ghostErr) + if !peer.backedOff() { + t.Fatal("a pruned-sidechain attack must bench the peer") + } + if _, ok := d.peers.jailed[peer.id]; !ok { + t.Fatal("a pruned-sidechain attack must jail the peer") + } +} + +func TestPrunedSidechainEscalatesToDrop(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("ghost", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + ghostErr := fmt.Errorf("%w: %s", errInvalidChain, sidechainGhostStateMsg) + + for i := 1; i < prunedSidechainDropThreshold; i++ { + d.respondToPeer(peer, peerFailurePrunedSidechain, ghostErr) + if remaining := peer.backoffRemaining(); remaining <= peerSoftBackoff { + t.Fatalf("ghost-state strike %d should jail the peer: have %v, want > %v", i, remaining, peerSoftBackoff) + } + if until, ok := d.peers.jailed[peer.id]; !ok || time.Until(until) <= peerSoftBackoff { + t.Fatalf("ghost-state strike %d should record a long jail", i) + } + select { + case id := <-dropped: + t.Fatalf("ghost-state strike %d must jail, not drop: %q", i, id) + default: + } + } + + d.respondToPeer(peer, peerFailurePrunedSidechain, ghostErr) + select { + case id := <-dropped: + if id != peer.id { + t.Fatalf("dropped wrong peer: have %q, want %q", id, peer.id) + } + default: + t.Fatal("a peer that keeps launching ghost-state attacks should eventually be dropped") + } + if _, ok := d.peers.ghostStrikes[peer.id]; ok { + t.Fatal("dropping a peer for repeated ghost-state should clear its tally") + } +} + +func TestRecordSoftFailureDecays(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordSoftFailure("peer", now); got != 1 { + t.Fatalf("first strike count mismatch: have %d, want 1", got) + } + if got := ps.recordSoftFailure("peer", now.Add(time.Minute)); got != 2 { + t.Fatalf("second strike count mismatch: have %d, want 2", got) + } + if got := ps.recordSoftFailure("peer", now.Add(softFailureWindow+time.Second)); got != 1 { + t.Fatalf("a strike past the window should reset the tally: have %d, want 1", got) + } +} + +func TestRecordSoftFailureIsolatesPeers(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + ps.recordSoftFailure("a", now) + ps.recordSoftFailure("a", now) + + if got := ps.recordSoftFailure("b", now); got != 1 { + t.Fatalf("distinct peers must not share a tally: have %d, want 1", got) + } + + ps.clearSoftFailures("a") + if got := ps.recordSoftFailure("a", now); got != 1 { + t.Fatalf("clearing must reset the tally: have %d, want 1", got) + } +} + +func TestReportBadBlockWithoutReporter(t *testing.T) { + t.Parallel() + + d := &Downloader{} + block := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(1)}) + + d.reportBadBlock(block) +} + +func TestReportBadBlockBoundsError(t *testing.T) { + t.Parallel() + + called := false + d := &Downloader{ + badBlock: func(*types.Header, *types.Header) { called = true }, + skeleton: &skeleton{db: rawdb.NewMemoryDatabase()}, + } + block := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(1)}) + + d.reportBadBlock(block) + if called { + t.Fatal("badBlock should not be invoked when beacon bounds are unavailable") + } +} + +func TestHandleSyncFailureUnclassified(t *testing.T) { + t.Parallel() + + downloader := &Downloader{peers: newPeerSet()} + if downloader.handleSyncFailure(nil, "peer", errors.New("unclassified")) { + t.Fatal("unclassified error should not trigger a peer response") + } +} + +func TestHandleSyncFailureClassified(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + if !d.handleSyncFailure(peer, peer.id, errInvalidChain) { + t.Fatal("a classified sync failure should be handled") + } + select { + case id := <-dropped: + if id != peer.id { + t.Fatalf("dropped wrong peer: have %q, want %q", id, peer.id) + } + default: + t.Fatal("a handled invalid chain should drop the peer") + } +} + +func TestHandleSyncFailureTimeoutBacksOff(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + if !d.handleSyncFailure(peer, peer.id, errTimeout) { + t.Fatal("a classified timeout should be handled") + } + select { + case id := <-dropped: + t.Fatalf("a transient timeout must not drop the peer: %q", id) + default: + } + if !peer.backedOff() { + t.Fatal("a transient timeout should back the peer off") + } + if _, ok := d.peers.jailed[peer.id]; !ok { + t.Fatal("a soft backoff should persist across reconnect") + } +} + +func TestBackoffEscalatesToJailAfterRepeatStrikes(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + for i := 0; i < softFailureJailThreshold-1; i++ { + d.respondToPeer(peer, peerFailureTimeout, errTimeout) + if remaining := peer.backoffRemaining(); remaining > peerSoftBackoff { + t.Fatalf("soft strike %d should stay within soft backoff: have %v, want <= %v", i+1, remaining, peerSoftBackoff) + } + clearPeerBackoff(peer) + } + + d.respondToPeer(peer, peerFailureTimeout, errTimeout) + if remaining := peer.backoffRemaining(); remaining <= peerSoftBackoff { + t.Fatalf("the escalating strike should jail the peer: have %v, want > %v", remaining, peerSoftBackoff) + } + if until, ok := d.peers.jailed[peer.id]; !ok || time.Until(until) <= peerSoftBackoff { + t.Fatal("the escalating strike should record a long jail") + } + if _, ok := d.peers.softStrikes[peer.id]; ok { + t.Fatal("escalating soft failures to a jail should clear the soft tally") + } + select { + case id := <-dropped: + t.Fatalf("escalation must jail, not drop: %q", id) + default: + } +} + +func clearPeerBackoff(peer *peerConnection) { + peer.lock.Lock() + peer.backoff = time.Time{} + peer.lock.Unlock() +} + +func TestBackoffForClaim(t *testing.T) { + t.Parallel() + + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if !peer.backoffForClaim(peerSoftBackoff) { + t.Fatal("the first claim on an un-benched peer should succeed") + } + if !peer.backedOff() { + t.Fatal("a claimed peer should be benched") + } + if peer.backoffForClaim(peerSoftBackoff) { + t.Fatal("a claim on an already-benched peer should report it was already benched") + } +} + +func TestBackoffSoftFailureDedupsWhileBenched(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := ps.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + if penalized, _ := ps.backoffSoftFailure(peer); !penalized { + t.Fatal("the first soft failure should penalize the peer") + } + if penalized, _ := ps.backoffSoftFailure(peer); penalized { + t.Fatal("a peer already benched must not be penalized again for a concurrent failure") + } + if got := ps.softStrikes[peer.id].count; got != 1 { + t.Fatalf("a benched peer must not accrue a second strike: have %d, want 1", got) + } +} + +func TestRecordMismatchDecays(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordMismatch("peer", now); got != 1 { + t.Fatalf("first mismatch count mismatch: have %d, want 1", got) + } + if got := ps.recordMismatch("peer", now.Add(time.Minute)); got != 2 { + t.Fatalf("second mismatch count mismatch: have %d, want 2", got) + } + if got := ps.recordMismatch("peer", now.Add(whitelistMismatchWindow+time.Second)); got != 1 { + t.Fatalf("a mismatch past the window should reset the tally: have %d, want 1", got) + } +} + +func TestRecordMismatchIsolatesPeers(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + ps.recordMismatch("a", now) + ps.recordMismatch("a", now) + + if got := ps.recordMismatch("b", now); got != 1 { + t.Fatalf("distinct peers must not share a mismatch tally: have %d, want 1", got) + } + + ps.clearMismatches("a") + if got := ps.recordMismatch("a", now); got != 1 { + t.Fatalf("clearing must reset the mismatch tally: have %d, want 1", got) + } +} + +func TestRecordGhostStateDecays(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordGhostState("peer", now); got != 1 { + t.Fatalf("first ghost-state count mismatch: have %d, want 1", got) + } + if got := ps.recordGhostState("peer", now.Add(time.Minute)); got != 2 { + t.Fatalf("second ghost-state count mismatch: have %d, want 2", got) + } + if got := ps.recordGhostState("peer", now.Add(prunedSidechainWindow+time.Second)); got != 1 { + t.Fatalf("a ghost-state past the window should reset the tally: have %d, want 1", got) + } +} + +func TestRecordGhostStateIsolatesPeers(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + ps.recordGhostState("a", now) + ps.recordGhostState("a", now) + + if got := ps.recordGhostState("b", now); got != 1 { + t.Fatalf("distinct peers must not share a ghost-state tally: have %d, want 1", got) + } + + ps.clearGhostStates("a") + if got := ps.recordGhostState("a", now); got != 1 { + t.Fatalf("clearing must reset the ghost-state tally: have %d, want 1", got) + } +} + +func TestRecordSoftFailureWindowBoundary(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordSoftFailure("p", now); got != 1 { + t.Fatalf("first strike count mismatch: have %d, want 1", got) + } + if got := ps.recordSoftFailure("p", now.Add(softFailureWindow)); got != 2 { + t.Fatalf("a strike exactly at the window edge must not reset the tally: have %d, want 2", got) + } +} + +func TestRecordMismatchWindowBoundary(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordMismatch("p", now); got != 1 { + t.Fatalf("first mismatch count mismatch: have %d, want 1", got) + } + if got := ps.recordMismatch("p", now.Add(whitelistMismatchWindow)); got != 2 { + t.Fatalf("a mismatch exactly at the window edge must not reset the tally: have %d, want 2", got) + } +} + +func TestRecordGhostStateWindowBoundary(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + + if got := ps.recordGhostState("p", now); got != 1 { + t.Fatalf("first ghost-state count mismatch: have %d, want 1", got) + } + if got := ps.recordGhostState("p", now.Add(prunedSidechainWindow)); got != 2 { + t.Fatalf("a ghost-state exactly at the window edge must not reset the tally: have %d, want 2", got) + } +} + +func TestRecordJailPrunesExpiredOnInterval(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + ps.jailed["expired"] = time.Now().Add(-time.Hour) + ps.lastJailPrune = time.Now().Add(-2 * backoffPruneInterval) + + peer := newPeerConnection("fresh", eth.ETH69, nil, log.New()) + ps.recordJail(peer, time.Now().Add(peerJailBackoff)) + + if _, ok := ps.jailed["expired"]; ok { + t.Fatal("an expired jail entry should be pruned once the prune interval elapses") + } + if _, ok := ps.jailed["fresh"]; !ok { + t.Fatal("the active jail entry should be recorded") + } +} + +func TestRecordJailKeepsLongerExpiry(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := ps.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + long := time.Now().Add(peerJailBackoff) + ps.recordJail(peer, long) + ps.recordJail(peer, time.Now().Add(peerSoftBackoff)) + + if got := ps.jailed[peer.id]; got.Before(long) { + t.Fatalf("a shorter jail must not overwrite a longer one: have %v, want >= %v", got, long) + } +} + +func TestStrikeMapsAreBounded(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + now := time.Now() + for i := 0; i < maxStrikeEntries+10; i++ { + id := fmt.Sprintf("peer-%d", i) + ps.recordSoftFailure(id, now) + ps.recordMismatch(id, now) + ps.recordGhostState(id, now) + ps.recordJail(newPeerConnection(fmt.Sprintf("jail-%d", i), eth.ETH69, nil, log.New()), now.Add(time.Hour)) + } + + if got := len(ps.softStrikes); got > maxStrikeEntries { + t.Fatalf("softStrikes exceeded cap: have %d, want <= %d", got, maxStrikeEntries) + } + if got := len(ps.mismatchStrikes); got > maxStrikeEntries { + t.Fatalf("mismatchStrikes exceeded cap: have %d, want <= %d", got, maxStrikeEntries) + } + if got := len(ps.ghostStrikes); got > maxStrikeEntries { + t.Fatalf("ghostStrikes exceeded cap: have %d, want <= %d", got, maxStrikeEntries) + } + if got := len(ps.jailed); got > maxStrikeEntries { + t.Fatalf("jailed exceeded cap: have %d, want <= %d", got, maxStrikeEntries) + } +} + +func TestStrikeEvictionRemovesOldest(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + base := time.Now() + for i := 0; i < maxStrikeEntries; i++ { + ps.recordSoftFailure(fmt.Sprintf("peer-%d", i), base.Add(time.Duration(i)*time.Millisecond)) + } + ps.recordSoftFailure("newcomer", base.Add(time.Duration(maxStrikeEntries)*time.Millisecond)) + + if _, ok := ps.softStrikes["peer-0"]; ok { + t.Fatal("eviction should remove the oldest strike record") + } + if _, ok := ps.softStrikes["newcomer"]; !ok { + t.Fatal("the newest strike record should be retained") + } + if _, ok := ps.softStrikes["peer-1"]; !ok { + t.Fatal("a newer strike record should not be evicted") + } +} + +func TestJailEvictionRemovesSoonestExpiry(t *testing.T) { + t.Parallel() + + ps := newPeerSet() + base := time.Now().Add(time.Hour) + for i := 0; i < maxStrikeEntries; i++ { + peer := newPeerConnection(fmt.Sprintf("jail-%d", i), eth.ETH69, nil, log.New()) + ps.recordJail(peer, base.Add(time.Duration(i)*time.Millisecond)) + } + newcomer := newPeerConnection("jail-new", eth.ETH69, nil, log.New()) + ps.recordJail(newcomer, base.Add(time.Duration(maxStrikeEntries)*time.Millisecond)) + + if _, ok := ps.jailed["jail-0"]; ok { + t.Fatal("jail eviction should remove the soonest-expiring entry") + } + if _, ok := ps.jailed["jail-new"]; !ok { + t.Fatal("the new jail entry should be retained") + } +} + +func TestDropForResponseBenchesPeer(t *testing.T) { + t.Parallel() + + d := &Downloader{peers: newPeerSet(), dropPeer: func(string) {}} + peer := newPeerConnection("bad", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + d.dropPeerForResponse(peer, peerFailureInvalidChain, errInvalidChain) + + until, ok := d.peers.jailed[peer.id] + if !ok { + t.Fatal("a dropped peer should be benched so a same-id reconnect stays jailed") + } + if remaining := time.Until(until); remaining <= peerJailBackoff { + t.Fatalf("a drop should bench longer than a plain jail: have %v, want > %v", remaining, peerJailBackoff) + } +} + +func TestWhitelistMismatchDoesNotDropOnFirstStrike(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + mismatchErr := fmt.Errorf("%w: %w", errInvalidChain, whitelist.ErrMismatch) + if !d.handleSyncFailure(peer, peer.id, mismatchErr) { + t.Fatal("a whitelist mismatch should be handled") + } + select { + case id := <-dropped: + t.Fatalf("a first whitelist mismatch must not drop the peer: %q", id) + default: + } + if remaining := peer.backoffRemaining(); remaining <= 0 || remaining > peerSoftBackoff { + t.Fatalf("a first whitelist mismatch should soft-backoff the peer: have %v, want (0, %v]", remaining, peerSoftBackoff) + } +} + +func TestWhitelistMismatchEscalatesBackoffJailDrop(t *testing.T) { + t.Parallel() + + dropped := make(chan string, 1) + d := &Downloader{peers: newPeerSet(), dropPeer: func(id string) { dropped <- id }} + peer := newPeerConnection("peer", eth.ETH69, nil, log.New()) + if err := d.peers.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + mismatchErr := fmt.Errorf("%w: %w", errInvalidChain, whitelist.ErrMismatch) + + for i := 1; i < whitelistMismatchJailThreshold; i++ { + d.respondToPeer(peer, peerFailureWhitelistMismatch, mismatchErr) + if remaining := peer.backoffRemaining(); remaining > peerSoftBackoff { + t.Fatalf("mismatch strike %d should stay within soft backoff: have %v, want <= %v", i, remaining, peerSoftBackoff) + } + select { + case id := <-dropped: + t.Fatalf("mismatch strike %d must not drop the peer: %q", i, id) + default: + } + } + + for i := whitelistMismatchJailThreshold; i < whitelistMismatchDropThreshold; i++ { + d.respondToPeer(peer, peerFailureWhitelistMismatch, mismatchErr) + if remaining := peer.backoffRemaining(); remaining <= peerSoftBackoff { + t.Fatalf("mismatch strike %d should jail the peer: have %v, want > %v", i, remaining, peerSoftBackoff) + } + if until, ok := d.peers.jailed[peer.id]; !ok || time.Until(until) <= peerSoftBackoff { + t.Fatalf("mismatch strike %d should record a long jail", i) + } + select { + case id := <-dropped: + t.Fatalf("mismatch strike %d must jail, not drop: %q", i, id) + default: + } + } + + d.respondToPeer(peer, peerFailureWhitelistMismatch, mismatchErr) + select { + case id := <-dropped: + if id != peer.id { + t.Fatalf("dropped wrong peer: have %q, want %q", id, peer.id) + } + default: + t.Fatal("a persistent whitelist mismatch should eventually drop the peer") + } + if _, ok := d.peers.mismatchStrikes[peer.id]; ok { + t.Fatal("dropping a peer for persistent mismatch should clear its tally") + } +} diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index e68d3d5001..eba61595d8 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "sort" "time" @@ -216,6 +215,7 @@ type skeleton struct { scratchHead uint64 // Block number of the first item in the scratch space requests map[uint64]*headerRequest // Header requests currently running + reqSeq uint64 // Monotonic source of unique request IDs headEvents chan *headUpdate // Notification channel for new heads terminate chan chan error // Termination channel to abort sync @@ -348,6 +348,40 @@ func (s *skeleton) Sync(head *types.Header, final *types.Header, force bool) err } } +func earlierBackoff(current, candidate time.Time) time.Time { + if candidate.IsZero() { + return current + } + if current.IsZero() || candidate.Before(current) { + return candidate + } + + return current +} + +func armBackoffTimer(prev *time.Timer, wake time.Time) (*time.Timer, <-chan time.Time) { + if prev != nil && !prev.Stop() { + select { + case <-prev.C: + default: + } + } + if wake.IsZero() { + return prev, nil + } + d := time.Until(wake) + if d <= 0 { + d = time.Nanosecond + } + if prev == nil { + prev = time.NewTimer(d) + return prev, prev.C + } + prev.Reset(d) + + return prev, prev.C +} + // sync is the internal version of Sync that executes a single sync cycle, either // until some termination condition is reached, or until the current cycle merges // with a previously aborted run. @@ -446,26 +480,30 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { s.syncStarting() } + var ( + backoffTimer *time.Timer + backoffCh <-chan time.Time + ) + defer func() { + if backoffTimer != nil { + backoffTimer.Stop() + } + }() + for { // Something happened, try to assign new tasks to any idle peers + var wake time.Time if !linked { - s.assignTasks(responses, requestFails, cancel) + wake = s.assignTasks(responses, requestFails, cancel) } + backoffTimer, backoffCh = armBackoffTimer(backoffTimer, wake) + // Wait for something to happen select { - case event := <-peering: - // A peer joined or left, the tasks queue and allocations need to be - // checked for potential assignment or reassignment - peerid := event.peer.id - if event.join { - log.Debug("Joining skeleton peer", "id", peerid) + case <-backoffCh: - s.idles[peerid] = event.peer - } else { - log.Debug("Leaving skeleton peer", "id", peerid) - s.revertRequests(peerid) - delete(s.idles, peerid) - } + case event := <-peering: + s.handlePeeringEvent(event) case errc := <-s.terminate: errc <- nil @@ -499,7 +537,7 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { } case req := <-requestFails: - s.revertRequest(req) + s.handleRequestFail(req) case res := <-responses: // Process the batch of headers. If though processing we managed to @@ -696,31 +734,41 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error return nil } -// assignTasks attempts to match idle peers to pending header retrievals. -func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRequest, cancel chan struct{}) { - // Sort the peers by download capacity to use faster ones if many available +func (s *skeleton) idleSkeletonPeers() (*peerCapacitySort, time.Time) { idlers := &peerCapacitySort{ peers: make([]*peerConnection, 0, len(s.idles)), caps: make([]int, 0, len(s.idles)), } targetTTL := s.peers.rates.TargetTimeout() + var nextBackoff time.Time for _, peer := range s.idles { + if peer.backedOff() { + nextBackoff = earlierBackoff(nextBackoff, peer.backoffExpiry()) + continue + } idlers.peers = append(idlers.peers, peer) idlers.caps = append(idlers.caps, s.peers.rates.Capacity(peer.id, eth.BlockHeadersMsg, targetTTL)) } + sort.Sort(idlers) + + return idlers, nextBackoff +} + +func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRequest, cancel chan struct{}) time.Time { + // Sort the peers by download capacity to use faster ones if many available + idlers, nextBackoff := s.idleSkeletonPeers() + if len(idlers.peers) == 0 { - return + return nextBackoff } - sort.Sort(idlers) - // Find header regions not yet downloading and fill them for task, owner := range s.scratchOwners { // If we're out of idle peers, stop assigning tasks if len(idlers.peers) == 0 { - return + return nextBackoff } // Skip any tasks already filling if owner != "" { @@ -728,7 +776,7 @@ func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRe } // If we've reached the genesis, stop assigning tasks if uint64(task*requestHeaders) >= s.scratchHead { - return + return nextBackoff } // Found a task and have peers available, assign it idle := idlers.peers[0] @@ -737,20 +785,8 @@ func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRe idlers.caps = idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id - var reqid uint64 + reqid := s.newRequestID() - for { - reqid = uint64(rand.Int63()) - if reqid == 0 { - continue - } - - if _, ok := s.requests[reqid]; ok { - continue - } - - break - } // Generate the network query and send it to the peer req := &headerRequest{ peer: idle.id, @@ -770,6 +806,12 @@ func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRe // Inject the request into the task to block further assignments s.scratchOwners[task] = idle.id } + return nextBackoff +} + +func (s *skeleton) newRequestID() uint64 { + s.reqSeq += 1 + return s.reqSeq } // executeTask executes a single fetch request, blocking until either a result @@ -811,23 +853,11 @@ func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { peer.log.Debug("Header request cancelled") s.scheduleRevertRequest(req) - case <-timeoutTimer.C: - // Header retrieval timed out, update the metrics - peer.log.Warn("Skeleton: header request timed out, dropping peer", "elapsed", ttl) - headerTimeoutMeter.Mark(1) - s.peers.rates.Update(peer.id, eth.BlockHeadersMsg, 0, 0) - s.scheduleRevertRequest(req) + case <-req.stale: + return - // At this point we either need to drop the offending peer, or we need a - // mechanism to allow waiting for the response and not cancel it. For now - // lets go with dropping since the header sizes are deterministic and the - // beacon sync runs exclusive (downloader is idle) so there should be no - // other load to make timeouts probable. If we notice that timeouts happen - // more often than we'd like, we can introduce a tracker for the requests - // gone stale and monitor them. However, in that case too, we need a way - // to protect against malicious peers never responding, so it would need - // a second, hard-timeout mechanism. - s.drop(peer.id) + case <-timeoutTimer.C: + s.handleHeaderTimeout(peer, req, ttl) case res := <-resCh: // Headers successfully retrieved, update the metrics @@ -949,6 +979,59 @@ func (s *skeleton) revertRequest(req *headerRequest) { s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } +func (s *skeleton) handlePeeringEvent(event *peeringEvent) { + peerid := event.peer.id + if event.join { + log.Debug("Joining skeleton peer", "id", peerid) + s.idles[peerid] = event.peer + } else { + log.Debug("Leaving skeleton peer", "id", peerid) + s.revertRequests(peerid) + delete(s.idles, peerid) + } +} + +func (s *skeleton) handleRequestFail(req *headerRequest) { + s.revertRequest(req) + if peer := s.peers.Peer(req.peer); peer != nil && peer.backedOff() { + s.idles[req.peer] = peer + } +} + +func (s *skeleton) dropForInvalidHeaders(peerid string) { + if peer := s.peers.Peer(peerid); peer != nil { + peer.backoffFor(peerDropBackoff) + s.peers.recordJail(peer, peer.backoffExpiry()) + } else { + s.peers.recordJailByID(peerid, time.Now().Add(peerDropBackoff)) + } + if s.drop == nil { + return + } + peerDropResponseMeter.Mark(1) + s.drop(peerid) +} + +func (s *skeleton) handleHeaderTimeout(peer *peerConnection, req *headerRequest, ttl time.Duration) { + select { + case <-req.stale: + return + default: + } + headerTimeoutMeter.Mark(1) + s.peers.rates.Update(peer.id, eth.BlockHeadersMsg, 0, 0) + if penalized, jailed := s.peers.backoffSoftFailure(peer); penalized { + if jailed { + peerJailMeter.Mark(1) + peer.log.Warn("Skeleton: escalating repeated header timeouts to local jail", "elapsed", ttl, "effective", common.PrettyDuration(peer.backoffRemaining())) + } else { + peerSoftBackoffMeter.Mark(1) + peer.log.Warn("Skeleton: header request timed out, backing off peer", "elapsed", ttl, "effective", common.PrettyDuration(peer.backoffRemaining())) + } + } + s.scheduleRevertRequest(req) +} + func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged bool) { res.peer.log.Trace("Processing header response", "head", res.headers[0].Number, "hash", res.headers[0].Hash(), "count", len(res.headers)) @@ -997,7 +1080,7 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo for i := 0; i < requestHeaders; i++ { s.scratchSpace[i] = nil } - s.drop(s.scratchOwners[0]) + s.dropForInvalidHeaders(s.scratchOwners[0]) s.scratchOwners[0] = "" break diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 418d17f086..2d1da15cc5 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -85,6 +85,7 @@ type skeletonTestPeer struct { served atomic.Uint64 // Number of headers served by this peer dropped atomic.Uint64 // Flag whether the peer was dropped (stop responding) + hang atomic.Bool } // newSkeletonTestPeer creates a new mock peer to test the skeleton sync with. @@ -117,6 +118,9 @@ func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, ski if p.dropped.Load() != 0 { return nil, errors.New("peer already dropped") } + if p.hang.Load() { + return ð.Request{Peer: p.id}, nil + } // Skeleton sync retrieves batches of headers going backward without gaps. // This ensures we can follow a clean parent progression without any reorg // hiccups. There is no need for any other type of header retrieval, so do @@ -538,6 +542,344 @@ func TestSkeletonSyncExtend(t *testing.T) { } } +func TestEarlierBackoff(t *testing.T) { + t.Parallel() + + early := time.Unix(1000, 0) + late := time.Unix(2000, 0) + + tests := []struct { + name string + current time.Time + candidate time.Time + want time.Time + }{ + {name: "zero candidate keeps current", current: late, candidate: time.Time{}, want: late}, + {name: "zero candidate with zero current stays zero", current: time.Time{}, candidate: time.Time{}, want: time.Time{}}, + {name: "zero current takes candidate", current: time.Time{}, candidate: late, want: late}, + {name: "earlier candidate wins", current: late, candidate: early, want: early}, + {name: "later candidate keeps current", current: early, candidate: late, want: early}, + {name: "equal keeps current", current: early, candidate: early, want: early}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := earlierBackoff(tt.current, tt.candidate); !got.Equal(tt.want) { + t.Fatalf("earlierBackoff(%v, %v) = %v, want %v", tt.current, tt.candidate, got, tt.want) + } + }) + } +} + +func TestSkeletonAssignTasksReportsBackoff(t *testing.T) { + chain := []*types.Header{{Number: big.NewInt(0)}} + + peerset := newPeerSet() + soft := newPeerConnection("soft", eth.ETH69, newSkeletonTestPeer("soft", chain), log.New("id", "soft")) + strong := newPeerConnection("strong", eth.ETH69, newSkeletonTestPeer("strong", chain), log.New("id", "strong")) + if err := peerset.Register(soft); err != nil { + t.Fatalf("failed to register soft peer: %v", err) + } + if err := peerset.Register(strong); err != nil { + t.Fatalf("failed to register strong peer: %v", err) + } + + strong.backoffFor(2 * time.Minute) + soft.backoffFor(30 * time.Second) + + skeleton := &skeleton{ + peers: peerset, + idles: map[string]*peerConnection{soft.id: soft, strong.id: strong}, + scratchSpace: make([]*types.Header, scratchHeaders), + scratchOwners: make([]string, scratchHeaders/requestHeaders), + requests: make(map[uint64]*headerRequest), + } + + success := make(chan *headerResponse, 1) + fail := make(chan *headerRequest, 1) + cancel := make(chan struct{}) + + wake := skeleton.assignTasks(success, fail, cancel) + if wake.IsZero() { + t.Fatal("expected a non-zero backoff wakeup when all peers are backed off") + } + if until := time.Until(wake); until <= 0 || until > 30*time.Second { + t.Fatalf("backoff wakeup mismatch: have %v remaining, want (0, %v]", until, 30*time.Second) + } + for _, owner := range skeleton.scratchOwners { + if owner != "" { + t.Fatalf("no task should be assigned to backed-off peers, got owner %q", owner) + } + } +} + +func TestSkeletonSyncWakesAfterBackoff(t *testing.T) { + chain := []*types.Header{{Number: big.NewInt(0)}} + for i := 1; i < 2*requestHeaders+2; i++ { + chain = append(chain, &types.Header{ + ParentHash: chain[i-1].Hash(), + Number: big.NewInt(int64(i)), + }) + } + + db := rawdb.NewMemoryDatabase() + rawdb.WriteBlock(db, types.NewBlockWithHeader(chain[0])) + rawdb.WriteReceipts(db, chain[0].Hash(), chain[0].Number.Uint64(), types.Receipts{}) + + peerset := newPeerSet() + testPeer := newSkeletonTestPeer("backed-off", chain) + peer := newPeerConnection("backed-off", eth.ETH69, testPeer, log.New("id", "backed-off")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + peer.backoffFor(100 * time.Millisecond) + + skeleton := newSkeleton(db, peerset, func(string) {}, newHookedBackfiller()) + if err := skeleton.Sync(chain[len(chain)-1], nil, true); err != nil { + t.Fatalf("failed to announce sync head: %v", err) + } + defer skeleton.Terminate() + + var progress skeletonProgress + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) + if len(progress.Subchains) == 1 && progress.Subchains[0].Tail == 1 { + break + } + time.Sleep(20 * time.Millisecond) + } + + if len(progress.Subchains) != 1 || progress.Subchains[0].Tail != 1 { + t.Fatalf("skeleton did not link after backoff expiry: %+v", progress.Subchains) + } + if testPeer.served.Load() == 0 { + t.Fatal("backed-off peer never served headers after backoff expiry") + } +} + +func TestSkeletonSyncBacksOffOnTimeout(t *testing.T) { + chain := []*types.Header{{Number: big.NewInt(0)}} + for i := 1; i < 2*requestHeaders+2; i++ { + chain = append(chain, &types.Header{ + ParentHash: chain[i-1].Hash(), + Number: big.NewInt(int64(i)), + }) + } + + db := rawdb.NewMemoryDatabase() + rawdb.WriteBlock(db, types.NewBlockWithHeader(chain[0])) + rawdb.WriteReceipts(db, chain[0].Hash(), chain[0].Number.Uint64(), types.Receipts{}) + + peerset := newPeerSet() + peerset.rates.OverrideTTLLimit = 100 * time.Millisecond + + testPeer := newSkeletonTestPeer("stuck", chain) + testPeer.hang.Store(true) + peer := newPeerConnection("stuck", eth.ETH69, testPeer, log.New("id", "stuck")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + var dropped atomic.Bool + skeleton := newSkeleton(db, peerset, func(string) { dropped.Store(true) }, newHookedBackfiller()) + if err := skeleton.Sync(chain[len(chain)-1], nil, true); err != nil { + t.Fatalf("failed to announce sync head: %v", err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !peer.backedOff() { + time.Sleep(20 * time.Millisecond) + } + + skeleton.Terminate() + + if !peer.backedOff() { + t.Fatal("timed-out skeleton peer should be backed off") + } + if dropped.Load() { + t.Fatal("timed-out skeleton peer should not be hard-dropped") + } + if peerset.persistentBackoff("stuck") <= 0 { + t.Fatal("a timed-out skeleton peer must persist a jail across reconnects") + } + if _, ok := skeleton.idles["stuck"]; !ok { + t.Fatal("a timed-out skeleton peer must be requeued into the idle set, not stranded") + } +} + +func TestSkeletonDropForInvalidHeadersBenches(t *testing.T) { + peerset := newPeerSet() + testPeer := newSkeletonTestPeer("junk", []*types.Header{{Number: big.NewInt(0)}}) + peer := newPeerConnection("junk", eth.ETH69, testPeer, log.New("id", "junk")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + var dropped atomic.Bool + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, func(string) { dropped.Store(true) }, newHookedBackfiller()) + skeleton.dropForInvalidHeaders("junk") + + if !peer.backedOff() { + t.Fatal("a peer dropped for invalid skeleton headers must be benched") + } + if !dropped.Load() { + t.Fatal("a peer dropped for invalid skeleton headers must be disconnected") + } + if got := peerset.persistentBackoff("junk"); got <= peerJailBackoff { + t.Fatalf("invalid-headers drop must persist the long drop bench across reconnects: got %v, want > %v", got, peerJailBackoff) + } +} + +func TestSkeletonDropForInvalidHeadersWithoutDropper(t *testing.T) { + peerset := newPeerSet() + testPeer := newSkeletonTestPeer("junk", []*types.Header{{Number: big.NewInt(0)}}) + peer := newPeerConnection("junk", eth.ETH69, testPeer, log.New("id", "junk")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, nil, newHookedBackfiller()) + skeleton.dropForInvalidHeaders("junk") + + if !peer.backedOff() { + t.Fatal("invalid-headers drop must still bench the peer when no dropper is configured") + } +} + +func TestSkeletonDropForInvalidHeadersBenchesDepartedPeer(t *testing.T) { + peerset := newPeerSet() + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, func(string) {}, newHookedBackfiller()) + skeleton.dropForInvalidHeaders("gone") + + if got := peerset.persistentBackoff("gone"); got <= peerJailBackoff { + t.Fatalf("a departed peer that delivered invalid headers must still be benched: got %v, want > %v", got, peerJailBackoff) + } +} + +func TestSkeletonHeaderTimeoutBacksOffPeer(t *testing.T) { + peerset := newPeerSet() + testPeer := newSkeletonTestPeer("slow", []*types.Header{{Number: big.NewInt(0)}}) + peer := newPeerConnection("slow", eth.ETH69, testPeer, log.New("id", "slow")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, func(string) {}, newHookedBackfiller()) + + req := &headerRequest{ + peer: "slow", + stale: make(chan struct{}), + cancel: make(chan struct{}), + revert: make(chan *headerRequest, 1), + } + skeleton.handleHeaderTimeout(peer, req, time.Second) + + if !peer.backedOff() { + t.Fatal("a skeleton header timeout must bench the peer") + } +} + +func TestSkeletonHeaderTimeoutSkipsDepartedPeer(t *testing.T) { + peerset := newPeerSet() + testPeer := newSkeletonTestPeer("departed", []*types.Header{{Number: big.NewInt(0)}}) + peer := newPeerConnection("departed", eth.ETH69, testPeer, log.New("id", "departed")) + if err := peerset.Register(peer); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, func(string) {}, newHookedBackfiller()) + + req := &headerRequest{ + peer: "departed", + stale: make(chan struct{}), + cancel: make(chan struct{}), + revert: make(chan *headerRequest, 1), + } + close(req.stale) + skeleton.handleHeaderTimeout(peer, req, time.Second) + + if peer.backedOff() { + t.Fatal("a header timeout for an already-reverted request must not bench the departed peer") + } + if got := peerset.persistentBackoff("departed"); got != 0 { + t.Fatalf("a header timeout for an already-reverted request must not jail the peer: got %v", got) + } +} + +func TestSkeletonHandleRequestFailRequeuesBenchedPeer(t *testing.T) { + peerset := newPeerSet() + benched := newPeerConnection("benched", eth.ETH69, newSkeletonTestPeer("benched", []*types.Header{{Number: big.NewInt(0)}}), log.New("id", "benched")) + fresh := newPeerConnection("fresh", eth.ETH69, newSkeletonTestPeer("fresh", []*types.Header{{Number: big.NewInt(0)}}), log.New("id", "fresh")) + if err := peerset.Register(benched); err != nil { + t.Fatalf("failed to register benched peer: %v", err) + } + if err := peerset.Register(fresh); err != nil { + t.Fatalf("failed to register fresh peer: %v", err) + } + skeleton := newSkeleton(rawdb.NewMemoryDatabase(), peerset, func(string) {}, newHookedBackfiller()) + skeleton.idles = make(map[string]*peerConnection) + skeleton.scratchHead = requestHeaders + skeleton.scratchOwners = make([]string, scratchHeaders/requestHeaders) + + benched.backoffFor(time.Minute) + skeleton.handleRequestFail(&headerRequest{peer: "benched", head: requestHeaders, stale: make(chan struct{})}) + if _, ok := skeleton.idles["benched"]; !ok { + t.Fatal("a benched peer must be requeued so the scheduler arms its wake-up timer") + } + + skeleton.handleRequestFail(&headerRequest{peer: "fresh", head: requestHeaders, stale: make(chan struct{})}) + if _, ok := skeleton.idles["fresh"]; ok { + t.Fatal("a non-benched failed peer must not be requeued, to avoid re-admitting a peer that delivered an unusable batch") + } +} + +func TestSkeletonInvalidHeadersBenchesPeerEndToEnd(t *testing.T) { + chain := []*types.Header{{Number: big.NewInt(0)}} + for i := 1; i < 2*requestHeaders+2; i++ { + chain = append(chain, &types.Header{ + ParentHash: chain[i-1].Hash(), + Number: big.NewInt(int64(i)), + }) + } + + db := rawdb.NewMemoryDatabase() + rawdb.WriteBlock(db, types.NewBlockWithHeader(chain[0])) + rawdb.WriteReceipts(db, chain[0].Hash(), chain[0].Number.Uint64(), types.Receipts{}) + + bad := append([]*types.Header{}, chain...) + corrupt := *chain[len(chain)-2] + corrupt.Extra = []byte("corrupt skeleton header") + bad[len(bad)-2] = &corrupt + testPeer := newSkeletonTestPeer("duper", bad) + peerset := newPeerSet() + if err := peerset.Register(newPeerConnection("duper", eth.ETH69, testPeer, log.New("id", "duper"))); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + + var dropped atomic.Bool + drop := func(id string) { + if p := peerset.Peer(id); p != nil { + p.peer.(*skeletonTestPeer).dropped.Add(1) + } + dropped.Store(true) + } + skeleton := newSkeleton(db, peerset, drop, newHookedBackfiller()) + if err := skeleton.Sync(chain[len(chain)-1], nil, true); err != nil { + t.Fatalf("failed to announce sync head: %v", err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !dropped.Load() { + time.Sleep(20 * time.Millisecond) + } + + skeleton.Terminate() + + if !dropped.Load() { + t.Fatal("a peer delivering invalid skeleton headers must be dropped") + } + if got := peerset.persistentBackoff("duper"); got <= peerJailBackoff { + t.Fatalf("the invalid-headers drop call site must persist the long drop bench: got %v, want > %v", got, peerJailBackoff) + } +} + // Tests that the skeleton sync correctly retrieves headers from one or more // peers without duplicates or other strange side effects. func TestSkeletonSyncRetrievals(t *testing.T) { diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 200e3d36eb..666d919549 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -607,11 +607,11 @@ func (f *TxFetcher) loop() { for peer, txset := range f.waitslots { if meta := txset[hash]; meta != nil { if delivery.metas[i].kind != meta.kind { - log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind) + log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind, "origin", delivery.origin, "direct", delivery.direct) f.dropPeer(peer) } else if delivery.metas[i].size != meta.size { if math.Abs(float64(delivery.metas[i].size)-float64(meta.size)) > 8 { - log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size) + log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size, "origin", delivery.origin, "direct", delivery.direct) // Normally we should drop a peer considering this is a protocol violation. // However, due to the RLP vs consensus format messyness, allow a few bytes @@ -635,11 +635,11 @@ func (f *TxFetcher) loop() { for peer, txset := range f.announces { if meta := txset[hash]; meta != nil { if delivery.metas[i].kind != meta.kind { - log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind) + log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind, "origin", delivery.origin, "direct", delivery.direct) f.dropPeer(peer) } else if delivery.metas[i].size != meta.size { if math.Abs(float64(delivery.metas[i].size)-float64(meta.size)) > 8 { - log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size) + log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size, "origin", delivery.origin, "direct", delivery.direct) // Normally we should drop a peer considering this is a protocol violation. // However, due to the RLP vs consensus format messyness, allow a few bytes @@ -744,7 +744,7 @@ func (f *TxFetcher) loop() { } // If we encountered a protocol violation, disconnect the peer if delivery.violation != nil { - log.Warn("Disconnect peer for protocol violation", "peer", delivery.origin, "error", delivery.violation) + log.Warn("Disconnect peer for protocol violation", "peer", delivery.origin, "direct", delivery.direct, "hashes", len(delivery.hashes), "error", delivery.violation) f.dropPeer(delivery.origin) } diff --git a/eth/peer_jail_p2p_test.go b/eth/peer_jail_p2p_test.go new file mode 100644 index 0000000000..ad03fe51c3 --- /dev/null +++ b/eth/peer_jail_p2p_test.go @@ -0,0 +1,478 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "fmt" + "math" + "math/big" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/downloader" + ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/params" +) + +func TestFakeEthPeerPrunedSidechainIsLocallyJailed(t *testing.T) { + fixture := newPrunedSidechainP2PFixture(t) + + handler, cleanup := newPeerJailP2PHandler(t, fixture.db, fixture.genesis, fixture.chain) + defer cleanup() + + host := startEthP2PServer(t, handler) + fake := startFakeETH69Server(t, fixture) + + host.AddPeer(fake.server.Self()) + waitForEthPeer(t, handler, fake.server.Self().ID().String()) + + peerID := fake.server.Self().ID().String() + beforePeers := host.PeerCount() + if beforePeers != 1 { + t.Fatalf("p2p peer count mismatch before sync: have %d, want 1", beforePeers) + } + + err := handler.downloader.LegacySync(peerID, fixture.head.Hash(), fixture.td, nil, downloader.FullSync) + if err == nil || !strings.Contains(err.Error(), "sidechain ghost-state attack") { + t.Fatalf("sync error mismatch: have %v, want sidechain ghost-state attack", err) + } + if afterPeers := host.PeerCount(); afterPeers != beforePeers { + t.Fatalf("p2p peer count changed: have %d, want %d", afterPeers, beforePeers) + } + if handler.peers.peer(peerID) == nil { + t.Fatalf("fake peer was removed from eth peer set") + } + + err = handler.downloader.LegacySync(peerID, fixture.head.Hash(), fixture.td, nil, downloader.FullSync) + if !errors.Is(err, downloader.ErrPeerBackedOff) { + t.Fatalf("backed off sync error mismatch: have %v, want %v", err, downloader.ErrPeerBackedOff) + } +} + +type prunedSidechainP2PFixture struct { + db ethdb.Database + genesis *core.Genesis + chain *core.BlockChain + remote *fakeEthChain + head *types.Block + td *big.Int +} + +func newPrunedSidechainP2PFixture(t *testing.T) *prunedSidechainP2PFixture { + t.Helper() + + chainConfig := *params.TestChainConfig + chainConfig.TerminalTotalDifficulty = big.NewInt(math.MaxInt64) + engine := ethash.NewFaker() + genesis, addTx := newSidechainP2PGenesis(t, &chainConfig) + genDB, blocks := generateCanonicalP2PChain(genesis, engine, addTx) + db := openP2PFixtureDB(t) + chain := newP2PFixtureChain(t, db, genesis, engine, blocks) + fork := generatePrunedSidechain(t, chain, genesis, engine, genDB, blocks, addTx) + + remote := newFakeEthChain(chain.Genesis(), blocks[:len(blocks)-state.TriesInMemory], fork) + head := fork[len(fork)-1] + td := new(big.Int).Add(chain.GetTd(chain.CurrentBlock().Hash(), chain.CurrentBlock().Number.Uint64()), big.NewInt(1)) + + return &prunedSidechainP2PFixture{ + db: db, + genesis: genesis, + chain: chain, + remote: remote, + head: head, + td: td, + } +} + +func newSidechainP2PGenesis(t *testing.T, chainConfig *params.ChainConfig) (*core.Genesis, func(uint64, *core.BlockGen)) { + t.Helper() + + key, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + if err != nil { + t.Fatal(err) + } + addr := crypto.PubkeyToAddress(key.PublicKey) + genesis := &core.Genesis{ + Config: chainConfig, + Alloc: types.GenesisAlloc{ + addr: {Balance: big.NewInt(math.MaxInt64)}, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + addTx := func(nonce uint64, b *core.BlockGen) { + tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, b.BaseFee(), nil), signer, key) + if err != nil { + t.Fatal(err) + } + b.AddTx(tx) + } + return genesis, addTx +} + +func generateCanonicalP2PChain(genesis *core.Genesis, engine consensus.Engine, addTx func(uint64, *core.BlockGen)) (ethdb.Database, []*types.Block) { + genDB, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, 2*state.TriesInMemory, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + addTx(uint64(i), b) + b.SetExtra([]byte("canonical")) + }) + return genDB, blocks +} + +func openP2PFixtureDB(t *testing.T) ethdb.Database { + t.Helper() + + datadir := t.TempDir() + pdb, err := pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatal(err) + } + db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: filepath.Join(datadir, "ancient")}) + if err != nil { + t.Fatal(err) + } + return db +} + +func newP2PFixtureChain(t *testing.T, db ethdb.Database, genesis *core.Genesis, engine consensus.Engine, blocks []*types.Block) *core.BlockChain { + t.Helper() + + chain, err := core.NewBlockChain(db, genesis, engine, core.DefaultConfig().WithStateScheme(rawdb.HashScheme)) + if err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + chain.Stop() + db.Close() + }) + + if n, err := chain.InsertChain(blocks, false); err != nil { + t.Fatalf("failed to insert canonical block %d: %v", n, err) + } + return chain +} + +func generatePrunedSidechain(t *testing.T, chain *core.BlockChain, genesis *core.Genesis, engine consensus.Engine, genDB ethdb.Database, blocks []*types.Block, addTx func(uint64, *core.BlockGen)) []*types.Block { + t.Helper() + + parent := blocks[len(blocks)-state.TriesInMemory-1] + if chain.HasBlockAndState(parent.Hash(), parent.NumberU64()) { + t.Fatalf("parent state is still available: number %d", parent.NumberU64()) + } + fork, _ := core.GenerateChain(genesis.Config, parent, engine, genDB, state.TriesInMemory+2, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + addTx(parent.NumberU64()+uint64(i), b) + b.SetExtra([]byte("side")) + }) + + canonical := chain.GetBlockByNumber(fork[0].NumberU64()) + if canonical == nil { + t.Fatalf("canonical block %d not found", fork[0].NumberU64()) + } + if canonical.Hash() == fork[0].Hash() { + t.Fatalf("side block hash matches canonical block hash") + } + if canonical.Root() != fork[0].Root() { + t.Fatalf("side block root mismatch: have %v, want %v", fork[0].Root(), canonical.Root()) + } + return fork +} + +func newPeerJailP2PHandler(t *testing.T, db ethdb.Database, genesis *core.Genesis, chain *core.BlockChain) (*handler, func()) { + t.Helper() + + handler, err := newHandler(&handlerConfig{ + Database: db, + Chain: chain, + TxPool: newTestTxPool(), + Network: genesis.Config.ChainID.Uint64(), + Sync: downloader.FullSync, + BloomCache: 1, + }) + if err != nil { + t.Fatal(err) + } + handler.Start(1000) + + return handler, func() { + handler.Stop() + } +} + +func startEthP2PServer(t *testing.T, handler *handler) *p2p.Server { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + server := &p2p.Server{ + Config: p2p.Config{ + PrivateKey: key, + ListenAddr: "127.0.0.1:0", + NoDiscovery: true, + MaxPeers: 10, + Protocols: ethproto.MakeProtocols((*ethHandler)(handler), handler.networkID, nil), + }, + } + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(server.Stop) + + return server +} + +type fakeETH69Server struct { + server *p2p.Server +} + +func startFakeETH69Server(t *testing.T, fixture *prunedSidechainP2PFixture) *fakeETH69Server { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + server := &p2p.Server{ + Config: p2p.Config{ + PrivateKey: key, + ListenAddr: "127.0.0.1:0", + NoDial: true, + NoDiscovery: true, + MaxPeers: 1, + Protocols: []p2p.Protocol{{ + Name: ethproto.ProtocolName, + Version: ethproto.ETH69, + Length: 18, + Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + return runFakeETH69Peer(fixture, rw) + }, + }}, + }, + } + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(server.Stop) + + return &fakeETH69Server{server: server} +} + +func serveGetBlockHeaders(fixture *prunedSidechainP2PFixture, rw p2p.MsgReadWriter, msg p2p.Msg) error { + var req ethproto.GetBlockHeadersPacket + if err := msg.Decode(&req); err != nil { + msg.Discard() + return err + } + msg.Discard() + return p2p.Send(rw, ethproto.BlockHeadersMsg, ðproto.BlockHeadersPacket{ + RequestId: req.RequestId, + BlockHeadersRequest: fixture.remote.headers(req.GetBlockHeadersRequest), + }) +} + +func serveGetBlockBodies(fixture *prunedSidechainP2PFixture, rw p2p.MsgReadWriter, msg p2p.Msg) error { + var req ethproto.GetBlockBodiesPacket + if err := msg.Decode(&req); err != nil { + msg.Discard() + return err + } + msg.Discard() + return p2p.Send(rw, ethproto.BlockBodiesMsg, ðproto.BlockBodiesPacket{ + RequestId: req.RequestId, + BlockBodiesResponse: fixture.remote.bodies(req.GetBlockBodiesRequest), + }) +} + +func runFakeETH69Peer(fixture *prunedSidechainP2PFixture, rw p2p.MsgReadWriter) error { + status := ðproto.StatusPacket69{ + ProtocolVersion: ethproto.ETH69, + NetworkID: fixture.genesis.Config.ChainID.Uint64(), + TD: fixture.td, + Genesis: fixture.chain.Genesis().Hash(), + ForkID: forkid.NewID(fixture.chain.Config(), fixture.chain.Genesis(), fixture.head.NumberU64(), fixture.head.Time()), + EarliestBlock: 0, + LatestBlock: fixture.head.NumberU64(), + LatestBlockHash: fixture.head.Hash(), + } + if err := fakeETH69Handshake(rw, status); err != nil { + return err + } + for { + msg, err := rw.ReadMsg() + if err != nil { + return err + } + switch msg.Code { + case ethproto.GetBlockHeadersMsg: + if err := serveGetBlockHeaders(fixture, rw, msg); err != nil { + return err + } + case ethproto.GetBlockBodiesMsg: + if err := serveGetBlockBodies(fixture, rw, msg); err != nil { + return err + } + default: + msg.Discard() + } + } +} + +func fakeETH69Handshake(rw p2p.MsgReadWriter, status *ethproto.StatusPacket69) error { + errc := make(chan error, 2) + go func() { + errc <- p2p.Send(rw, ethproto.StatusMsg, status) + }() + go func() { + msg, err := rw.ReadMsg() + if err != nil { + errc <- err + return + } + defer msg.Discard() + if msg.Code != ethproto.StatusMsg { + errc <- fmt.Errorf("unexpected status code %d", msg.Code) + return + } + var remote ethproto.StatusPacket69 + errc <- msg.Decode(&remote) + }() + + var first error + for i := 0; i < 2; i++ { + if err := <-errc; err != nil && first == nil { + first = err + } + } + return first +} + +type fakeEthChain struct { + headersByHash map[common.Hash]*types.Header + headersByNumber map[uint64]*types.Header + blocksByHash map[common.Hash]*types.Block +} + +func newFakeEthChain(genesis *types.Block, canonicalPrefix []*types.Block, fork []*types.Block) *fakeEthChain { + chain := &fakeEthChain{ + headersByHash: make(map[common.Hash]*types.Header), + headersByNumber: make(map[uint64]*types.Header), + blocksByHash: make(map[common.Hash]*types.Block), + } + chain.addBlock(genesis) + for _, block := range canonicalPrefix { + chain.addBlock(block) + } + for _, block := range fork { + chain.addBlock(block) + } + return chain +} + +func (c *fakeEthChain) addBlock(block *types.Block) { + header := block.Header() + c.headersByHash[block.Hash()] = header + c.headersByNumber[block.NumberU64()] = header + c.blocksByHash[block.Hash()] = block +} + +func (c *fakeEthChain) headers(req *ethproto.GetBlockHeadersRequest) ethproto.BlockHeadersRequest { + if req == nil || req.Amount == 0 { + return nil + } + var number uint64 + if req.Origin.Hash != (common.Hash{}) { + header := c.headersByHash[req.Origin.Hash] + if header == nil { + return nil + } + number = header.Number.Uint64() + } else { + number = req.Origin.Number + } + + headers := make([]*types.Header, 0, req.Amount) + step := req.Skip + 1 + for uint64(len(headers)) < req.Amount { + header := c.headersByNumber[number] + if header == nil { + break + } + headers = append(headers, header) + if req.Reverse { + if number < step { + break + } + number -= step + continue + } + number += step + } + return headers +} + +func (c *fakeEthChain) bodies(hashes ethproto.GetBlockBodiesRequest) ethproto.BlockBodiesResponse { + bodies := make(ethproto.BlockBodiesResponse, 0, len(hashes)) + for _, hash := range hashes { + block := c.blocksByHash[hash] + if block == nil { + continue + } + bodies = append(bodies, ðproto.BlockBody{ + Transactions: block.Transactions(), + Uncles: block.Uncles(), + Withdrawals: block.Withdrawals(), + }) + } + return bodies +} + +func waitForEthPeer(t *testing.T, handler *handler, id string) { + t.Helper() + + deadline := time.After(5 * time.Second) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-deadline: + t.Fatalf("timed out waiting for eth peer %s", id) + case <-ticker.C: + if handler.peers.peer(id) != nil { + return + } + } + } +} diff --git a/eth/peerset.go b/eth/peerset.go index 43ec2e1832..29cdd2d660 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -23,6 +23,7 @@ import ( "math/big" "slices" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -401,23 +402,32 @@ func (ps *peerSet) getAllPeers() []*ethPeer { } // peerWithHighestTD retrieves the known peer with the currently highest total -// difficulty, but below the given PoS switchover threshold. -func (ps *peerSet) peerWithHighestTD() *eth.Peer { +// difficulty. +func (ps *peerSet) peerWithHighestTD(backoff func(string) time.Duration) (*eth.Peer, time.Duration) { ps.lock.RLock() defer ps.lock.RUnlock() var ( - bestPeer *eth.Peer - bestTd *big.Int + bestPeer *eth.Peer + bestTd *big.Int + shortestBackoff time.Duration ) for _, p := range ps.peers { + if backoff != nil { + if remaining := backoff(p.ID()); remaining > 0 { + if shortestBackoff == 0 || remaining < shortestBackoff { + shortestBackoff = remaining + } + continue + } + } if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { bestPeer, bestTd = p.Peer, td } } - return bestPeer + return bestPeer, shortestBackoff } // close disconnects all peers. diff --git a/eth/peerset_test.go b/eth/peerset_test.go index 16d4d9dc1b..458d7b3091 100644 --- a/eth/peerset_test.go +++ b/eth/peerset_test.go @@ -2,7 +2,9 @@ package eth import ( "crypto/rand" + "math/big" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -61,3 +63,61 @@ func TestPeerSetForgetTransactionsEmpty(t *testing.T) { // ForgetTransactions should not panic with no peers ps.ForgetTransactions([]common.Hash{{1}, {2}, {3}}) } + +func TestPeerWithHighestTDSkipsBackedOffPeers(t *testing.T) { + ps := newPeerSet() + defer ps.close() + + low := registerPeerWithTD(t, ps, 10) + high := registerPeerWithTD(t, ps, 20) + + best, retry := ps.peerWithHighestTD(func(id string) time.Duration { + if id == high.ID() { + return 30 * time.Second + } + return 0 + }) + if best == nil || best.ID() != low.ID() { + t.Fatalf("best peer mismatch: have %v, want %v", best, low.ID()) + } + if retry != 30*time.Second { + t.Fatalf("retry delay mismatch: have %v, want %v", retry, 30*time.Second) + } + + best, retry = ps.peerWithHighestTD(func(id string) time.Duration { + if id == low.ID() { + return 10 * time.Second + } + return time.Minute + }) + if best != nil { + t.Fatalf("unexpected peer selected while all peers were backed off: %v", best.ID()) + } + if retry != 10*time.Second { + t.Fatalf("retry delay mismatch: have %v, want %v", retry, 10*time.Second) + } +} + +func registerPeerWithTD(t *testing.T, ps *peerSet, td int64) *eth.Peer { + t.Helper() + + app, net := p2p.MsgPipe() + t.Cleanup(func() { + app.Close() + net.Close() + }) + + var id enode.ID + if _, err := rand.Read(id[:]); err != nil { + t.Fatalf("failed to create peer id: %v", err) + } + + peer := eth.NewPeer(eth.ETH68, p2p.NewPeer(id, "test", nil), net, nil) + peer.SetHead(common.Hash{byte(td)}, big.NewInt(td)) + t.Cleanup(peer.Close) + + if err := ps.registerPeer(peer, nil, nil); err != nil { + t.Fatalf("failed to register peer: %v", err) + } + return peer +} diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index 0eba432e5b..f53c72da91 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -25,9 +25,9 @@ import ( ) var ( - // errDisconnected is returned if a request is attempted to be made to a peer + // ErrDisconnected is returned if a request is attempted to be made to a peer // that was already closed. - errDisconnected = errors.New("disconnected") + ErrDisconnected = errors.New("disconnected") // errDanglingResponse is returned if a response arrives with a request id // which does not match to any existing pending requests. @@ -70,7 +70,7 @@ func (r *Request) Close() error { return err } case <-r.peer.term: - return errDisconnected + return ErrDisconnected } } @@ -148,7 +148,7 @@ func (p *Peer) dispatchRequest(req *Request) error { case p.reqDispatch <- reqOp: return <-reqOp.fail case <-p.term: - return errDisconnected + return ErrDisconnected } } @@ -191,12 +191,12 @@ func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) erro case <-res.Req.Cancel: return nil // Request cancelled, silently discard response case <-p.term: - return errDisconnected + return ErrDisconnected } } case <-p.term: - return errDisconnected + return ErrDisconnected } } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index bc7d5de9f4..cb522863ef 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -106,6 +106,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe Peer: p, rw: rw, version: version, + td: new(big.Int), knownTxs: newKnownCache(maxKnownTxs), knownBlocks: newKnownCache(maxKnownBlocks), queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), diff --git a/eth/sync.go b/eth/sync.go index 30c25a1507..e86ce4f142 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/downloader/whitelist" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" ) @@ -59,6 +60,9 @@ type chainSyncer struct { warned time.Time peerEventCh chan struct{} doneCh chan error // non-nil when sync is running + + peersUnavailableUntil time.Time + peersUnavailableAtCount int } // chainSyncOp is a scheduled sync operation. @@ -105,49 +109,122 @@ func (cs *chainSyncer) loop() { cs.force = time.NewTimer(forceSyncCycle) defer cs.force.Stop() + retry := newResettableTimer() + defer retry.stop() + for { - if op := cs.nextSyncOp(); op != nil { + if op, wait := cs.nextSyncOp(); op != nil { + retry.stop() cs.startSync(op) + } else { + retry.reset(wait) } select { case <-cs.peerEventCh: // Peer information changed, recheck. + cs.onPeerEvent() case err := <-cs.doneCh: - cs.doneCh = nil - cs.force.Reset(forceSyncCycle) - cs.forced = false - - // If we've reached the merge transition but no beacon client is available, or - // it has not yet switched us over, keep warning the user that their infra is - // potentially flaky. - if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - - cs.warned = time.Now() - } + cs.onSyncDone(err) case <-cs.force.C: cs.forced = true - + case <-retry.C(): + retry.markFired() case <-cs.handler.quitSync: - // Disable all insertion on the blockchain. This needs to happen before - // terminating the downloader because the downloader waits for blockchain - // inserts, and these can take a long time to finish. - cs.handler.chain.StopInsert() - cs.handler.downloader.Terminate() - - if cs.doneCh != nil { - <-cs.doneCh - } - + cs.shutdown() return } } } +func (cs *chainSyncer) onSyncDone(err error) { + cs.doneCh = nil + cs.force.Reset(forceSyncCycle) + cs.forced = false + + if errors.Is(err, downloader.ErrPeersUnavailable) || errors.Is(err, downloader.ErrPeerBackedOff) || errors.Is(err, whitelist.ErrNoRemote) { + cs.peersUnavailableUntil = time.Now().Add(forceSyncCycle) + cs.peersUnavailableAtCount = cs.handler.peers.len() + } else { + cs.peersUnavailableUntil = time.Time{} + } + + // If we've reached the merge transition but no beacon client is available, or + // it has not yet switched us over, keep warning the user that their infra is + // potentially flaky. + if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { + log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") + + cs.warned = time.Now() + } +} + +func (cs *chainSyncer) onPeerEvent() { + if !cs.peersUnavailableUntil.IsZero() && cs.handler.peers.len() != cs.peersUnavailableAtCount { + cs.peersUnavailableUntil = time.Time{} + } +} + +func (cs *chainSyncer) shutdown() { + // Disable all insertion on the blockchain. This needs to happen before + // terminating the downloader because the downloader waits for blockchain + // inserts, and these can take a long time to finish. + cs.handler.chain.StopInsert() + cs.handler.downloader.Terminate() + + if cs.doneCh != nil { + <-cs.doneCh + } +} + +type resettableTimer struct { + timer *time.Timer + active bool +} + +func newResettableTimer() *resettableTimer { + timer := time.NewTimer(time.Hour) + timer.Stop() + + return &resettableTimer{timer: timer} +} + +func (r *resettableTimer) C() <-chan time.Time { + return r.timer.C +} + +func (r *resettableTimer) markFired() { + r.active = false +} + +func (r *resettableTimer) stop() { + if !r.active { + return + } + if !r.timer.Stop() { + select { + case <-r.timer.C: + default: + } + } + r.active = false +} + +func (r *resettableTimer) reset(wait time.Duration) { + r.stop() + if wait <= 0 { + return + } + r.timer.Reset(wait) + r.active = true +} + // nextSyncOp determines whether sync is required at this time. -func (cs *chainSyncer) nextSyncOp() *chainSyncOp { +func (cs *chainSyncer) nextSyncOp() (*chainSyncOp, time.Duration) { if cs.doneCh != nil { - return nil // Sync already running + return nil, 0 // Sync already running + } + if remaining := time.Until(cs.peersUnavailableUntil); remaining > 0 { + return nil, remaining } // Ensure we're at minimum peer count. minPeers := defaultMinSyncPeers @@ -158,14 +235,14 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { } if cs.handler.peers.len() < minPeers { - return nil + return nil, 0 } // We have enough peers, pick the one with the highest TD, but avoid going // over the terminal total difficulty. Above that we expect the consensus // clients to direct the chain head to sync to. - peer := cs.handler.peers.peerWithHighestTD() + peer, retry := cs.handler.peers.peerWithHighestTD(cs.handler.downloader.PeerBackoff) if peer == nil { - return nil + return nil, retry } mode, ourTD := cs.modeAndLocalHead() @@ -185,10 +262,10 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { cs.warned = time.Now() } - return nil // We're in sync + return nil, retry // In sync with the available peer; the retry hint still wakes us when benched higher-TD peers expire } - return op + return op, 0 } func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { diff --git a/eth/sync_test.go b/eth/sync_test.go index 3ab0ec1d7d..7f2ef8c267 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -17,14 +17,22 @@ package eth import ( + "errors" + "math/big" "testing" "time" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/downloader/whitelist" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" ) // Tests that snap sync is disabled after a successful sync cycle. @@ -33,6 +41,418 @@ func TestSnapSyncDisabling69(t *testing.T) { testSnapSyncDisabling(t, eth.ETH69, // Skipping as eth/68 nodes are filtered out during snap sync // func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } +func TestChainSyncerNextSyncOpStates(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.doneCh = make(chan error, 1) + if op, wait := syncer.nextSyncOp(); op != nil || wait != 0 { + t.Fatalf("running sync mismatch: op %v wait %v, want nil/0", op, wait) + } + + syncer.doneCh = nil + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + op, wait := syncer.nextSyncOp() + if op == nil { + t.Fatal("expected sync operation") + } + if op.peer.ID() != peer.ID() { + t.Fatalf("sync peer mismatch: have %v, want %v", op.peer.ID(), peer.ID()) + } + if wait != 0 { + t.Fatalf("sync wait mismatch: have %v, want 0", wait) + } +} + +func TestChainSyncerNextSyncOpSkipsBackedOffPeer(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + setDownloaderPeerBackoff(t, handler.downloader, peer.ID(), time.Hour) + op, wait := syncer.nextSyncOp() + if op != nil { + t.Fatalf("expected no sync op while peer backed off, got %v", op) + } + if wait <= 0 || wait > time.Hour { + t.Fatalf("retry wait mismatch: have %v, want (0, 1h]", wait) + } + + setDownloaderPeerBackoff(t, handler.downloader, peer.ID(), 0) + op, _ = syncer.nextSyncOp() + if op == nil { + t.Fatal("expected sync op after backoff cleared") + } +} + +func TestChainSyncerCoolsOffOnPeersUnavailable(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.force = time.NewTimer(forceSyncCycle) + defer syncer.force.Stop() + + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + syncer.onSyncDone(downloader.ErrPeersUnavailable) + if op, wait := syncer.nextSyncOp(); op != nil || wait <= 0 { + t.Fatalf("expected no sync op and a positive cooldown wait, got op=%v wait=%v", op, wait) + } + + syncer.peersUnavailableUntil = time.Time{} + if op, _ := syncer.nextSyncOp(); op == nil { + t.Fatal("expected a sync op once the peers-unavailable cooldown is cleared") + } +} + +func TestChainSyncerArmsRetryForBenchedHigherTDPeer(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.forced = true + + low := registerPeerWithTD(t, handler.peers, 0) + if err := handler.downloader.RegisterPeer(low.ID(), eth.ETH68, ðPeer{Peer: low}); err != nil { + t.Fatal(err) + } + high := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(high.ID(), eth.ETH68, ðPeer{Peer: high}); err != nil { + t.Fatal(err) + } + setDownloaderPeerBackoff(t, handler.downloader, high.ID(), time.Hour) + + op, wait := syncer.nextSyncOp() + if op != nil { + t.Fatalf("expected no sync op when in sync with the only eligible peer, got %v", op) + } + if wait <= 0 { + t.Fatal("retry timer must be armed for the benched higher-TD peer's backoff expiry") + } +} + +func TestChainSyncerCoolsOffOnPeerBackedOff(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.force = time.NewTimer(forceSyncCycle) + defer syncer.force.Stop() + + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + syncer.onSyncDone(downloader.ErrPeerBackedOff) + if op, wait := syncer.nextSyncOp(); op != nil || wait <= 0 { + t.Fatalf("ErrPeerBackedOff must arm the retry cooldown, got op=%v wait=%v", op, wait) + } +} + +func TestChainSyncerCoolsOffOnNoRemote(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.force = time.NewTimer(forceSyncCycle) + defer syncer.force.Stop() + + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + syncer.onSyncDone(whitelist.ErrNoRemote) + if op, wait := syncer.nextSyncOp(); op != nil || wait <= 0 { + t.Fatalf("ErrNoRemote must arm the retry cooldown without benching, got op=%v wait=%v", op, wait) + } +} + +func TestChainSyncerCooldownSurvivesBlockAnnounce(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + syncer := newChainSyncer(handler) + syncer.force = time.NewTimer(forceSyncCycle) + defer syncer.force.Stop() + + peer := registerPeerWithTD(t, handler.peers, 1_000_000) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + + syncer.onSyncDone(downloader.ErrPeersUnavailable) + if syncer.peersUnavailableUntil.IsZero() { + t.Fatal("cooldown should be armed after ErrPeersUnavailable") + } + + syncer.onPeerEvent() + if syncer.peersUnavailableUntil.IsZero() { + t.Fatal("a peer event with no peer-set change (e.g. a block announce) must not clear the cooldown") + } + + peer2 := registerPeerWithTD(t, handler.peers, 2_000_000) + if err := handler.downloader.RegisterPeer(peer2.ID(), eth.ETH68, ðPeer{Peer: peer2}); err != nil { + t.Fatal(err) + } + syncer.onPeerEvent() + if !syncer.peersUnavailableUntil.IsZero() { + t.Fatal("a genuine peer-set change must clear the cooldown") + } +} + +func TestChainSyncerLoopRetriesBackedOffPeer(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + peer := registerPeerWithTD(t, handler.peers, 0) + if err := handler.downloader.RegisterPeer(peer.ID(), eth.ETH68, ðPeer{Peer: peer}); err != nil { + t.Fatal(err) + } + setDownloaderPeerBackoff(t, handler.downloader, peer.ID(), 5*time.Millisecond) + + handler.wg.Add(1) + done := make(chan struct{}) + go func() { + handler.chainSync.loop() + close(done) + }() + + deadline := time.Now().Add(2 * time.Second) + for handler.downloader.PeerBackoff(peer.ID()) > 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + close(handler.quitSync) + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("chain syncer did not stop") + } +} + +func TestResettableTimer(t *testing.T) { + rt := newResettableTimer() + + select { + case <-rt.C(): + t.Fatal("fresh timer should not fire") + case <-time.After(20 * time.Millisecond): + } + + rt.reset(10 * time.Millisecond) + select { + case <-rt.C(): + case <-time.After(time.Second): + t.Fatal("reset timer did not fire") + } + rt.markFired() + + rt.reset(10 * time.Millisecond) + rt.stop() + select { + case <-rt.C(): + t.Fatal("stopped timer should not fire") + case <-time.After(30 * time.Millisecond): + } + + rt.reset(0) + select { + case <-rt.C(): + t.Fatal("reset(0) should not arm the timer") + case <-time.After(20 * time.Millisecond): + } + + rt.reset(5 * time.Millisecond) + time.Sleep(25 * time.Millisecond) + rt.markFired() + rt.stop() + select { + case <-rt.C(): + default: + t.Fatal("stop on inactive timer should leave the pending tick untouched") + } +} + +func TestResettableTimerActiveState(t *testing.T) { + rt := newResettableTimer() + if rt.active { + t.Fatal("a fresh timer must be inactive") + } + + rt.reset(time.Hour) + if !rt.active { + t.Fatal("reset with a positive wait must arm the timer") + } + + rt.stop() + if rt.active { + t.Fatal("stop must mark the timer inactive") + } + + rt.reset(time.Hour) + rt.markFired() + if rt.active { + t.Fatal("markFired must mark the timer inactive") + } + rt.stop() +} + +func TestResettableTimerResetDrainsStaleTick(t *testing.T) { + rt := newResettableTimer() + rt.reset(5 * time.Millisecond) + time.Sleep(25 * time.Millisecond) + + rt.reset(time.Hour) + select { + case <-rt.C(): + t.Fatal("reset must drain a stale tick before re-arming the timer") + case <-time.After(40 * time.Millisecond): + } + rt.stop() +} + +func TestChainSyncerOnSyncDone(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + cs := newChainSyncer(handler) + cs.force = time.NewTimer(time.Hour) + defer cs.force.Stop() + cs.doneCh = make(chan error, 1) + cs.forced = true + + cs.onSyncDone(nil) + + if cs.doneCh != nil { + t.Fatal("onSyncDone should clear doneCh") + } + if cs.forced { + t.Fatal("onSyncDone should reset forced to false") + } +} + +func TestChainSyncerOnSyncDoneMergeWarning(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + newSyncer := func() *chainSyncer { + cs := newChainSyncer(handler) + cs.force = time.NewTimer(time.Hour) + t.Cleanup(func() { cs.force.Stop() }) + return cs + } + + t.Run("recent warning is not refreshed", func(t *testing.T) { + cs := newSyncer() + before := time.Now() + cs.warned = before + + cs.onSyncDone(downloader.ErrMergeTransition) + + if !cs.warned.Equal(before) { + t.Fatalf("recent warning timestamp should be untouched: have %v, want %v", cs.warned, before) + } + }) + + t.Run("stale warning is refreshed", func(t *testing.T) { + cs := newSyncer() + before := time.Now().Add(-11 * time.Second) + cs.warned = before + + cs.onSyncDone(downloader.ErrMergeTransition) + + if !cs.warned.After(before) { + t.Fatalf("stale warning timestamp should be refreshed: have %v, want after %v", cs.warned, before) + } + }) + + t.Run("non-merge error never warns", func(t *testing.T) { + cs := newSyncer() + before := time.Now().Add(-11 * time.Second) + cs.warned = before + + cs.onSyncDone(errors.New("some other failure")) + + if !cs.warned.Equal(before) { + t.Fatalf("non-merge error should not touch warning timestamp: have %v, want %v", cs.warned, before) + } + }) +} + +func TestChainSyncerShutdownReturnsWithoutPendingSync(t *testing.T) { + handler, cleanup := newChainSyncerTestHandler(t) + defer cleanup() + + cs := newChainSyncer(handler) + done := make(chan struct{}) + go func() { + cs.shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not return") + } +} + +func newChainSyncerTestHandler(t *testing.T) (*handler, func()) { + t.Helper() + + db := rawdb.NewMemoryDatabase() + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + } + chain, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) + if err != nil { + t.Fatal(err) + } + + handler, err := newHandler(&handlerConfig{ + Database: db, + Chain: chain, + TxPool: newTestTxPool(), + Network: 1, + Sync: downloader.SnapSync, + BloomCache: 1, + }) + if err != nil { + chain.Stop() + db.Close() + t.Fatal(err) + } + return handler, func() { + handler.downloader.Terminate() + chain.Stop() + db.Close() + } +} + +func setDownloaderPeerBackoff(t *testing.T, d *downloader.Downloader, id string, duration time.Duration) { + t.Helper() + + if !d.SetPeerBackoffForTesting(id, duration) { + t.Fatalf("downloader peer %q not found", id) + } +} + // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {