rawdb: add freezer safety margin to prevent data loss after unclean shutdown

Add a safety margin (freezerBatchLimit = 30,000 blocks) before deleting
frozen blocks from LevelDB. This ensures that after an unclean shutdown,
when repair() truncates unflushed freezer writes, the data still exists
in LevelDB for re-freezing. Critical for L2 nodes that cannot re-download
blocks from peers.

Refuse to start when data loss exceeds the safety margin, with a clear
error message telling the operator to restore from a snapshot.

Add edge case tests for the safety margin cleanup logic.

Disk overhead: ~30K blocks duplicated temporarily (~30-600 MB).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: joshuacolvin0

core/rawdb/accessors_chain.go

@@ -258,6 +258,26 @@ func WriteLastPivotNumber(db ethdb.KeyValueWriter, pivot uint64) {
 	}
 }
 
+// ReadFreezerCleanupTail retrieves the block number up to which frozen blocks
+// have been cleaned up (deleted) from the key-value database.
+func ReadFreezerCleanupTail(db ethdb.KeyValueReader) (uint64, bool) {
+	data, _ := db.Get(freezerCleanupTailKey)
+	if len(data) != 8 {
+		return 0, false
+	}
+	return binary.BigEndian.Uint64(data), true
+}
+
+// WriteFreezerCleanupTail stores the block number up to which frozen blocks
+// have been cleaned up from the key-value database.
+func WriteFreezerCleanupTail(db ethdb.KeyValueWriter, number uint64) {
+	var buf [8]byte
+	binary.BigEndian.PutUint64(buf[:], number)
+	if err := db.Put(freezerCleanupTailKey, buf[:]); err != nil {
+		log.Crit("Failed to store freezer cleanup tail", "err", err)
+	}
+}
+
 // ReadTxIndexTail retrieves the number of oldest indexed block
 // whose transaction indices has been indexed.
 func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 {

core/rawdb/chain_freezer.go

@@ -40,6 +40,21 @@ const (
 	freezerBatchLimit = 30000
 )
 
+// freezerCleanupMargin is the number of blocks to keep in the key-value
+// database after they have been frozen into the ancient store. This acts
+// as a safety margin: after an unclean shutdown, repair() truncates
+// unflushed freezer writes. The data still exists in LevelDB and can be
+// re-frozen. Without this margin, a crash could leave blocks missing
+// from both stores, making the node unable to start (especially for L2
+// nodes without peers).
+//
+// Set to freezerBatchLimit because each freeze cycle writes at most
+// freezerBatchLimit blocks to the ancient store. If the node crashes
+// before these writes are fsynced (SyncAncient), repair() may truncate
+// the unflushed entries on restart. Keeping this many blocks in LevelDB
+// ensures the truncated data can be re-frozen.
+var freezerCleanupMargin uint64 = freezerBatchLimit
+
 // chainFreezer is a wrapper of chain ancient store with additional chain freezing
 // feature. The background thread will keep moving ancient chain segments from
 // key-value database to flat files for saving space on live database.
@@ -220,13 +235,37 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 		if err := f.SyncAncient(); err != nil {
 			log.Crit("Failed to flush frozen tables", "err", err)
 		}
-		// Wipe out all data from the active database
+		// Delete blocks from LevelDB that are safely behind the freeze point.
+		// Recently-frozen blocks are kept in both stores so that after an
+		// unclean shutdown, repair()-truncated entries can be re-frozen.
+		frozen, _ = f.Ancients() // no error will occur, safe to ignore; reload after freezeRange
+		cleanupStart := uint64(1) // always keep genesis
+		if prev, ok := ReadFreezerCleanupTail(db); ok && prev > 1 {
+			cleanupStart = prev
+		} else if frozen > freezerCleanupMargin {
+			// First run with safety margin: prior code already deleted frozen
+			// blocks from LevelDB immediately, so skip ahead to avoid pointless
+			// reads of already-deleted blocks.
+			cleanupStart = frozen - freezerCleanupMargin
+			WriteFreezerCleanupTail(db, cleanupStart)
+		}
+		cleanupLimit := uint64(0)
+		if frozen > freezerCleanupMargin {
+			cleanupLimit = frozen - freezerCleanupMargin
+		}
+		// Cap per-cycle work to avoid stalling when cleanup has a large backlog
+		// (e.g., first run after upgrade). During catch-up the node keeps more
+		// blocks in LevelDB than ultimately needed (extra disk, not extra latency).
+		if cleanupLimit > cleanupStart+freezerBatchLimit {
+			cleanupLimit = cleanupStart + freezerBatchLimit
+		}
+		// Wipe out canonical data from the active database.
 		batch := db.NewBatch()
-		for i := 0; i < len(ancients); i++ {
-			// Always keep the genesis block in active database
-			if first+uint64(i) != 0 {
-				DeleteBlockWithoutNumber(batch, ancients[i], first+uint64(i))
-				DeleteCanonicalHash(batch, first+uint64(i))
+		for number := cleanupStart; number < cleanupLimit; number++ {
+			hash := ReadCanonicalHash(nfdb, number)
+			if hash != (common.Hash{}) {
+				DeleteBlockWithoutNumber(batch, hash, number)
+				DeleteCanonicalHash(batch, number)
 			}
 		}
 		if err := batch.Write(); err != nil {
@@ -236,15 +275,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 
 		// Wipe out side chains also and track dangling side chains
 		var dangling []common.Hash
-		frozen, _ = f.Ancients() // Needs reload after during freezeRange
-		for number := first; number < frozen; number++ {
-			// Always keep the genesis block in active database
-			if number != 0 {
-				dangling = ReadAllHashes(db, number)
-				for _, hash := range dangling {
-					log.Trace("Deleting side chain", "number", number, "hash", hash)
-					DeleteBlock(batch, hash, number)
-				}
+		for number := cleanupStart; number < cleanupLimit; number++ {
+			dangling = ReadAllHashes(db, number)
+			for _, hash := range dangling {
+				log.Trace("Deleting side chain", "number", number, "hash", hash)
+				DeleteBlock(batch, hash, number)
 			}
 		}
 		if err := batch.Write(); err != nil {
@@ -253,37 +288,38 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 		batch.Reset()
 
 		// Step into the future and delete any dangling side chains
-		if frozen > 0 {
-			tip := frozen
-			for len(dangling) > 0 {
-				drop := make(map[common.Hash]struct{})
-				for _, hash := range dangling {
-					log.Debug("Dangling parent from Freezer", "number", tip-1, "hash", hash)
-					drop[hash] = struct{}{}
+		tip := cleanupLimit
+		for len(dangling) > 0 {
+			drop := make(map[common.Hash]struct{})
+			for _, hash := range dangling {
+				log.Debug("Dangling parent from Freezer", "number", tip-1, "hash", hash)
+				drop[hash] = struct{}{}
+			}
+			children := ReadAllHashes(db, tip)
+			for i := 0; i < len(children); i++ {
+				// Dig up the child and ensure it's dangling
+				child := ReadHeader(nfdb, children[i], tip)
+				if child == nil {
+					log.Error("Missing dangling header", "number", tip, "hash", children[i])
+					continue
 				}
-				children := ReadAllHashes(db, tip)
-				for i := 0; i < len(children); i++ {
-					// Dig up the child and ensure it's dangling
-					child := ReadHeader(nfdb, children[i], tip)
-					if child == nil {
-						log.Error("Missing dangling header", "number", tip, "hash", children[i])
-						continue
-					}
-					if _, ok := drop[child.ParentHash]; !ok {
-						children = append(children[:i], children[i+1:]...)
-						i--
-						continue
-					}
-					// Delete all block data associated with the child
-					log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash)
-					DeleteBlock(batch, children[i], tip)
+				if _, ok := drop[child.ParentHash]; !ok {
+					children = append(children[:i], children[i+1:]...)
+					i--
+					continue
 				}
-				dangling = children
-				tip++
-			}
-			if err := batch.Write(); err != nil {
-				log.Crit("Failed to delete dangling side blocks", "err", err)
+				// Delete all block data associated with the child
+				log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash)
+				DeleteBlock(batch, children[i], tip)
 			}
+			dangling = children
+			tip++
+		}
+		if err := batch.Write(); err != nil {
+			log.Crit("Failed to delete dangling side blocks", "err", err)
+		}
+		if cleanupStart < cleanupLimit {
+			WriteFreezerCleanupTail(db, cleanupLimit)
 		}
 
 		// Log something friendly for the user