wallet: Do not download cfilters before birthday.

Especially for spv wallets with a birthday, cfilters for blocks before
the birthday are not used for anything as there can be no transactions
before the wallet existed. Do not download them but check if we need
them whenever doing a rescan from height.

Author: JoeGruffins

chain/sync.go

@@ -24,6 +24,7 @@ import (
 	"github.com/decred/dcrd/blockchain/stake/v5"
 	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrd/crypto/blake256"
+	"github.com/decred/dcrd/gcs/v4"
 	"github.com/decred/dcrd/mixing/mixpool"
 	"github.com/decred/dcrd/wire"
 	"github.com/jrick/wsrpc/v2"
@@ -228,6 +229,29 @@ func (s *Syncer) getHeaders(ctx context.Context) error {
 		return err
 	}
 
+	birthday, err := s.wallet.BirthState(ctx)
+	if err != nil {
+		return err
+	}
+	// If a birthday is set, it will likely not be found yet. Watch for it
+	// and do not download cfilters before it.
+	var afterBirthday func(h *wire.BlockHeader) bool
+	switch {
+	case birthday == nil:
+		afterBirthday = func(*wire.BlockHeader) bool {
+			return true
+		}
+	case birthday.SetFromTime:
+		afterBirthday = func(h *wire.BlockHeader) bool {
+			return h.Timestamp.After(birthday.Time)
+		}
+	default:
+		// Birthday is setting from height or was already found.
+		afterBirthday = func(h *wire.BlockHeader) bool {
+			return h.Height >= birthday.Height
+		}
+	}
+
 	startedSynced := s.atomicWalletSynced.Load() == 1
 
 	cnet := s.wallet.ChainParams().Net
@@ -261,15 +285,22 @@ func (s *Syncer) getHeaders(ctx context.Context) error {
 			g.Go(func() error {
 				header := headers[i]
 				hash := header.BlockHash()
-				filter, proofIndex, proof, err := s.rpc.CFilterV2(ctx, &hash)
-				if err != nil {
-					return err
-				}
-
-				err = validate.CFilterV2HeaderCommitment(cnet, header,
-					filter, proofIndex, proof)
-				if err != nil {
-					return err
+				var filter *gcs.FilterV2
+				if afterBirthday(header) {
+					var (
+						proofIndex uint32
+						proof      []chainhash.Hash
+					)
+					filter, proofIndex, proof, err = s.rpc.CFilterV2(ctx, &hash)
+					if err != nil {
+						return err
+					}
+
+					err = validate.CFilterV2HeaderCommitment(cnet, header,
+						filter, proofIndex, proof)
+					if err != nil {
+						return err
+					}
 				}
 
 				nodes[i] = wallet.NewBlockNode(header, &hash, filter)

rpc/documentation/api.md

@@ -1109,6 +1109,8 @@ ___
 The `ImportPrivateKey` method imports a private key in Wallet Import Format
 (WIF) encoding to a wallet account.  A rescan may optionally be started to
 search for transactions involving the private key's associated payment address.
+If the private key deals with transactions before the wallet birthday, if set,
+a rescan must be performed to download missing cfilters.
 
 **Request:** `ImportPrivateKeyRequest`
 
@@ -1144,7 +1146,9 @@ ___
 
 The `ImportScript` method imports a script into the wallet.  A rescan may
 optionally be started to search for transactions involving the script, either
-as an output or in a P2SH input.
+as an output or in a P2SH input. If the script deals with transactions before
+the wallet birthday, if set, a rescan must be performed to download missing
+cfilters.
 
 **Request:** `ImportScriptRequest`
 
@@ -1190,7 +1194,9 @@ seed for a hierarchical deterministic private key that is imported into the
 wallet with the supplied name and locked with the supplied password. Addresses
 derived from this account MUST NOT be sent any funds. They are solely for the
 use of creating stake submission scripts. A rescan may optionally be started to
-search for tickets using submission scripts derived from this account.
+search for tickets using submission scripts derived from this account. If tickets
+would exist before the wallet birthday, if set, a rescan must be performed to
+download missing cfilters.
 
 **Request:** `ImportVotingAccountFromSeedRequest`
 
@@ -2689,7 +2695,10 @@ or account must be unlocked.
 #### `BirthBlock`
 
 The `BirthBlock` method returns the wallets birthday block if set. Rescans
-should generally be started from after this block.
+should generally be started from after this block. If a birthday is set cfilters
+from before the birthday may not be downloaded. A rescan from height will move
+the birthday to the rescan height and download all missing cfilters from that
+height.
 
 **Request:** `BirthBlockRequest`
 

spv/sync.go

@@ -1660,6 +1660,29 @@ func (s *Syncer) initialSyncHeaders(ctx context.Context) error {
 		return res
 	}
 
+	birthday, err := s.wallet.BirthState(ctx)
+	if err != nil {
+		return err
+	}
+	// If a birthday is set, it will likely not be found yet. Watch for it
+	// and do not download cfilters before it.
+	var afterBirthday func(h *wire.BlockHeader) bool
+	switch {
+	case birthday == nil:
+		afterBirthday = func(*wire.BlockHeader) bool {
+			return true
+		}
+	case birthday.SetFromTime:
+		afterBirthday = func(h *wire.BlockHeader) bool {
+			return h.Timestamp.After(birthday.Time)
+		}
+	default:
+		// Birthday is setting from height or was already found.
+		afterBirthday = func(h *wire.BlockHeader) bool {
+			return h.Height >= birthday.Height
+		}
+	}
+
 	// Stage 1: fetch headers.
 	headersChan := make(chan *headersBatch)
 	g.Go(func() error {
@@ -1735,6 +1758,9 @@ func (s *Syncer) initialSyncHeaders(ctx context.Context) error {
 			s.sidechainMu.Lock()
 			var missingCfilter []*wallet.BlockNode
 			for i := range batch.bestChain {
+				if !afterBirthday(batch.bestChain[i].Header) {
+					continue
+				}
 				if batch.bestChain[i].FilterV2 == nil {
 					missingCfilter = batch.bestChain[i:]
 					break

wallet/rescan.go

@@ -386,8 +386,47 @@ func (w *Wallet) Rescan(ctx context.Context, n NetworkBackend, startHash *chainh
 func (w *Wallet) RescanFromHeight(ctx context.Context, n NetworkBackend, startHeight int32) error {
 	const op errors.Op = "wallet.RescanFromHeight"
 
+	bs, err := w.BirthState(ctx)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	if bs != nil && int32(bs.Height) > startHeight {
+		// If our birthday is before the rescan height, we may
+		// not have the cfilters needed. Set birthday to the rescan
+		// height and download the filters. This may take some time
+		// depending on network conditions and amount of filters missing.
+		bs := &udb.BirthdayState{
+			SetFromHeight: true,
+			Height:        uint32(startHeight),
+		}
+		if err := w.SetBirthStateAndScan(ctx, bs); err != nil {
+			return errors.E(op, err)
+		}
+		fetchMissing := true
+		if err := walletdb.Update(ctx, w.db, func(dbtx walletdb.ReadWriteTx) error {
+			if _, err := udb.MissingCFiltersHeight(dbtx, startHeight); err != nil {
+				// errors.NotExist is returned if we already have all filters
+				// from start height. If we have them there is no need to
+				// fetch them again.
+				if errors.Is(err, errors.NotExist) {
+					fetchMissing = false
+					return nil
+				}
+				return errors.E(op, err)
+			}
+			return w.txStore.SetMissingMainChainCFilters(dbtx, false)
+		}); err != nil {
+			return errors.E(op, err)
+		}
+		if fetchMissing {
+			if err := w.FetchMissingCFilters(ctx, n); err != nil {
+				return errors.E(op, err)
+			}
+		}
+	}
+
 	var startHash chainhash.Hash
-	err := walletdb.View(ctx, w.db, func(tx walletdb.ReadTx) error {
+	err = walletdb.View(ctx, w.db, func(tx walletdb.ReadTx) error {
 		txmgrNs := tx.ReadBucket(wtxmgrNamespaceKey)
 		var err error
 		startHash, err = w.txStore.GetMainChainBlockHashForHeight(

wallet/udb/txmined.go

@@ -198,8 +198,8 @@ func (s *Store) MainChainTip(dbtx walletdb.ReadTx) (chainhash.Hash, int32) {
 // If the block is already inserted and part of the main chain, an errors.Exist
 // error is returned.
 //
-// The main chain tip may not be extended unless compact filters have been saved
-// for all existing main chain blocks.
+// The main chain may be extended without cfilters if this block is before the
+// wallet birthday. If the filter is nil it will not be saved to the database.
 func (s *Store) ExtendMainChain(ns walletdb.ReadWriteBucket, header *wire.BlockHeader, blockHash *chainhash.Hash, f *gcs2.FilterV2) error {
 	height := int32(header.Height)
 	if height < 1 {
@@ -266,9 +266,12 @@ func (s *Store) ExtendMainChain(ns walletdb.ReadWriteBucket, header *wire.BlockH
 		return err
 	}
 
-	// Save the compact filter.
-	bcf2Key := blockcf2.Key(&header.MerkleRoot)
-	return putRawCFilter(ns, blockHash[:], valueRawCFilter2(bcf2Key, f.Bytes()))
+	// Save the compact filter if we have it.
+	if f != nil {
+		bcf2Key := blockcf2.Key(&header.MerkleRoot)
+		return putRawCFilter(ns, blockHash[:], valueRawCFilter2(bcf2Key, f.Bytes()))
+	}
+	return nil
 }
 
 // ProcessedTxsBlockMarker returns the hash of the block which records the last
@@ -402,19 +405,37 @@ func (s *Store) IsMissingMainChainCFilters(dbtx walletdb.ReadTx) bool {
 	return len(v) != 1 || v[0] == 0
 }
 
+// SetMissingMainChainCFilters sets whether we have all of the main chain
+// cfilters. Should be used to set missing to false if the wallet birthday is
+// moved back in time.
+func (s *Store) SetMissingMainChainCFilters(dbtx walletdb.ReadWriteTx, have bool) error {
+	haveB := []byte{0}
+	if have {
+		haveB = []byte{1}
+	}
+	err := dbtx.ReadWriteBucket(wtxmgrBucketKey).Put(rootHaveCFilters, haveB)
+	if err != nil {
+		return errors.E(errors.IO, err)
+	}
+	return nil
+}
+
 // MissingCFiltersHeight returns the first main chain block height
 // with a missing cfilter.  Errors with NotExist when all main chain
 // blocks record cfilters.
-func (s *Store) MissingCFiltersHeight(dbtx walletdb.ReadTx) (int32, error) {
+func MissingCFiltersHeight(dbtx walletdb.ReadTx, fromHeight int32) (int32, error) {
 	ns := dbtx.ReadBucket(wtxmgrBucketKey)
 	c := ns.NestedReadBucket(bucketBlocks).ReadCursor()
 	defer c.Close()
-	for k, v := c.First(); k != nil; k, v = c.Next() {
+	for k, v := c.Seek(keyBlockRecord(fromHeight)); k != nil; k, v = c.Next() {
 		hash := extractRawBlockRecordHash(v)
 		_, _, err := fetchRawCFilter2(ns, hash)
-		if errors.Is(err, errors.NotExist) {
-			height := int32(byteOrder.Uint32(k))
-			return height, nil
+		if err != nil {
+			if errors.Is(err, errors.NotExist) {
+				height := int32(byteOrder.Uint32(k))
+				return height, nil
+			}
+			return 0, errors.E(errors.IO, err)
 		}
 	}
 	return 0, errors.E(errors.NotExist)
@@ -442,42 +463,37 @@ func (s *Store) InsertMissingCFilters(dbtx walletdb.ReadWriteTx, blockHashes []*
 	}
 
 	for i, blockHash := range blockHashes {
-		// Ensure that blockHashes are ordered and that all previous cfilters in the
-		// main chain are known.
+		// Ensure that blockHashes are ordered.
+		header := existsBlockHeader(ns, blockHash[:])
+		if header == nil {
+			return errors.E(errors.NotExist, errors.Errorf("missing header for block %v", blockHash))
+		}
 		ok := i == 0 && *blockHash == s.chainParams.GenesisHash
-		var bcf2Key [gcs2.KeySize]byte
 		if !ok {
-			header := existsBlockHeader(ns, blockHash[:])
-			if header == nil {
-				return errors.E(errors.NotExist, errors.Errorf("missing header for block %v", blockHash))
-			}
 			parentHash := extractBlockHeaderParentHash(header)
-			merkleRoot := extractBlockHeaderMerkleRoot(header)
-			merkleRootHash, err := chainhash.NewHash(merkleRoot)
-			if err != nil {
-				return errors.E(errors.Invalid, errors.Errorf("invalid stored header %v", blockHash))
-			}
-			bcf2Key = blockcf2.Key(merkleRootHash)
-			if i == 0 {
-				_, _, err := fetchRawCFilter2(ns, parentHash)
-				ok = err == nil
-			} else {
-				ok = bytes.Equal(parentHash, blockHashes[i-1][:])
+			if i != 0 {
+				if !bytes.Equal(parentHash, blockHashes[i-1][:]) {
+					return errors.E(errors.Invalid, "block hashes are not ordered")
+				}
 			}
 		}
-		if !ok {
-			return errors.E(errors.Invalid, "block hashes are not ordered or previous cfilters are missing")
-		}
 
 		// Record cfilter for this block
-		err := putRawCFilter(ns, blockHash[:], valueRawCFilter2(bcf2Key, filters[i].Bytes()))
+		merkleRoot := extractBlockHeaderMerkleRoot(header)
+		merkleRootHash, err := chainhash.NewHash(merkleRoot)
+		if err != nil {
+			return errors.E(errors.Invalid, errors.Errorf("invalid stored header %v", blockHash))
+		}
+		bcf2Key := blockcf2.Key(merkleRootHash)
+		err = putRawCFilter(ns, blockHash[:], valueRawCFilter2(bcf2Key, filters[i].Bytes()))
 		if err != nil {
 			return err
 		}
 	}
 
 	// Mark all main chain cfilters as saved if the last block hash is the main
-	// chain tip.
+	// chain tip. Even if this is not the head block, all cfilters may be saved
+	// at this point. The caller may need to check and set rootHaveCFilters.
 	tip, _ := s.MainChainTip(dbtx)
 	if bytes.Equal(tip[:], blockHashes[len(blockHashes)-1][:]) {
 		err := ns.Put(rootHaveCFilters, []byte{1})