Merge branch 'stx-develop' into test/improve-vm-tests

Author: federico-stacks

changelog.d/marf-snapshot-burnchain.added

@@ -0,0 +1 @@
+Add snapshot copy of canonical burnchain.sqlite tables into squashed output, using the squashed sortition DB as the canonical set.

changelog.d/marf-snapshot-spv.added

@@ -0,0 +1 @@
+Add snapshot copy of Bitcoin SPV headers and complete chain-work intervals up to the squash height into squashed output.

stacks-profiler/src/platform.rs

@@ -105,6 +105,7 @@ mod linux {
 mod windows {
     /// Win32 FILETIME (100-nanosecond intervals).
     #[repr(C)]
+    #[allow(clippy::upper_case_acronyms)]
     struct FILETIME {
         low: u32,
         high: u32,

stackslib/src/burnchains/bitcoin/spv.rs

@@ -54,6 +54,14 @@ pub const BITCOIN_GENESIS_BLOCK_HASH_REGTEST: &str =
 pub const BLOCK_DIFFICULTY_CHUNK_SIZE: u64 = 2016;
 const BLOCK_DIFFICULTY_INTERVAL: u32 = 14 * 24 * 60 * 60; // two weeks, in seconds
 
+/// Number of complete difficulty intervals at `burn_height`: interval `k`
+/// is complete iff its last header height,
+/// `(k + 1) * BLOCK_DIFFICULTY_CHUNK_SIZE - 1`, is at or below
+/// `burn_height`.
+pub fn num_complete_chain_work_intervals(burn_height: u64) -> u64 {
+    burn_height.saturating_add(1) / BLOCK_DIFFICULTY_CHUNK_SIZE
+}
+
 pub const SPV_DB_VERSION: &str = "3";
 
 const SPV_INITIAL_SCHEMA: &[&str] = &[
@@ -743,7 +751,7 @@ impl SpvClient {
         header: BlockHeader,
         height: u64,
     ) -> Result<(), btc_error> {
-        let sql = "INSERT OR REPLACE INTO headers 
+        let sql = "INSERT OR REPLACE INTO headers
         (version, prev_blockhash, merkle_root, time, bits, nonce, height, hash)
         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)";
         let args = params![
@@ -915,6 +923,22 @@ impl SpvClient {
         self.write_block_headers(height, headers)
     }
 
+    /// Insert a `chain_work` interval row directly (test fixtures only;
+    /// production rows go through [`SpvClient::update_chain_work`]).
+    #[cfg(test)]
+    pub fn test_insert_chain_work(
+        conn: &DBConn,
+        interval: u64,
+        work: &str,
+    ) -> Result<(), btc_error> {
+        conn.execute(
+            "INSERT INTO chain_work (interval, work) VALUES (?1, ?2)",
+            params![u64_to_sql(interval)?, work],
+        )
+        .map_err(|e| btc_error::from(db_error::SqliteError(e)))?;
+        Ok(())
+    }
+
     /// Insert block headers into the headers DB.
     /// Verify that the first header's parent exists and connects with this header chain, and verify that
     /// the headers are themselves contiguous.
@@ -994,7 +1018,7 @@ impl SpvClient {
             Some(child_header) => {
                 // contiguous?
                 if last_block_header.header.bitcoin_hash() != child_header.header.prev_blockhash {
-                    warn!("Received discontiguous headers at height {}: we have child {:?} ({}), but were given {:?} ({})", 
+                    warn!("Received discontiguous headers at height {}: we have child {:?} ({}), but were given {:?} ({})",
                           end_height, &child_header, child_header.header.bitcoin_hash(), &last_block_header, &last_block_header.header.bitcoin_hash());
                     return Err(btc_error::NoncontiguousHeader);
                 }

stackslib/src/burnchains/db.rs

@@ -945,6 +945,114 @@ impl BurnchainDB {
         }
     }
 
+    /// Count the burn header hashes yielded by `canonical_sql` (a SELECT
+    /// producing one TEXT column) that have no `burnchain_db_block_headers`
+    /// row in `headers_schema`. Both arguments are interpolated into SQL;
+    /// pass only trusted fixed fragments.
+    pub(crate) fn count_canonical_burn_hashes_missing_from(
+        conn: &Connection,
+        headers_schema: &str,
+        canonical_sql: &str,
+    ) -> Result<u64, DBError> {
+        conn.query_row(
+            &format!(
+                "SELECT COUNT(*) FROM ({canonical_sql}) \
+                 WHERE burn_header_hash NOT IN \
+                     (SELECT block_hash FROM {headers_schema}.burnchain_db_block_headers)"
+            ),
+            NO_PARAMS,
+            |row| row.get(0),
+        )
+        .map_err(DBError::from)
+    }
+}
+
+// Raw-row test fixture writers. Each helper owns its table's column
+// list so fixtures can't drift from the schema; values are raw
+// TEXT/ints because fixtures use readable labels, not valid hashes
+// (which the typed write paths would reject).
+#[cfg(test)]
+impl BurnchainDB {
+    /// Insert a `burnchain_db_block_headers` row with no transactions.
+    pub fn test_insert_block_header_row(
+        conn: &Connection,
+        block_height: u64,
+        block_hash: &str,
+        parent_block_hash: &str,
+    ) -> Result<(), DBError> {
+        conn.execute(
+            "INSERT INTO burnchain_db_block_headers \
+             (block_height, block_hash, parent_block_hash, num_txs, timestamp) \
+             VALUES (?1, ?2, ?3, 0, 0)",
+            params![u64_to_sql(block_height)?, block_hash, parent_block_hash],
+        )?;
+        Ok(())
+    }
+
+    /// Insert a `burnchain_db_block_ops` row with an opaque op payload.
+    pub fn test_insert_block_ops_row(
+        conn: &Connection,
+        block_hash: &str,
+        op: &str,
+        txid: &str,
+    ) -> rusqlite::Result<()> {
+        conn.execute(
+            "INSERT INTO burnchain_db_block_ops (block_hash, op, txid) VALUES (?1, ?2, ?3)",
+            params![block_hash, op, txid],
+        )?;
+        Ok(())
+    }
+
+    /// Insert a minimal `block_commit_metadata` row.
+    pub fn test_insert_block_commit_metadata_row(
+        conn: &Connection,
+        burn_block_hash: &str,
+        txid: &str,
+        block_height: u64,
+        anchor_block: Option<u64>,
+    ) -> Result<(), DBError> {
+        conn.execute(
+            "INSERT INTO block_commit_metadata \
+             (burn_block_hash, txid, block_height, vtxindex, anchor_block, \
+              anchor_block_descendant) \
+             VALUES (?1, ?2, ?3, 0, ?4, NULL)",
+            params![
+                burn_block_hash,
+                txid,
+                u64_to_sql(block_height)?,
+                opt_u64_to_sql(anchor_block)?,
+            ],
+        )?;
+        Ok(())
+    }
+
+    /// Insert an `anchor_blocks` row.
+    pub fn test_insert_anchor_block_row(
+        conn: &Connection,
+        reward_cycle: u64,
+    ) -> Result<(), DBError> {
+        conn.execute(
+            "INSERT INTO anchor_blocks (reward_cycle) VALUES (?1)",
+            params![u64_to_sql(reward_cycle)?],
+        )?;
+        Ok(())
+    }
+
+    /// Insert an `overrides` row.
+    pub fn test_insert_override_row(
+        conn: &Connection,
+        reward_cycle: u64,
+        affirmation_map: &str,
+    ) -> Result<(), DBError> {
+        conn.execute(
+            "INSERT INTO overrides (reward_cycle, affirmation_map) VALUES (?1, ?2)",
+            params![u64_to_sql(reward_cycle)?, affirmation_map],
+        )?;
+        Ok(())
+    }
+}
+
+impl BurnchainDB {
     // do NOT call directly; only call directly in tests.
     // This is only `pub` because the tests for it live in a different file.
     pub fn store_new_burnchain_block_ops_unchecked(

stackslib/src/chainstate/burn/db/sortdb.rs

@@ -4554,10 +4554,21 @@ impl SortitionDB {
         query_row(conn, qry, NO_PARAMS).map(|opt| opt.expect("CORRUPTION: No burnchain tips"))
     }
 
+    /// Get the distinct burn header hashes of all snapshots, forks included.
+    /// Only on a squashed sortition DB is this exactly the canonical
+    /// burnchain.
+    pub fn get_all_snapshot_burn_header_hashes(
+        conn: &Connection,
+    ) -> Result<Vec<BurnchainHeaderHash>, db_error> {
+        let mut stmt = conn.prepare("SELECT DISTINCT burn_header_hash FROM snapshots")?;
+        let rows = stmt.query_map(NO_PARAMS, |row| row.get(0))?;
+        rows.collect::<Result<Vec<_>, _>>().map_err(db_error::from)
+    }
+
     /// Get the burn header hash of the snapshot with the given sortition
     /// ID, or `None` if no such snapshot exists. Returns the raw stored
     /// TEXT so callers can preserve the value byte-for-byte.
-    pub(crate) fn get_snapshot_burn_header_hash(
+    pub fn get_snapshot_burn_header_hash(
         conn: &Connection,
         sortition_id: &SortitionId,
     ) -> Result<Option<String>, db_error> {