diff --git a/stackslib/src/burnchains/db.rs b/stackslib/src/burnchains/db.rs index 6d3962c57a..36fc86fb2b 100644 --- a/stackslib/src/burnchains/db.rs +++ b/stackslib/src/burnchains/db.rs @@ -943,27 +943,6 @@ impl BurnchainDB { Ok(None) } } - - /// Count the burn header hashes yielded by `canonical_sql` (a SELECT - /// producing one TEXT column) that have no `burnchain_db_block_headers` - /// row in `headers_schema`. Both arguments are interpolated into SQL; - /// pass only trusted fixed fragments. - pub(crate) fn count_canonical_burn_hashes_missing_from( - conn: &Connection, - headers_schema: &str, - canonical_sql: &str, - ) -> Result { - conn.query_row( - &format!( - "SELECT COUNT(*) FROM ({canonical_sql}) \ - WHERE burn_header_hash NOT IN \ - (SELECT block_hash FROM {headers_schema}.burnchain_db_block_headers)" - ), - NO_PARAMS, - |row| row.get(0), - ) - .map_err(DBError::from) - } } // Raw-row test fixture writers. Each helper owns its table's column diff --git a/stackslib/src/chainstate/burn/db/sortdb.rs b/stackslib/src/chainstate/burn/db/sortdb.rs index 6cf4d4b17f..d3fcf34eba 100644 --- a/stackslib/src/chainstate/burn/db/sortdb.rs +++ b/stackslib/src/chainstate/burn/db/sortdb.rs @@ -4614,106 +4614,6 @@ impl SortitionDB { query_row(conn, qry, NO_PARAMS).map(|opt| opt.expect("CORRUPTION: No burnchain tips")) } - /// Get the distinct burn header hashes of all snapshots, forks included. - /// Only on a squashed sortition DB is this exactly the canonical - /// burnchain. - pub fn get_all_snapshot_burn_header_hashes( - conn: &Connection, - ) -> Result, db_error> { - let mut stmt = conn.prepare("SELECT DISTINCT burn_header_hash FROM snapshots")?; - let rows = stmt.query_map(NO_PARAMS, |row| row.get(0))?; - rows.collect::, _>>().map_err(db_error::from) - } - - /// Get the burn header hash of the snapshot with the given sortition - /// ID, or `None` if no such snapshot exists. Returns the raw stored - /// TEXT so callers can preserve the value byte-for-byte. - pub fn get_snapshot_burn_header_hash( - conn: &Connection, - sortition_id: &SortitionId, - ) -> Result, db_error> { - conn.prepare_cached("SELECT burn_header_hash FROM snapshots WHERE sortition_id = ?1")? - .query_row(params![sortition_id], |row| row.get(0)) - .optional() - .map_err(db_error::from) - } - - /// Source-projection SQL for copying a Stacks-tip memo table - /// (`stacks_chain_tips`, or `stacks_chain_tips_by_burn_view` with - /// `include_burn_view`) from the schema `from`, keeping rows whose - /// `sortition_id` is in `sortition_id_sql`. With a `boundary`, memo - /// rows above its Stacks height are rewritten down to the anchor (see - /// [`SortitionTipCopyBoundary`]) - pub(crate) fn stacks_tip_memo_copy_sql( - table: &str, - from: &str, - sortition_id_sql: &str, - include_burn_view: bool, - boundary: Option<&SortitionTipCopyBoundary>, - ) -> String { - let Some(boundary) = boundary else { - return format!( - "SELECT * FROM {from}.{table} WHERE sortition_id IN ({sortition_id_sql})" - ); - }; - let max_height = boundary.max_stacks_height; - let anchor_height = boundary.anchor_block_height; - let anchor_ch = &boundary.anchor_consensus_hash; - let anchor_bhh = &boundary.anchor_block_hash; - let anchor_burn_view_ch = &boundary.anchor_burn_view_consensus_hash; - let burn_view_column = if include_burn_view { - format!( - "CASE WHEN block_height > {max_height} THEN '{anchor_burn_view_ch}' \ - ELSE burn_view_consensus_hash END, " - ) - } else { - String::new() - }; - let anchor_match = if include_burn_view { - format!( - "(consensus_hash = '{anchor_ch}' \ - AND burn_view_consensus_hash = '{anchor_burn_view_ch}')" - ) - } else { - format!("consensus_hash = '{anchor_ch}'") - }; - format!( - "SELECT sortition_id, \ - CASE WHEN block_height > {max_height} THEN '{anchor_ch}' ELSE consensus_hash END, \ - {burn_view_column}\ - CASE WHEN block_height > {max_height} THEN '{anchor_bhh}' ELSE block_hash END, \ - CASE WHEN block_height > {max_height} THEN {anchor_height} ELSE block_height END \ - FROM {from}.{table} \ - WHERE sortition_id IN ({sortition_id_sql}) \ - AND (block_height <= {max_height} OR {anchor_match})" - ) - } - - /// Whether every Stacks-tip memo row (in both memo tables) sits at or - /// below the boundary's Stacks height. A `None` boundary trivially - /// passes. Validation counterpart of - /// [`Self::stacks_tip_memo_copy_sql`]'s height rewrite. - pub(crate) fn stacks_tip_memos_within_boundary( - conn: &Connection, - boundary: Option<&SortitionTipCopyBoundary>, - ) -> Result { - let Some(boundary) = boundary else { - return Ok(true); - }; - let max_height = u64_to_sql(boundary.max_stacks_height)?; - // Check if ANY above-boundary memo row exists. - conn.query_row( - "SELECT NOT EXISTS( \ - SELECT 1 FROM stacks_chain_tips WHERE block_height > ?1 \ - UNION ALL \ - SELECT 1 FROM stacks_chain_tips_by_burn_view WHERE block_height > ?1 \ - )", - params![max_height], - |row| row.get(0), - ) - .map_err(db_error::from) - } - /// Get the canonical burn chain tip -- the tip of the longest burn chain we know about. /// Break ties deterministically by ordering on burnchain block hash. pub fn get_canonical_chain_tip_bhh(conn: &Connection) -> Result { diff --git a/stackslib/src/chainstate/stacks/db/snapshot/blocks.rs b/stackslib/src/chainstate/stacks/db/snapshot/blocks.rs index ba3c97fe09..3bf99f347d 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/blocks.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/blocks.rs @@ -17,12 +17,13 @@ use std::collections::HashSet; use std::fs; use std::path::Path; +use rusqlite::types::Value; use rusqlite::{params, Connection, OpenFlags}; use stacks_common::types::chainstate::{BlockHeaderHash, ConsensusHash, StacksBlockId}; use super::common::{ - assert_source_schema, clone_schemas_from_source, copied_rows, execute_copy_specs, - with_offline_write_session, TableCopySpec, + classify_hint, clone_schemas_from_source, copied_rows, execute_copy_specs, + with_offline_write_session, DbSnapshotSpec, NoBind, TableCopySpec, }; use crate::chainstate::stacks::db::StacksChainState; use crate::chainstate::stacks::index::Error; @@ -53,26 +54,36 @@ pub struct NakamotoBlockCopyStats { pub total_blob_bytes: u64, } -/// Tables copied from the source Nakamoto staging-blocks DB. The index-side +/// The Nakamoto staging (`nakamoto.sqlite`) snapshot spec. The index-side /// staging tables (`staging_microblocks*`) come from the index DB and are /// classified in `index.rs`. -pub(super) const NAKAMOTO_STAGING_TABLES: &[&str] = &["nakamoto_staging_blocks", "db_version"]; +pub(super) struct NakamotoStagingDbSnapshotSpec; -/// Every table the Nakamoto staging snapshot accounts for. nakamoto.sqlite is not -/// MARF-backed, so unlike the other slices no MARF infra tables are exempted. -fn known_nakamoto_staging_tables() -> Vec<&'static str> { - NAKAMOTO_STAGING_TABLES.to_vec() +impl DbSnapshotSpec for NakamotoStagingDbSnapshotSpec { + type Bind = NoBind; + + fn copy_spec_list() -> &'static [TableCopySpec] { + nakamoto_copy_specs() + } + + fn db_label() -> &'static str { + "Nakamoto staging DB" + } + + fn classify_hint() -> &'static str { + classify_hint!(nakamoto_copy_specs) + } + + fn bind_params(&self, bind: NoBind) -> Result, Error> { + match bind {} + } } -/// The blocks snapshot's source-schema guard (see [`assert_source_schema`]); +/// The blocks snapshot's source-schema guard (see +/// [`DbSnapshotSpec::assert_source_classified`]); /// `test_no_unclassified_nakamoto_staging_tables` runs it against a fresh schema. pub(super) fn assert_source_tables_classified(src_conn: &Connection) -> Result<(), Error> { - assert_source_schema( - src_conn, - &known_nakamoto_staging_tables(), - "Nakamoto staging DB", - "NAKAMOTO_STAGING_TABLES in snapshot/blocks.rs", - ) + NakamotoStagingDbSnapshotSpec::assert_source_classified(src_conn) } /// Return the `(sequence, microblock_hash)` rows of processed, @@ -207,23 +218,22 @@ fn populate_microblock_temp_tables( /// Copy specs for the confirmed-microblock tables, filtered by the temp /// tables [`populate_microblock_temp_tables`] builds. -fn microblock_copy_specs() -> Vec { - vec![ - TableCopySpec { - table: "staging_microblocks", - source_sql: "SELECT s.* FROM src.staging_microblocks s \ +fn microblock_copy_specs() -> &'static [TableCopySpec] { + static SPECS: &[TableCopySpec] = &[ + TableCopySpec::sql( + "staging_microblocks", + "SELECT s.* FROM src.staging_microblocks s \ WHERE s.microblock_hash IN (SELECT hash FROM temp.selected_microblocks) \ AND s.index_block_hash IN (SELECT ibh FROM temp.selected_parents) \ - AND s.orphaned = 0" - .into(), - }, - TableCopySpec { - table: "staging_microblocks_data", - source_sql: "SELECT s.* FROM src.staging_microblocks_data s \ - WHERE s.block_hash IN (SELECT hash FROM temp.selected_microblocks)" - .into(), - }, - ] + AND s.orphaned = 0", + ), + TableCopySpec::sql( + "staging_microblocks_data", + "SELECT s.* FROM src.staging_microblocks_data s \ + WHERE s.block_hash IN (SELECT hash FROM temp.selected_microblocks)", + ), + ]; + SPECS } /// Copy confirmed canonical epoch-2 microblock streams into the squashed index. @@ -243,7 +253,7 @@ pub fn copy_confirmed_epoch2_microblocks( if !selected_hashes.is_empty() { populate_microblock_temp_tables(conn, &selected_hashes, &selected_parents)?; - let results = execute_copy_specs(conn, µblock_copy_specs())?; + let results = execute_copy_specs(conn, microblock_copy_specs(), |bind| match bind {})?; stats.microblock_rows_copied = copied_rows(&results, "staging_microblocks"); stats.microblock_bytes_copied = conn.query_row( @@ -322,21 +332,18 @@ pub fn copy_epoch2_block_files( } /// Copy specs for the Nakamoto staging DB. -pub(super) fn nakamoto_copy_specs() -> Vec { - vec![ - TableCopySpec { - table: "db_version", - source_sql: "SELECT * FROM src.db_version".into(), - }, - TableCopySpec { - table: "nakamoto_staging_blocks", - source_sql: "SELECT s.* FROM src.nakamoto_staging_blocks s \ +pub(super) fn nakamoto_copy_specs() -> &'static [TableCopySpec] { + static SPECS: &[TableCopySpec] = &[ + TableCopySpec::sql("db_version", "SELECT * FROM src.db_version"), + TableCopySpec::sql( + "nakamoto_staging_blocks", + "SELECT s.* FROM src.nakamoto_staging_blocks s \ WHERE s.orphaned = 0 \ AND s.index_block_hash IN \ - (SELECT index_block_hash FROM idx.nakamoto_block_headers)" - .into(), - }, - ] + (SELECT index_block_hash FROM idx.nakamoto_block_headers)", + ), + ]; + SPECS } /// Create and populate `nakamoto.sqlite` with canonical `nakamoto_staging_blocks` rows. @@ -369,9 +376,10 @@ pub fn copy_nakamoto_staging_blocks( &[("src", src_nakamoto_path), ("idx", squashed_index_path)], "", |conn| { - clone_schemas_from_source(conn, NAKAMOTO_STAGING_TABLES)?; + let spec = NakamotoStagingDbSnapshotSpec; + clone_schemas_from_source(conn, &NakamotoStagingDbSnapshotSpec::table_names())?; - let results = execute_copy_specs(conn, &nakamoto_copy_specs())?; + let results = spec.run_copy(conn)?; let total_blob_bytes: i64 = conn.query_row( "SELECT COALESCE(SUM(LENGTH(data)), 0) FROM nakamoto_staging_blocks", diff --git a/stackslib/src/chainstate/stacks/db/snapshot/burnchain.rs b/stackslib/src/chainstate/stacks/db/snapshot/burnchain.rs index 04d5654661..cf0524c0f9 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/burnchain.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/burnchain.rs @@ -16,55 +16,83 @@ use std::fs; use std::path::Path; +use rusqlite::types::Value; use rusqlite::{Connection, OpenFlags}; use stacks_common::types::chainstate::BurnchainHeaderHash; +use stacks_common::types::sqlite::NO_PARAMS; use super::common::{ - assert_source_schema, clone_schemas_from_source, copied_rows, execute_copy_specs, - with_offline_write_session, TableCopySpec, + classify_hint, clone_schemas_from_source, copied_rows, with_offline_write_session, + DbSnapshotSpec, NoBind, TableCopySpec, }; -use crate::burnchains::db::BurnchainDB; +use super::sortition::SortitionSnapshotExt; use crate::chainstate::burn::db::sortdb::SortitionDB; use crate::chainstate::stacks::index::Error; -use crate::util_lib::db::sqlite_open; +use crate::util_lib::db::{sqlite_open, Error as DBError}; -/// Tables copied (with canonical-filtered content) into the squashed burnchain DB. -pub(super) const COPIED_TABLES: &[&str] = &[ - "burnchain_db_block_headers", - "burnchain_db_block_ops", - "block_commit_metadata", - "anchor_blocks", - "db_config", -]; +/// Snapshot-only reads over a burnchain DB connection. +pub(crate) trait BurnchainSnapshotExt { + /// Count the burn header hashes yielded by `canonical_sql` that have no + /// `burnchain_db_block_headers` row. Both arguments are interpolated + /// into SQL; pass only trusted fixed fragments. + fn count_canonical_burn_hashes_missing_from( + &self, + headers_schema: &str, + canonical_sql: &str, + ) -> Result; +} + +impl BurnchainSnapshotExt for Connection { + fn count_canonical_burn_hashes_missing_from( + &self, + headers_schema: &str, + canonical_sql: &str, + ) -> Result { + self.query_row( + &format!( + "SELECT COUNT(*) FROM ({canonical_sql}) \ + WHERE burn_header_hash NOT IN \ + (SELECT block_hash FROM {headers_schema}.burnchain_db_block_headers)" + ), + NO_PARAMS, + |row| row.get(0), + ) + .map_err(DBError::from) + } +} + +/// The burnchain (`burnchain.sqlite`) snapshot spec. +pub(super) struct BurnchainDbSnapshotSpec; + +impl DbSnapshotSpec for BurnchainDbSnapshotSpec { + type Bind = NoBind; + + fn copy_spec_list() -> &'static [TableCopySpec] { + burnchain_copy_specs() + } + + fn db_label() -> &'static str { + "burnchain DB" + } -/// Tables the burnchain copy clones for schema fidelity but does not populate. -/// `overrides` (reward-cycle affirmation-map overrides) is never read or written -/// by any production path, so it is intentionally schema-only: cloning its schema -/// keeps the dst complete and drift-guarded without copying rows. -pub(super) const SCHEMA_ONLY_TABLES: &[&str] = &["overrides"]; + fn classify_hint() -> &'static str { + classify_hint!(burnchain_copy_specs) + } + + fn bind_params(&self, bind: NoBind) -> Result, Error> { + match bind {} + } +} /// The canonical burn-hash set staged by [`populate_canonical_burn_hashes`], /// as a SELECT fragment. const CANONICAL_BURN_HASHES_SQL: &str = "SELECT burn_header_hash FROM canonical_burn_hashes"; -/// Every table whose schema must exist in the squashed dst (copied + schema-only). -fn all_required_tables() -> Vec<&'static str> { - COPIED_TABLES - .iter() - .chain(SCHEMA_ONLY_TABLES) - .copied() - .collect() -} - -/// The burnchain snapshot's source-schema guard (see [`assert_source_schema`]); +/// The burnchain snapshot's source-schema guard (see +/// [`DbSnapshotSpec::assert_source_classified`]); /// `test_no_unclassified_burnchain_tables` runs it against a fresh schema. pub(super) fn assert_source_tables_classified(src_conn: &Connection) -> Result<(), Error> { - assert_source_schema( - src_conn, - &all_required_tables(), - "burnchain DB", - "COPIED_TABLES (to copy) or SCHEMA_ONLY_TABLES (to skip) in snapshot/burnchain.rs", - ) + BurnchainDbSnapshotSpec::assert_source_classified(src_conn) } /// Row-count statistics returned by [`copy_burnchain_db`]. @@ -101,7 +129,7 @@ fn read_squashed_sortition_canonical_set( let tip_height = SortitionDB::get_highest_known_burn_chain_tip(conn) .map_err(|e| Error::CorruptionError(format!("cannot read squashed sortition tip: {e}")))? .block_height; - let hashes = SortitionDB::get_all_snapshot_burn_header_hashes(conn).map_err(|e| { + let hashes = conn.get_all_snapshot_burn_header_hashes().map_err(|e| { Error::CorruptionError(format!("cannot read squashed sortition snapshots: {e}")) })?; Ok((tip_height, hashes)) @@ -178,44 +206,40 @@ pub fn copy_burnchain_db( ) } -/// Build the copy specs for the burnchain DB, in dependency order: -/// canonical headers and ops (burn-hash filtered), commit metadata, and -/// `anchor_blocks` derived from the copied commit metadata. -pub(super) fn burnchain_copy_specs() -> Vec { - let bhh = CANONICAL_BURN_HASHES_SQL; - vec![ - TableCopySpec { - table: "db_config", - source_sql: "SELECT * FROM src.db_config".into(), - }, - TableCopySpec { - table: "burnchain_db_block_headers", - source_sql: format!( - "SELECT * FROM src.burnchain_db_block_headers WHERE block_hash IN ({bhh})" - ), - }, - TableCopySpec { - table: "burnchain_db_block_ops", - source_sql: format!( - "SELECT * FROM src.burnchain_db_block_ops WHERE block_hash IN ({bhh})" - ), - }, - TableCopySpec { - table: "block_commit_metadata", - source_sql: format!( - "SELECT * FROM src.block_commit_metadata WHERE burn_block_hash IN ({bhh})" - ), - }, - TableCopySpec { - table: "anchor_blocks", - source_sql: "SELECT * FROM src.anchor_blocks \ +/// Build the copy specs for the burnchain DB, in dependency order: canonical +/// headers and ops (burn-hash filtered), commit metadata, and `anchor_blocks` +/// derived from the copied commit metadata. `overrides` is schema-only: +/// reward-cycle affirmation-map overrides are never read or written by any +/// production path, so its schema is cloned for fidelity but no rows are copied. +pub(super) fn burnchain_copy_specs() -> &'static [TableCopySpec] { + static SPECS: &[TableCopySpec] = &[ + TableCopySpec::sql("db_config", "SELECT * FROM src.db_config"), + TableCopySpec::sql( + "burnchain_db_block_headers", + "SELECT * FROM src.burnchain_db_block_headers \ + WHERE block_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "burnchain_db_block_ops", + "SELECT * FROM src.burnchain_db_block_ops \ + WHERE block_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "block_commit_metadata", + "SELECT * FROM src.block_commit_metadata \ + WHERE burn_block_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "anchor_blocks", + "SELECT * FROM src.anchor_blocks \ WHERE reward_cycle IN ( \ SELECT DISTINCT anchor_block FROM block_commit_metadata \ WHERE anchor_block IS NOT NULL \ - )" - .into(), - }, - ] + )", + ), + TableCopySpec::schema_only("overrides"), + ]; + SPECS } /// Consistency assertion: the squashed sortition DB's tip must match the @@ -236,24 +260,22 @@ fn copy_burnchain_db_inner( conn: &Connection, canonical_hashes: &[BurnchainHeaderHash], ) -> Result { - clone_schemas_from_source(conn, &all_required_tables())?; + let spec = BurnchainDbSnapshotSpec; + clone_schemas_from_source(conn, &BurnchainDbSnapshotSpec::table_names())?; populate_canonical_burn_hashes(conn, canonical_hashes)?; // Completeness assertion: every canonical burn hash must exist in source. - let missing_count = BurnchainDB::count_canonical_burn_hashes_missing_from( - conn, - "src", - CANONICAL_BURN_HASHES_SQL, - ) - .map_err(|e| Error::CorruptionError(format!("cannot check canonical burn hashes: {e}")))?; + let missing_count = conn + .count_canonical_burn_hashes_missing_from("src", CANONICAL_BURN_HASHES_SQL) + .map_err(|e| Error::CorruptionError(format!("cannot check canonical burn hashes: {e}")))?; if missing_count > 0 { return Err(Error::CorruptionError(format!( "{missing_count} canonical burn hashes missing from source burnchain DB" ))); } - let results = execute_copy_specs(conn, &burnchain_copy_specs())?; + let results = spec.run_copy(conn)?; let stats = BurnchainDbCopyStats { block_headers_rows: copied_rows(&results, "burnchain_db_block_headers"), diff --git a/stackslib/src/chainstate/stacks/db/snapshot/common.rs b/stackslib/src/chainstate/stacks/db/snapshot/common.rs index be16fc3c21..7041581103 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/common.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/common.rs @@ -16,20 +16,117 @@ use std::time::Instant; use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; -use rusqlite::{params, Connection, OptionalExtension}; +use rusqlite::types::Value; +use rusqlite::{params, params_from_iter, Connection, OptionalExtension}; use crate::chainstate::stacks::index::Error; -/// A spec for copying a single table from the ATTACHed `src` database. -/// -/// The `source_sql` is the exact `SELECT` used to filter source rows. -/// Copy uses plain `INSERT ... SELECT` (no `OR IGNORE`/`OR REPLACE`) so that -/// unexpected pre-population in the destination fails loudly. -pub struct TableCopySpec { +/// Build a [`DbSnapshotSpec::classify_hint`] string -- `"() in "`, +/// optionally with extra prose before the file -- from the spec function itself, +/// so a rename or file move can't leave the guard message stale. `$f` is forced +/// to resolve to a real item at compile time, and `file!()` reports the call +/// site (the DB's own module), not this one. +macro_rules! classify_hint { + ($f:path $(, $extra:expr)?) => {{ + const _: () = { + let _ = $f; + }; + concat!(stringify!($f), "()", $($extra,)? " in ", file!()) + }}; +} +pub(super) use classify_hint; + +/// How a table's destination rows are produced. +pub enum TableCopySource { + /// Row-copy via plain `INSERT INTO {table} {sql}` selecting from the + /// ATTACHed `src` (no `OR IGNORE`/`OR REPLACE`, so unexpected + /// pre-population in the destination fails loudly). A `&'static str` SELECT; + /// any runtime values are `?N` placeholders bound at execution time (see the + /// `binds` closure on [`execute_copy_specs`]), e.g. + /// `"SELECT * FROM src.signer_stats WHERE reward_cycle <= ?1"`. + Sql(&'static str), + /// The table's schema is cloned into the destination for fidelity, but no + /// rows are copied here -- it is populated by a later phase or left empty. + SchemaOnly, +} + +/// Bind marker for a DB whose copy specs carry no runtime `?N` placeholders +/// (every filter is a `'static` temp-table subquery). Uninhabited, so +/// [`DbSnapshotSpec::bind_params`] for such a DB is a `match bind {}`. +#[derive(Debug, Clone, Copy)] +pub enum NoBind {} + +/// A spec for materializing a single table in the squashed destination from the +/// ATTACHed `src` database. A slice's spec list is the single source of truth +/// for the tables it accounts for; the table-name sets are derived from it via +/// [`TableCopySpecs`]. `B` is the DB's own bind enum ([`NoBind`] when it has no +/// runtime placeholders), so a spec can only carry a bind its own DB resolves. +pub struct TableCopySpec { pub table: &'static str, - /// The exact SELECT for the source side, e.g. - /// `"SELECT * FROM src.snapshots WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)"`. - pub source_sql: String, + pub source: TableCopySource, + /// Positional bind values for the SELECT's `?N` placeholders, resolved at + /// execution time by [`DbSnapshotSpec::bind_params`]. `None` for specs with + /// no placeholders. + pub bind: Option, +} + +impl TableCopySpec { + pub const fn sql(table: &'static str, sql: &'static str) -> Self { + Self { + table, + source: TableCopySource::Sql(sql), + bind: None, + } + } + + pub const fn sql_with_bind(table: &'static str, sql: &'static str, bind: B) -> Self { + Self { + table, + source: TableCopySource::Sql(sql), + bind: Some(bind), + } + } + + pub const fn schema_only(table: &'static str) -> Self { + Self { + table, + source: TableCopySource::SchemaOnly, + bind: None, + } + } +} + +/// A borrowed view over a copy-spec list, exposing the table-name sets the copy +/// and the source-schema guard derive from it. +pub struct TableCopySpecs<'a, B>(&'a [TableCopySpec]); + +impl<'a, B> TableCopySpecs<'a, B> { + pub fn new(specs: &'a [TableCopySpec]) -> Self { + Self(specs) + } + + pub fn as_slice(&self) -> &'a [TableCopySpec] { + self.0 + } + + pub fn iter(&self) -> std::slice::Iter<'a, TableCopySpec> { + self.0.iter() + } + + /// Every table the specs account for (row-copied plus schema-only): the set + /// whose schema must be cloned into the destination. + pub fn table_names(&self) -> Vec<&'static str> { + self.iter().map(|s| s.table).collect() + } + + /// The subset whose schema is cloned but no rows are copied + /// ([`TableCopySource::SchemaOnly`]). + pub fn schema_only(&self) -> Vec<&'static str> { + self.iter() + .filter(|s| matches!(s.source, TableCopySource::SchemaOnly)) + .map(|s| s.table) + .collect() + } } /// Clone table and index schemas from the source DB (via `sqlite_master`) @@ -87,22 +184,39 @@ pub fn clone_schemas_from_source(conn: &Connection, tables: &[&str]) -> Result<( /// them once at the end (one rebuild vs N per-row B-tree updates). /// `sqlite_autoindex_*` have `sql IS NULL` and are skipped. /// -/// Returns a vec of (table_name, rows_copied). -pub fn execute_copy_specs( +/// Returns a vec of (table_name, rows_copied), including `(table, 0)` for +/// schema-only specs. +/// +/// `binds` resolves a spec's `bind` into the positional bind values for its +/// `?N` placeholders; it is called only for specs that carry a bind. It returns +/// `Result` so a bind builder can reject an out-of-range value. +pub fn execute_copy_specs( conn: &Connection, - specs: &[TableCopySpec], + specs: &[TableCopySpec], + binds: impl Fn(B) -> Result, Error>, ) -> Result, Error> { let mut results = Vec::with_capacity(specs.len()); for spec in specs { + // SchemaOnly tables are schema-cloned by `clone_schemas_from_source`; + // there are no rows to copy here. + let TableCopySource::Sql(source_sql) = &spec.source else { + results.push((spec.table, 0)); + continue; + }; + let params = match spec.bind { + Some(bind) => binds(bind)?, + None => Vec::new(), + }; let t_total = Instant::now(); let rows = with_indexes_dropped(conn, spec.table, |conn| { - let sql = format!("INSERT INTO {} {}", spec.table, spec.source_sql); - Ok(conn.execute(&sql, []).map_err(Error::SQLError)? as u64) + let sql = format!("INSERT INTO {} {source_sql}", spec.table); + Ok(conn + .execute(&sql, params_from_iter(params.iter())) + .map_err(Error::SQLError)? as u64) })?; info!( - "[copy] {} ({} rows) in {:?}", + "[copy] {} ({rows} rows) in {:?}", spec.table, - rows, t_total.elapsed(), ); results.push((spec.table, rows)); @@ -112,7 +226,7 @@ pub fn execute_copy_specs( /// Look up the rows-copied count for `table` in [`execute_copy_specs`] /// results. Panics if `table` had no spec: that is a bug in the caller's -/// spec list, not a data error. +/// spec list, not a data error. Schema-only specs are reported as 0 rows. pub fn copied_rows(results: &[(&'static str, u64)], table: &str) -> u64 { results .iter() @@ -318,11 +432,93 @@ pub(super) const MARF_INFRA_TABLES: &[&str] = &[ "migrated_version", ]; +/// The snapshot specification for a single source database: its copy specs plus +/// the metadata the copy and the source-schema guard need. One struct per source +/// DB (`IndexDbSnapshotSpec`, `SortitionDbSnapshotSpec`, ...) implements it, +/// carrying whatever runtime parameters its bind values need. +/// +/// The trait is used for shared code, not polymorphism (there are no `dyn` +/// callers). Source-schema classification is instance-independent, so its +/// methods are associated functions (`Self::assert_source_classified`) and need +/// no instance; only [`Self::run_copy`] and its helpers take `&self`. +pub trait DbSnapshotSpec { + /// This DB's bind enum -- the runtime values its `?N` placeholders resolve + /// to. [`NoBind`] for DBs whose specs carry no placeholders. + type Bind: Copy + 'static; + + /// The DB's canonical copy specs: the single source of truth for its table + /// set. Both [`Self::table_names`] and the default [`Self::copy_specs`] derive + /// from this. A DB whose executed specs vary with a runtime parameter (the + /// sortition boundary) returns the boundary-invariant list here and overrides + /// [`Self::copy_specs`] for the runtime variant. + fn copy_spec_list() -> &'static [TableCopySpec]; + + /// Every table the snapshot accounts for (row-copied plus schema-only): the + /// set whose schema must be cloned into the destination. Independent of any + /// runtime parameter, hence an associated fn. + fn table_names() -> Vec<&'static str> { + TableCopySpecs::new(Self::copy_spec_list()).table_names() + } + + /// Tables recognized by the source-schema guard beyond the copy specs -- + /// MARF infra (created by the squash engine) or deliberately-ignored tables. + /// Default: none (non-MARF DBs). + fn extra_recognized_tables() -> Vec<&'static str> { + Vec::new() + } + + /// Human label for the source DB in guard errors. + fn db_label() -> &'static str; + + /// Where a newly added source table should be classified. + fn classify_hint() -> &'static str; + + /// The source-schema guard's recognized set: the copy-spec tables plus + /// [`Self::extra_recognized_tables`]. + fn known_tables() -> Vec<&'static str> { + let mut tables = Self::table_names(); + tables.extend(Self::extra_recognized_tables()); + tables + } + + /// Reject a source DB whose schema has tables this spec doesn't classify + /// (the snapshot would omit them and a node booting from it would recreate + /// them empty). + fn assert_source_classified(src_conn: &Connection) -> Result<(), Error> { + assert_source_schema( + src_conn, + &Self::known_tables(), + Self::db_label(), + Self::classify_hint(), + ) + } + + /// The copy specs to execute. Defaults to [`Self::copy_spec_list`]; a DB + /// whose specs vary with a runtime parameter (the sortition boundary) + /// overrides this while keeping the same table set. + fn copy_specs(&self) -> TableCopySpecs<'static, Self::Bind> { + TableCopySpecs::new(Self::copy_spec_list()) + } + + /// Positional bind values for a spec's `?N` placeholders. Called only for + /// specs that carry a bind; DBs with [`NoBind`] specs never reach it, so + /// their body is `match bind {}`. + fn bind_params(&self, bind: Self::Bind) -> Result, Error>; + + /// Row-copy this DB's specs into `conn`, binding each spec's `?N` + /// placeholders from [`Self::bind_params`]. Returns `(table, rows_copied)`. + fn run_copy(&self, conn: &Connection) -> Result, Error> { + let specs = self.copy_specs(); + execute_copy_specs(conn, specs.as_slice(), |b| self.bind_params(b)) + } +} + #[cfg(test)] mod tests { use rstest::rstest; + use rusqlite::Connection; - use super::percent_encode_path; + use super::{copied_rows, execute_copy_specs, percent_encode_path, NoBind, TableCopySpec}; /// Representative paths survive the `file:` URI percent-encoding used /// by [`super::with_offline_write_session`]'s read-only ATTACH. @@ -337,4 +533,18 @@ mod tests { fn percent_encode_path_cases(#[case] input: &str, #[case] expected: &str) { assert_eq!(percent_encode_path(input), expected); } + + #[test] + fn execute_copy_specs_reports_schema_only_zero_rows() { + let conn = Connection::open_in_memory().unwrap(); + let specs = [TableCopySpec::::schema_only("schema_only_table")]; + + let results = execute_copy_specs(&conn, &specs, |_| { + panic!("schema-only specs must not request bind params") + }) + .unwrap(); + + assert_eq!(results, vec![("schema_only_table", 0)]); + assert_eq!(copied_rows(&results, "schema_only_table"), 0); + } } diff --git a/stackslib/src/chainstate/stacks/db/snapshot/index.rs b/stackslib/src/chainstate/stacks/db/snapshot/index.rs index b004518b10..c77b55c749 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/index.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/index.rs @@ -16,75 +16,67 @@ use std::collections::HashSet; use std::time::Instant; +use rusqlite::types::Value; use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; use stacks_common::types::chainstate::StacksBlockId; use super::common::{ - assert_source_schema, clone_schemas_from_source, copied_rows, execute_copy_specs, - with_offline_write_session, TableCopySpec, MARF_INFRA_TABLES, + classify_hint, clone_schemas_from_source, copied_rows, with_offline_write_session, + DbSnapshotSpec, TableCopySpec, MARF_INFRA_TABLES, }; use super::fork_storage::{collect_canonical_leaf_hashes, copy_canonical_fork_storage}; use crate::burnchains::PoxConstants; use crate::chainstate::stacks::index::{trie_sql, Error, MARFValue}; -use crate::util_lib::db::sqlite_open; +use crate::util_lib::db::{sqlite_open, u64_to_sql}; -/// Tables copied (with canonical-filtered content) into the squashed index DB. -pub(crate) const COPIED_TABLES: &[&str] = &[ - "db_config", - "block_headers", - "nakamoto_block_headers", - "payments", - "transactions", - "nakamoto_tenure_events", - "nakamoto_reward_sets", - "signer_stats", - "matured_rewards", - "burnchain_txids", - "epoch_transitions", - "staging_blocks", -]; - -/// Tables the index copy clones for schema fidelity but does not populate. They -/// are intentionally schema-only in this slice (never written by the index -/// copy); cloning their schema prevents missing-table crashes if any code path -/// references them. -pub(crate) const SCHEMA_ONLY_TABLES: &[&str] = &[ - "staging_microblocks", - "staging_microblocks_data", - "invalidated_microblocks_data", // Epoch 2.x block orphaning only (blocks.rs:2189) - "user_supporters", // Dead table: zero runtime references -]; +/// The index snapshot's `?N` bind: the `signer_stats` reward-cycle bound. +#[derive(Clone, Copy)] +pub(super) enum IndexBind { + /// `signer_stats` filters `reward_cycle <= ?1`. + MaxRewardCycle, +} -/// Every table whose schema must exist in the squashed dst (copied + schema-only). -fn all_required_tables() -> Vec<&'static str> { - COPIED_TABLES - .iter() - .chain(SCHEMA_ONLY_TABLES) - .copied() - .collect() +/// The index (`index.sqlite`) side-table snapshot spec. `max_reward_cycle` feeds +/// the `signer_stats` `?N` bind only -- the table-name set is independent of it. +/// The MARF infra tables ([`MARF_INFRA_TABLES`]) are created by the squash +/// engine, so they're recognized by the guard but not row-copied. +pub(super) struct IndexDbSnapshotSpec { + pub max_reward_cycle: u64, } -/// Every table the index snapshot accounts for: content-copied ([`COPIED_TABLES`]), -/// schema-cloned but unpopulated ([`SCHEMA_ONLY_TABLES`]), or owned by the MARF -/// squash itself ([`MARF_INFRA_TABLES`]). -fn known_index_tables() -> Vec<&'static str> { - COPIED_TABLES - .iter() - .chain(SCHEMA_ONLY_TABLES) - .chain(MARF_INFRA_TABLES) - .copied() - .collect() +impl DbSnapshotSpec for IndexDbSnapshotSpec { + type Bind = IndexBind; + + fn copy_spec_list() -> &'static [TableCopySpec] { + index_copy_specs() + } + + fn extra_recognized_tables() -> Vec<&'static str> { + MARF_INFRA_TABLES.to_vec() + } + + fn db_label() -> &'static str { + "index DB" + } + + fn classify_hint() -> &'static str { + classify_hint!(index_copy_specs) + } + + fn bind_params(&self, bind: IndexBind) -> Result, Error> { + match bind { + IndexBind::MaxRewardCycle => { + Ok(vec![Value::Integer(u64_to_sql(self.max_reward_cycle)?)]) + } + } + } } -/// The index snapshot's source-schema guard (see [`assert_source_schema`]); +/// The index snapshot's source-schema guard (see +/// [`DbSnapshotSpec::assert_source_classified`]); /// `test_no_unclassified_source_tables` runs it against a fresh schema. pub(super) fn assert_source_tables_classified(src_conn: &Connection) -> Result<(), Error> { - assert_source_schema( - src_conn, - &known_index_tables(), - "index DB", - "COPIED_TABLES (to copy) or SCHEMA_ONLY_TABLES (to schema-clone only) in snapshot/index.rs", - ) + IndexDbSnapshotSpec::assert_source_classified(src_conn) } /// Row-count statistics returned by [`copy_index_side_tables`]. @@ -202,76 +194,80 @@ fn derive_max_reward_cycle( /// - `db_config` is copied in full. /// - `staging_blocks` adds a check for processend and non-orphaned blocks. /// - `signer_stats` is cut off at the canonical tip's reward cycle. -pub(super) fn index_copy_specs(max_reward_cycle: u64) -> Vec { - let cb = "SELECT index_block_hash FROM canonical_blocks"; - vec![ - TableCopySpec { - table: "db_config", - source_sql: "SELECT * FROM src.db_config".into(), - }, - TableCopySpec { - table: "block_headers", - source_sql: format!("SELECT * FROM src.block_headers WHERE index_block_hash IN ({cb})"), - }, - TableCopySpec { - table: "nakamoto_block_headers", - source_sql: format!( - "SELECT * FROM src.nakamoto_block_headers WHERE index_block_hash IN ({cb})" - ), - }, - TableCopySpec { - table: "payments", - source_sql: format!("SELECT * FROM src.payments WHERE index_block_hash IN ({cb})"), - }, - TableCopySpec { - table: "transactions", - source_sql: format!("SELECT * FROM src.transactions WHERE index_block_hash IN ({cb})"), - }, - TableCopySpec { - table: "nakamoto_tenure_events", - source_sql: format!( - "SELECT * FROM src.nakamoto_tenure_events WHERE block_id IN ({cb})" - ), - }, - TableCopySpec { - table: "nakamoto_reward_sets", - source_sql: format!( - "SELECT * FROM src.nakamoto_reward_sets WHERE index_block_hash IN ({cb})" - ), - }, - TableCopySpec { - table: "matured_rewards", - source_sql: format!( - "SELECT * FROM src.matured_rewards WHERE child_index_block_hash IN ({cb})" - ), - }, - TableCopySpec { - table: "burnchain_txids", - source_sql: format!( - "SELECT * FROM src.burnchain_txids WHERE index_block_hash IN ({cb})" - ), - }, - TableCopySpec { - table: "epoch_transitions", - source_sql: format!("SELECT * FROM src.epoch_transitions WHERE block_id IN ({cb})"), - }, - TableCopySpec { - table: "staging_blocks", +pub(super) fn index_copy_specs() -> &'static [TableCopySpec] { + // `signer_stats`'s reward-cycle bound is the only runtime value, passed as + // the `?1` bind. + static SPECS: &[TableCopySpec] = &[ + TableCopySpec::sql("db_config", "SELECT * FROM src.db_config"), + TableCopySpec::sql( + "block_headers", + "SELECT * FROM src.block_headers \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "nakamoto_block_headers", + "SELECT * FROM src.nakamoto_block_headers \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "payments", + "SELECT * FROM src.payments \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "transactions", + "SELECT * FROM src.transactions \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "nakamoto_tenure_events", + "SELECT * FROM src.nakamoto_tenure_events \ + WHERE block_id IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "nakamoto_reward_sets", + "SELECT * FROM src.nakamoto_reward_sets \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "matured_rewards", + "SELECT * FROM src.matured_rewards \ + WHERE child_index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "burnchain_txids", + "SELECT * FROM src.burnchain_txids \ + WHERE index_block_hash IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "epoch_transitions", + "SELECT * FROM src.epoch_transitions \ + WHERE block_id IN (SELECT index_block_hash FROM canonical_blocks)", + ), + TableCopySpec::sql( + "staging_blocks", // Only canonical, fully-processed, non-orphaned blocks. - source_sql: format!( - "SELECT s.* FROM src.staging_blocks s \ - WHERE s.index_block_hash IN ({cb}) \ + "SELECT s.* FROM src.staging_blocks s \ + WHERE s.index_block_hash IN (SELECT index_block_hash FROM canonical_blocks) \ AND s.processed = 1 \ - AND s.orphaned = 0" - ), - }, - TableCopySpec { - table: "signer_stats", - source_sql: format!( - "SELECT * FROM src.signer_stats WHERE reward_cycle <= {max_reward_cycle}" - ), - }, - ] + AND s.orphaned = 0", + ), + TableCopySpec::sql_with_bind( + "signer_stats", + "SELECT * FROM src.signer_stats WHERE reward_cycle <= ?1", + IndexBind::MaxRewardCycle, + ), + // Schema-only: schema cloned for fidelity, not row-copied here. + // `staging_microblocks`/`_data` are populated later by the + // block-preservation phase; the other two stay empty. + TableCopySpec::schema_only("staging_microblocks"), + TableCopySpec::schema_only("staging_microblocks_data"), + // Only written when orphaning epoch 2.x blocks. + TableCopySpec::schema_only("invalidated_microblocks_data"), + // Dead table: zero runtime references. + TableCopySpec::schema_only("user_supporters"), + ]; + SPECS } /// Copy required non-MARF tables from the source `index.sqlite` into the @@ -294,7 +290,9 @@ pub fn copy_index_side_tables( let leaf_hashes = collect_canonical_leaf_hashes::(dst_path)?; with_offline_write_session(dst_path, &[("src", src_path)], "", |conn| { - clone_schemas_from_source(conn, &all_required_tables())?; + // Clone only the spec tables' schemas; MARF infra tables are created by + // the squash engine, not cloned here. + clone_schemas_from_source(conn, &IndexDbSnapshotSpec::table_names())?; copy_tables_inner(conn, &leaf_hashes, first_burn_height, reward_cycle_len) }) } @@ -323,8 +321,8 @@ fn copy_tables_inner( let max_reward_cycle = derive_max_reward_cycle(conn, &canonical_tip, first_burn_height, reward_cycle_len)?; - let specs = index_copy_specs(max_reward_cycle); - let results = execute_copy_specs(conn, &specs)?; + let spec = IndexDbSnapshotSpec { max_reward_cycle }; + let results = spec.run_copy(conn)?; conn.execute_batch("DROP TABLE IF EXISTS canonical_blocks") .map_err(Error::SQLError)?; diff --git a/stackslib/src/chainstate/stacks/db/snapshot/sortition.rs b/stackslib/src/chainstate/stacks/db/snapshot/sortition.rs index 98463e8fdc..73fbab4d2c 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/sortition.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/sortition.rs @@ -15,64 +15,214 @@ use std::collections::HashSet; -use rusqlite::{params, Connection, OpenFlags}; -use stacks_common::types::chainstate::SortitionId; +use rusqlite::types::Value; +use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; +use stacks_common::types::chainstate::{BurnchainHeaderHash, SortitionId}; +use stacks_common::types::sqlite::NO_PARAMS; use super::common::{ - assert_source_schema, clone_schemas_from_source, copied_rows, execute_copy_specs, - with_offline_write_session, TableCopySpec, MARF_INFRA_TABLES, + classify_hint, clone_schemas_from_source, copied_rows, with_offline_write_session, + DbSnapshotSpec, TableCopySpec, TableCopySpecs, MARF_INFRA_TABLES, }; use super::fork_storage::{collect_canonical_leaf_hashes, copy_canonical_fork_storage}; -use crate::chainstate::burn::db::sortdb::SortitionDB; pub use crate::chainstate::burn::db::sortdb::SortitionTipCopyBoundary; use crate::chainstate::stacks::index::{trie_sql, Error, MARFValue}; -use crate::util_lib::db::sqlite_open; - -/// Required sortition tables always present in production. -pub(super) const REQUIRED_TABLES: &[&str] = &[ - "db_config", - "snapshots", - "leader_keys", - "block_commits", - "block_commit_parents", - "snapshot_transition_ops", - "stacks_chain_tips", - "stacks_chain_tips_by_burn_view", - "missed_commits", - "stack_stx", - "transfer_stx", - "delegate_stx", - "vote_for_aggregate_key", - "preprocessed_reward_sets", - "epochs", -]; +use crate::util_lib::db::{sqlite_open, u64_to_sql, Error as db_error}; + +/// Snapshot-only reads over a sortition DB connection. +pub(crate) trait SortitionSnapshotExt { + /// Distinct burn header hashes of all snapshots, forks included. Only on a + /// squashed sortition DB is this exactly the canonical burnchain. + fn get_all_snapshot_burn_header_hashes(&self) -> Result, db_error>; + /// Burn header hash of the snapshot with the given sortition ID (raw stored + /// TEXT, byte-for-byte), or `None` if no such snapshot exists. + fn get_snapshot_burn_header_hash( + &self, + sortition_id: &SortitionId, + ) -> Result, db_error>; + /// Whether every Stacks-tip memo row (both memo tables) sits at or below the + /// boundary's Stacks height. A `None` boundary trivially passes. The copy + /// counterpart of [`stacks_tip_memo_copy_sql`]'s height rewrite. + fn stacks_tip_memos_within_boundary( + &self, + boundary: Option<&SortitionTipCopyBoundary>, + ) -> Result; +} + +impl SortitionSnapshotExt for Connection { + fn get_all_snapshot_burn_header_hashes(&self) -> Result, db_error> { + let mut stmt = self.prepare("SELECT DISTINCT burn_header_hash FROM snapshots")?; + let rows = stmt.query_map(NO_PARAMS, |row| row.get(0))?; + rows.collect::, _>>().map_err(db_error::from) + } + + fn get_snapshot_burn_header_hash( + &self, + sortition_id: &SortitionId, + ) -> Result, db_error> { + self.prepare_cached("SELECT burn_header_hash FROM snapshots WHERE sortition_id = ?1")? + .query_row(params![sortition_id], |row| row.get(0)) + .optional() + .map_err(db_error::from) + } + + fn stacks_tip_memos_within_boundary( + &self, + boundary: Option<&SortitionTipCopyBoundary>, + ) -> Result { + let Some(boundary) = boundary else { + return Ok(true); + }; + let max_height = u64_to_sql(boundary.max_stacks_height)?; + // Check if ANY above-boundary memo row exists. + self.query_row( + "SELECT NOT EXISTS( \ + SELECT 1 FROM stacks_chain_tips WHERE block_height > ?1 \ + UNION ALL \ + SELECT 1 FROM stacks_chain_tips_by_burn_view WHERE block_height > ?1 \ + )", + params![max_height], + |row| row.get(0), + ) + .map_err(db_error::from) + } +} + +/// Static source-projection SQL template for copying a Stacks-tip memo table +/// (`stacks_chain_tips`, or `stacks_chain_tips_by_burn_view` when +/// `include_burn_view`) from `src`, keeping rows whose `sortition_id` is in the +/// `canonical_sortitions` temp table. With a boundary (`has_boundary`), memo rows +/// above the boundary's Stacks height are rewritten down to the anchor (see +/// [`SortitionTipCopyBoundary`]); the anchor values are the `?1..?5` placeholders +/// supplied by [`stacks_tip_memo_copy_binds`]. +const fn stacks_tip_memo_copy_sql(include_burn_view: bool, has_boundary: bool) -> &'static str { + match (include_burn_view, has_boundary) { + (false, false) => { + "SELECT * FROM src.stacks_chain_tips \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)" + } + (true, false) => { + "SELECT * FROM src.stacks_chain_tips_by_burn_view \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)" + } + (false, true) => { + "SELECT sortition_id, \ + CASE WHEN block_height > ?1 THEN ?2 ELSE consensus_hash END, \ + CASE WHEN block_height > ?1 THEN ?3 ELSE block_hash END, \ + CASE WHEN block_height > ?1 THEN ?4 ELSE block_height END \ + FROM src.stacks_chain_tips \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions) \ + AND (block_height <= ?1 OR consensus_hash = ?2)" + } + (true, true) => { + "SELECT sortition_id, \ + CASE WHEN block_height > ?1 THEN ?2 ELSE consensus_hash END, \ + CASE WHEN block_height > ?1 THEN ?5 ELSE burn_view_consensus_hash END, \ + CASE WHEN block_height > ?1 THEN ?3 ELSE block_hash END, \ + CASE WHEN block_height > ?1 THEN ?4 ELSE block_height END \ + FROM src.stacks_chain_tips_by_burn_view \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions) \ + AND (block_height <= ?1 OR (consensus_hash = ?2 AND burn_view_consensus_hash = ?5))" + } + } +} + +/// Positional `?N` bind values for [`stacks_tip_memo_copy_sql`]'s boundary-rewrite +/// template; empty when `boundary` is `None` (the plain template has no +/// placeholders). Order: `?1` max Stacks height, `?2` anchor consensus hash, `?3` +/// anchor block hash, `?4` anchor block height, and (for `include_burn_view`) +/// `?5` anchor burn-view consensus hash. +fn stacks_tip_memo_copy_binds( + boundary: Option<&SortitionTipCopyBoundary>, + include_burn_view: bool, +) -> Result, Error> { + let Some(boundary) = boundary else { + return Ok(Vec::new()); + }; + let mut params = vec![ + Value::Integer(u64_to_sql(boundary.max_stacks_height)?), + Value::Text(boundary.anchor_consensus_hash.to_string()), + Value::Text(boundary.anchor_block_hash.to_string()), + Value::Integer(u64_to_sql(boundary.anchor_block_height)?), + ]; + if include_burn_view { + params.push(Value::Text( + boundary.anchor_burn_view_consensus_hash.to_string(), + )); + } + Ok(params) +} /// Tables that may appear in a source sortition DB but are deliberately not /// copied. `snapshot_burn_distributions` is written only under the `testing` /// feature (`SortitionDBTx::store_burn_distribution`), never in production. pub(super) const IGNORED_TABLES: &[&str] = &["snapshot_burn_distributions"]; -/// Every table the sortition snapshot accounts for: content-copied -/// ([`REQUIRED_TABLES`]), owned by the MARF squash itself -/// ([`MARF_INFRA_TABLES`]), or deliberately skipped ([`IGNORED_TABLES`]). -fn known_sortition_tables() -> Vec<&'static str> { - REQUIRED_TABLES - .iter() - .chain(MARF_INFRA_TABLES) - .chain(IGNORED_TABLES) - .copied() - .collect() +/// The sortition (`marf.sqlite` side-tables) snapshot spec. The `boundary` +/// selects the `stacks_chain_tips*` memo template (plain vs rewrite) and feeds +/// the rewrite anchors as `?N` binds; the table-name set is independent of it. +/// MARF infra ([`MARF_INFRA_TABLES`], created by the squash engine) and +/// deliberately-skipped ([`IGNORED_TABLES`]) tables are recognized by the guard +/// but not row-copied. +pub(super) struct SortitionDbSnapshotSpec { + pub boundary: Option, +} + +/// The sortition snapshot's `?N` bind: the `stacks_chain_tips*` memo tables' +/// boundary-rewrite anchors. Present only when copying with a boundary. +#[derive(Clone, Copy)] +pub(super) enum SortitionBind { + TipMemo { include_burn_view: bool }, } -/// The sortition snapshot's source-schema guard (see [`assert_source_schema`]); -/// `test_no_unclassified_sortition_tables` runs it against a fresh schema. +impl DbSnapshotSpec for SortitionDbSnapshotSpec { + type Bind = SortitionBind; + + fn copy_spec_list() -> &'static [TableCopySpec] { + // Boundary-invariant list for classification/`table_names`: `has_boundary` + // only swaps the two memo SQL templates, not the table set (asserted by + // `test_sortition_copy_specs_boundary_invariant_table_set`). The executed + // specs are chosen by the `copy_specs` override below. + sortition_copy_specs(false) + } + + fn extra_recognized_tables() -> Vec<&'static str> { + MARF_INFRA_TABLES + .iter() + .copied() + .chain(IGNORED_TABLES.iter().copied()) + .collect() + } + + fn db_label() -> &'static str { + "sortition DB" + } + + fn classify_hint() -> &'static str { + classify_hint!( + sortition_copy_specs, + " (to copy) or IGNORED_TABLES (to skip)" + ) + } + + fn copy_specs(&self) -> TableCopySpecs<'static, SortitionBind> { + TableCopySpecs::new(sortition_copy_specs(self.boundary.is_some())) + } + + fn bind_params(&self, bind: SortitionBind) -> Result, Error> { + match bind { + SortitionBind::TipMemo { include_burn_view } => { + stacks_tip_memo_copy_binds(self.boundary.as_ref(), include_burn_view) + } + } + } +} + +/// The sortition snapshot's source-schema guard (see +/// [`DbSnapshotSpec::assert_source_classified`]); `test_no_unclassified_sortition_tables` +/// runs it against a fresh schema. The recognized set is independent of the boundary. pub(super) fn assert_source_tables_classified(src_conn: &Connection) -> Result<(), Error> { - assert_source_schema( - src_conn, - &known_sortition_tables(), - "sortition DB", - "REQUIRED_TABLES (to copy) or IGNORED_TABLES (to skip) in snapshot/sortition.rs", - ) + SortitionDbSnapshotSpec::assert_source_classified(src_conn) } /// Row-count statistics returned by [`copy_sortition_side_tables`]. @@ -117,7 +267,7 @@ fn populate_canonical_sortitions( let mut burn_hashes: HashSet = HashSet::new(); let mut orphans: u64 = 0; for (_, sortition_id, _) in &canonical { - match SortitionDB::get_snapshot_burn_header_hash(src_conn, sortition_id)? { + match src_conn.get_snapshot_burn_header_hash(sortition_id)? { Some(burn_header_hash) => { burn_hashes.insert(burn_header_hash); } @@ -162,6 +312,23 @@ fn validate_tip_boundary(boundary: Option<&SortitionTipCopyBoundary>) -> Result< Ok(()) } +/// A `stacks_chain_tips*` memo-table spec. With a boundary the SQL is the +/// anchor-rewrite template (its `?N` anchors bound via [`SortitionBind::TipMemo`]); +/// without one it is the plain template, which has no placeholders and so carries +/// no bind. +const fn memo_spec( + table: &'static str, + include_burn_view: bool, + has_boundary: bool, +) -> TableCopySpec { + let sql = stacks_tip_memo_copy_sql(include_burn_view, has_boundary); + if has_boundary { + TableCopySpec::sql_with_bind(table, sql, SortitionBind::TipMemo { include_burn_view }) + } else { + TableCopySpec::sql(table, sql) + } +} + /// Build the copy specs for sortition side tables. /// /// Tables are grouped by their filter key: @@ -171,99 +338,87 @@ fn validate_tip_boundary(boundary: Option<&SortitionTipCopyBoundary>) -> Result< /// /// The set of tables is independent of `boundary`; the boundary only rewrites /// the `stacks_chain_tips*` source SQL. -pub(super) fn sortition_copy_specs( - boundary: Option<&SortitionTipCopyBoundary>, -) -> Vec { - let sid = "SELECT sortition_id FROM canonical_sortitions"; - let bhh = "SELECT burn_header_hash FROM canonical_burn_hashes"; - - vec![ - TableCopySpec { - table: "db_config", - source_sql: "SELECT * FROM src.db_config".into(), - }, - // sortition_id-filtered tables - TableCopySpec { - table: "snapshots", - source_sql: format!("SELECT * FROM src.snapshots WHERE sortition_id IN ({sid})"), - }, - TableCopySpec { - table: "leader_keys", - source_sql: format!("SELECT * FROM src.leader_keys WHERE sortition_id IN ({sid})"), - }, - TableCopySpec { - table: "block_commits", - source_sql: format!("SELECT * FROM src.block_commits WHERE sortition_id IN ({sid})"), - }, - TableCopySpec { - table: "block_commit_parents", - source_sql: format!( - "SELECT * FROM src.block_commit_parents WHERE block_commit_sortition_id IN ({sid})" - ), - }, - TableCopySpec { - table: "snapshot_transition_ops", - source_sql: format!( - "SELECT * FROM src.snapshot_transition_ops WHERE sortition_id IN ({sid})" - ), - }, - TableCopySpec { - table: "stacks_chain_tips", - source_sql: SortitionDB::stacks_tip_memo_copy_sql( - "stacks_chain_tips", - "src", - sid, - false, - boundary, - ), - }, - TableCopySpec { - table: "stacks_chain_tips_by_burn_view", - source_sql: SortitionDB::stacks_tip_memo_copy_sql( - "stacks_chain_tips_by_burn_view", - "src", - sid, - true, - boundary, - ), - }, - TableCopySpec { - table: "preprocessed_reward_sets", - source_sql: format!( - "SELECT * FROM src.preprocessed_reward_sets WHERE sortition_id IN ({sid})" - ), - }, - TableCopySpec { - table: "missed_commits", - source_sql: format!( - "SELECT * FROM src.missed_commits WHERE intended_sortition_id IN ({sid})" - ), - }, - // burn_header_hash-filtered tables - TableCopySpec { - table: "stack_stx", - source_sql: format!("SELECT * FROM src.stack_stx WHERE burn_header_hash IN ({bhh})"), - }, - TableCopySpec { - table: "transfer_stx", - source_sql: format!("SELECT * FROM src.transfer_stx WHERE burn_header_hash IN ({bhh})"), - }, - TableCopySpec { - table: "delegate_stx", - source_sql: format!("SELECT * FROM src.delegate_stx WHERE burn_header_hash IN ({bhh})"), - }, - TableCopySpec { - table: "vote_for_aggregate_key", - source_sql: format!( - "SELECT * FROM src.vote_for_aggregate_key WHERE burn_header_hash IN ({bhh})" - ), - }, - // Full-copy tables - TableCopySpec { - table: "epochs", - source_sql: "SELECT * FROM src.epochs".to_string(), - }, - ] +pub(super) fn sortition_copy_specs(has_boundary: bool) -> &'static [TableCopySpec] { + // The only runtime values are the `stacks_chain_tips*` boundary-rewrite + // anchors, supplied as `?N` binds; `has_boundary` selects the rewrite vs + // plain memo template (see `stacks_tip_memo_copy_sql`). The plain and rewrite + // lists differ only in those two memo specs, so a local macro builds both + // from one definition. + macro_rules! sortition_specs { + ($has_boundary:expr) => { + &[ + TableCopySpec::sql("db_config", "SELECT * FROM src.db_config"), + // sortition_id-filtered tables + TableCopySpec::sql( + "snapshots", + "SELECT * FROM src.snapshots \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + TableCopySpec::sql( + "leader_keys", + "SELECT * FROM src.leader_keys \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + TableCopySpec::sql( + "block_commits", + "SELECT * FROM src.block_commits \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + TableCopySpec::sql( + "block_commit_parents", + "SELECT * FROM src.block_commit_parents \ + WHERE block_commit_sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + TableCopySpec::sql( + "snapshot_transition_ops", + "SELECT * FROM src.snapshot_transition_ops \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + memo_spec("stacks_chain_tips", false, $has_boundary), + memo_spec("stacks_chain_tips_by_burn_view", true, $has_boundary), + TableCopySpec::sql( + "preprocessed_reward_sets", + "SELECT * FROM src.preprocessed_reward_sets \ + WHERE sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + TableCopySpec::sql( + "missed_commits", + "SELECT * FROM src.missed_commits \ + WHERE intended_sortition_id IN (SELECT sortition_id FROM canonical_sortitions)", + ), + // burn_header_hash-filtered tables + TableCopySpec::sql( + "stack_stx", + "SELECT * FROM src.stack_stx \ + WHERE burn_header_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "transfer_stx", + "SELECT * FROM src.transfer_stx \ + WHERE burn_header_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "delegate_stx", + "SELECT * FROM src.delegate_stx \ + WHERE burn_header_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + TableCopySpec::sql( + "vote_for_aggregate_key", + "SELECT * FROM src.vote_for_aggregate_key \ + WHERE burn_header_hash IN (SELECT burn_header_hash FROM canonical_burn_hashes)", + ), + // Full-copy tables + TableCopySpec::sql("epochs", "SELECT * FROM src.epochs"), + ] + }; + } + static PLAIN: &[TableCopySpec] = sortition_specs!(false); + static REWRITE: &[TableCopySpec] = sortition_specs!(true); + if has_boundary { + REWRITE + } else { + PLAIN + } } /// Copy required non-MARF tables from the source sortition DB into the @@ -297,7 +452,9 @@ pub fn copy_sortition_side_tables_with_boundary( let leaf_hashes = collect_canonical_leaf_hashes::(dst_path)?; with_offline_write_session(dst_path, &[("src", src_path)], "", |conn| { - clone_schemas_from_source(conn, REQUIRED_TABLES)?; + // Clone only the spec tables' schemas; MARF infra is created by the + // squash engine and ignored tables are intentionally absent from the dst. + clone_schemas_from_source(conn, &SortitionDbSnapshotSpec::table_names())?; copy_sortition_tables_inner(&src_conn, conn, &leaf_hashes, stacks_boundary) }) } @@ -317,9 +474,11 @@ fn copy_sortition_tables_inner( populate_canonical_sortitions(src_conn, session_conn)?; // Execute descriptor-driven copies. - let specs = sortition_copy_specs(stacks_boundary); - let results = execute_copy_specs(session_conn, &specs)?; - if !SortitionDB::stacks_tip_memos_within_boundary(session_conn, stacks_boundary)? { + let spec = SortitionDbSnapshotSpec { + boundary: stacks_boundary.cloned(), + }; + let results = spec.run_copy(session_conn)?; + if !session_conn.stacks_tip_memos_within_boundary(stacks_boundary)? { return Err(Error::CorruptionError( "copied sortition tip row points past the Stacks MARF boundary".into(), )); diff --git a/stackslib/src/chainstate/stacks/db/snapshot/spv.rs b/stackslib/src/chainstate/stacks/db/snapshot/spv.rs index e87705d4e7..7a2bb177a5 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/spv.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/spv.rs @@ -16,35 +16,64 @@ use std::fs; use std::path::Path; +use rusqlite::types::Value; use rusqlite::{Connection, OpenFlags}; use super::common::{ - assert_source_schema, clone_schemas_from_source, copied_rows, execute_copy_specs, - with_offline_write_session, TableCopySpec, + classify_hint, clone_schemas_from_source, copied_rows, with_offline_write_session, + DbSnapshotSpec, TableCopySpec, }; use crate::burnchains::bitcoin::spv::num_complete_chain_work_intervals; use crate::chainstate::stacks::index::Error; -use crate::util_lib::db::sqlite_open; +use crate::util_lib::db::{sqlite_open, u64_to_sql}; -/// Tables required for the current headers.sqlite schema. -pub(super) const REQUIRED_TABLES: &[&str] = &["headers", "db_config", "chain_work"]; +/// The SPV headers snapshot's `?N` binds, both derived from `burn_height`. +#[derive(Clone, Copy)] +pub(super) enum SpvBind { + /// `headers` filters `height <= ?1`. + BurnHeight, + /// `chain_work` filters `interval < ?1`. + CompleteChainWorkIntervals, +} + +/// The SPV headers (`headers.sqlite`) snapshot spec. The row-copied specs +/// are the whole schema. `burn_height` only feeds the `?N` binds, not the +/// table-name set. +pub(super) struct SpvDbSnapshotSpec { + pub burn_height: u64, +} + +impl DbSnapshotSpec for SpvDbSnapshotSpec { + type Bind = SpvBind; + + fn copy_spec_list() -> &'static [TableCopySpec] { + spv_copy_specs() + } -/// Every table the SPV headers snapshot accounts for. headers.sqlite is not -/// MARF-backed, so unlike the other slices there are no MARF infra tables to -/// exempt — the content-copied [`REQUIRED_TABLES`] are the whole schema. -fn known_spv_tables() -> Vec<&'static str> { - REQUIRED_TABLES.to_vec() + fn db_label() -> &'static str { + "headers.sqlite" + } + + fn classify_hint() -> &'static str { + classify_hint!(spv_copy_specs) + } + + fn bind_params(&self, bind: SpvBind) -> Result, Error> { + match bind { + SpvBind::BurnHeight => Ok(vec![Value::Integer(u64_to_sql(self.burn_height)?)]), + SpvBind::CompleteChainWorkIntervals => Ok(vec![Value::Integer(u64_to_sql( + num_complete_chain_work_intervals(self.burn_height), + )?)]), + } + } } -/// The spv snapshot's source-schema guard (see [`assert_source_schema`]); -/// `test_no_unclassified_spv_tables` runs it against a fresh schema. +/// The spv snapshot's source-schema guard (see +/// [`DbSnapshotSpec::assert_source_classified`]); `test_no_unclassified_spv_tables` +/// runs it against a fresh schema. The recognized set is independent of +/// `burn_height`. pub(super) fn assert_source_tables_classified(src_conn: &Connection) -> Result<(), Error> { - assert_source_schema( - src_conn, - &known_spv_tables(), - "headers.sqlite", - "REQUIRED_TABLES in snapshot/spv.rs", - ) + SpvDbSnapshotSpec::assert_source_classified(src_conn) } /// Row-count statistics returned by [`copy_spv_headers`]. @@ -81,36 +110,35 @@ pub fn copy_spv_headers( }) } -/// Build the copy specs for the SPV headers DB: `db_config` verbatim, -/// `headers` up to `burn_height`, `chain_work` for complete difficulty -/// intervals only. -pub(super) fn spv_copy_specs(burn_height: u64) -> Vec { - let complete_intervals = num_complete_chain_work_intervals(burn_height); - vec![ - TableCopySpec { - table: "db_config", - source_sql: "SELECT * FROM src.db_config".into(), - }, - TableCopySpec { - table: "headers", - source_sql: format!("SELECT * FROM src.headers WHERE height <= {burn_height}"), - }, - TableCopySpec { - table: "chain_work", - source_sql: format!( - "SELECT * FROM src.chain_work WHERE interval < {complete_intervals}" - ), - }, - ] +/// The copy specs for the SPV headers DB: `db_config` verbatim, `headers` up to +/// `?1` (burn height), `chain_work` for complete difficulty intervals +/// (`interval < ?1`). The runtime values are bound via +/// [`SpvDbSnapshotSpec::bind_params`]. +pub(super) fn spv_copy_specs() -> &'static [TableCopySpec] { + static SPECS: &[TableCopySpec] = &[ + TableCopySpec::sql("db_config", "SELECT * FROM src.db_config"), + TableCopySpec::sql_with_bind( + "headers", + "SELECT * FROM src.headers WHERE height <= ?1", + SpvBind::BurnHeight, + ), + TableCopySpec::sql_with_bind( + "chain_work", + "SELECT * FROM src.chain_work WHERE interval < ?1", + SpvBind::CompleteChainWorkIntervals, + ), + ]; + SPECS } fn copy_spv_headers_inner( conn: &Connection, burn_height: u64, ) -> Result { - clone_schemas_from_source(conn, REQUIRED_TABLES)?; + let spec = SpvDbSnapshotSpec { burn_height }; + clone_schemas_from_source(conn, &SpvDbSnapshotSpec::table_names())?; - let results = execute_copy_specs(conn, &spv_copy_specs(burn_height))?; + let results = spec.run_copy(conn)?; let stats = SpvHeadersCopyStats { headers_rows: copied_rows(&results, "headers"), diff --git a/stackslib/src/chainstate/stacks/db/snapshot/tests/blocks.rs b/stackslib/src/chainstate/stacks/db/snapshot/tests/blocks.rs index 709be1b6cb..94d199791f 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/tests/blocks.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/tests/blocks.rs @@ -30,9 +30,9 @@ use tempfile::tempdir; use super::super::blocks::{ assert_source_tables_classified, copy_confirmed_epoch2_microblocks, copy_epoch2_block_files, - copy_nakamoto_staging_blocks, nakamoto_copy_specs, NAKAMOTO_STAGING_TABLES, + copy_nakamoto_staging_blocks, nakamoto_copy_specs, }; -use super::super::common::clone_schemas_from_source; +use super::super::common::{clone_schemas_from_source, TableCopySpecs}; use super::{ create_dest_db_with_canonical_blocks, create_source_db, insert_epoch2_block_header_with_ibh, insert_nakamoto_header, @@ -161,22 +161,24 @@ fn test_no_unclassified_nakamoto_staging_tables() { .expect("fresh production Nakamoto staging schema must be fully classified"); } -/// Every cloned table (NAKAMOTO_STAGING_TABLES) must have a row-copy spec and vice versa, -/// else it would be cloned but never populated (present-but-empty). +/// Copy-spec coverage guard: no duplicate spec tables, and every Nakamoto +/// staging spec row-copies (none accidentally schema-only). The `known_*` set +/// derives from the specs, so a dropped spec is caught by the +/// unclassified-table guard. #[test] -fn test_nakamoto_copy_specs_match_staging_tables() { - let spec_tables: Vec<&str> = nakamoto_copy_specs().iter().map(|s| s.table).collect(); - let spec_set: HashSet<&str> = spec_tables.iter().copied().collect(); +fn test_nakamoto_copy_specs_well_formed() { + let tables: Vec<&str> = nakamoto_copy_specs().iter().map(|s| s.table).collect(); + let spec_set: HashSet<&str> = tables.iter().copied().collect(); assert_eq!( - spec_tables.len(), + tables.len(), spec_set.len(), "duplicate table in nakamoto_copy_specs" ); - let staging_set: HashSet<&str> = NAKAMOTO_STAGING_TABLES.iter().copied().collect(); - assert_eq!( - spec_set, staging_set, - "every NAKAMOTO_STAGING_TABLES entry must have exactly one copy spec (and vice versa); \ - a cloned-but-uncopied table would be present-but-empty in the squash" + assert!( + TableCopySpecs::new(nakamoto_copy_specs()) + .schema_only() + .is_empty(), + "nakamoto staging has no schema-only tables; every spec must row-copy" ); } diff --git a/stackslib/src/chainstate/stacks/db/snapshot/tests/burnchain.rs b/stackslib/src/chainstate/stacks/db/snapshot/tests/burnchain.rs index 9a37dc1eb2..6fa774a20e 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/tests/burnchain.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/tests/burnchain.rs @@ -24,8 +24,9 @@ use stacks_common::types::chainstate::BurnchainHeaderHash; use tempfile::tempdir; use super::super::burnchain::{ - assert_source_tables_classified, burnchain_copy_specs, copy_burnchain_db, COPIED_TABLES, + assert_source_tables_classified, burnchain_copy_specs, copy_burnchain_db, }; +use super::super::common::TableCopySpecs; use crate::burnchains::db::BurnchainDB; use crate::burnchains::{Burnchain, PoxConstants}; use crate::chainstate::burn::db::sortdb::tests::test_append_snapshot; @@ -78,16 +79,16 @@ fn test_unclassified_source_table_is_rejected() { ); } -/// Every cloned table (`COPIED_TABLES`) must have a row-copy spec and vice versa, -/// else it would be cloned but never populated (present-but-empty). +/// Copy-spec coverage guard: no table appears twice. Set completeness is covered +/// by `test_no_unclassified_burnchain_tables`, which derives the recognized set +/// from these specs and runs the guard against a fresh schema, so re-listing the +/// table names here would just duplicate the spec list. #[test] -fn test_copy_specs_match_copied_tables() { - let copied: HashSet<&str> = COPIED_TABLES.iter().copied().collect(); - - let specs: Vec<&str> = burnchain_copy_specs().iter().map(|s| s.table).collect(); - let spec_set: HashSet<&str> = specs.iter().copied().collect(); - assert_eq!(specs.len(), spec_set.len(), "duplicate spec tables"); - assert_eq!(spec_set, copied); +fn test_burnchain_copy_specs_well_formed() { + let specs = burnchain_copy_specs(); + let tables = TableCopySpecs::new(specs).table_names(); + let unique: HashSet<&str> = tables.iter().copied().collect(); + assert_eq!(tables.len(), unique.len(), "duplicate spec tables"); } /// Create a burnchain.sqlite source via the production initializer @@ -360,8 +361,8 @@ fn test_burnchain_db_block_ops_follow_canonical_headers() { assert_eq!(op, "op_c"); } -/// `anchor_blocks` and `overrides` are restricted to reward cycles -/// referenced by the copied commit metadata. +/// `anchor_blocks` are restricted to reward cycles referenced by the copied +/// commit metadata; `overrides` is schema-only. #[test] fn test_burnchain_db_anchor_blocks_filtered() { let dir = tempdir().unwrap(); diff --git a/stackslib/src/chainstate/stacks/db/snapshot/tests/index.rs b/stackslib/src/chainstate/stacks/db/snapshot/tests/index.rs index d1d8ced57e..d799cc7445 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/tests/index.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/tests/index.rs @@ -13,15 +13,16 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Index side-table (`index.sqlite`) copy/validate tests. +//! Index side-table (`index.sqlite`) copy tests. use std::collections::HashSet; use rusqlite::{params, Connection}; use tempfile::tempdir; +use super::super::common::TableCopySpecs; use super::super::index::{ - assert_source_tables_classified, copy_index_side_tables, index_copy_specs, COPIED_TABLES, + assert_source_tables_classified, copy_index_side_tables, index_copy_specs, }; use super::{ append_canonical_block, assert_corruption_containing, create_dest_db_with_canonical_blocks, @@ -430,17 +431,16 @@ fn test_canonical_block_missing_from_src_is_corruption() { ); } -/// Copy-spec coverage guard: every table in [`COPIED_TABLES`] has exactly -/// one copy spec, so a table can't be classified as copied yet receive no -/// rows - or two specs' worth of duplicates. +/// Copy-spec coverage guard: no table appears twice (which would double-clone / +/// double-copy). Set completeness -- that the specs match the production schema +/// -- is covered by `test_no_unclassified_source_tables`, which derives the +/// recognized set from these specs and runs the guard against a fresh schema, so +/// re-listing the table names here would just duplicate the spec list. #[test] -fn test_copy_specs_match_copied_tables() { - let copied: HashSet<&str> = COPIED_TABLES.iter().copied().collect(); - - let specs: Vec<&str> = index_copy_specs(0).iter().map(|s| s.table).collect(); - let spec_set: HashSet<&str> = specs.iter().copied().collect(); - assert_eq!(specs.len(), spec_set.len(), "duplicate spec tables"); - assert_eq!(spec_set, copied); +fn test_index_copy_specs_well_formed() { + let tables = TableCopySpecs::new(index_copy_specs()).table_names(); + let unique: HashSet<&str> = tables.iter().copied().collect(); + assert_eq!(tables.len(), unique.len(), "duplicate spec tables"); } /// Drift guard: every table the chainstate migrations create must be diff --git a/stackslib/src/chainstate/stacks/db/snapshot/tests/sortition.rs b/stackslib/src/chainstate/stacks/db/snapshot/tests/sortition.rs index 795b729aec..c1dfe70913 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/tests/sortition.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/tests/sortition.rs @@ -26,10 +26,10 @@ use stacks_common::types::chainstate::{ use stacks_common::types::StacksEpochId; use tempfile::tempdir; +use super::super::common::{TableCopySource, TableCopySpecs}; use super::super::sortition::{ assert_source_tables_classified, copy_sortition_side_tables, copy_sortition_side_tables_with_boundary, sortition_copy_specs, SortitionTipCopyBoundary, - REQUIRED_TABLES, }; use super::{hex_id, label_block_id}; use crate::burnchains::PoxConstants; @@ -664,25 +664,72 @@ fn test_no_unclassified_sortition_tables() { .expect("fresh production sortition schema must be fully classified"); } -/// Every table whose schema is cloned ([`REQUIRED_TABLES`]) must also have a -/// row-copy spec, and vice versa. Without this, a table could pass the -/// classification guard and be cloned into the dst yet never populated — -/// recreated-empty inconsistency one level below the unclassified-table check. +/// Copy-spec coverage guard: no duplicate spec tables, and every sortition spec +/// row-copies (none accidentally schema-only). A schema-only sortition table +/// would be cloned into the dst yet never populated (present-but-empty) - the +/// `known_*` set now derives from the specs, so a dropped spec is instead caught +/// by the unclassified-table guard. #[test] -fn test_sortition_copy_specs_match_required_tables() { - let spec_tables: Vec<&str> = sortition_copy_specs(None).iter().map(|s| s.table).collect(); - let spec_set: HashSet<&str> = spec_tables.iter().copied().collect(); +fn test_sortition_copy_specs_well_formed() { + let tables: Vec<&str> = sortition_copy_specs(false) + .iter() + .map(|s| s.table) + .collect(); + let spec_set: HashSet<&str> = tables.iter().copied().collect(); assert_eq!( - spec_tables.len(), + tables.len(), spec_set.len(), "duplicate table in sortition_copy_specs" ); - let required_set: HashSet<&str> = REQUIRED_TABLES.iter().copied().collect(); + assert!( + TableCopySpecs::new(sortition_copy_specs(false)) + .schema_only() + .is_empty(), + "sortition has no schema-only tables; every spec must row-copy" + ); +} + +/// The boundary only selects per-table SQL templates (plain vs anchor-rewrite for +/// the `stacks_chain_tips*` memo tables); it must never change which tables are +/// copied. The plain and rewrite spec lists therefore expose an identical +/// table-name set, and differ only in the SQL of the two memo tables. +#[test] +fn test_sortition_copy_specs_boundary_invariant_table_set() { + let plain = TableCopySpecs::new(sortition_copy_specs(false)); + let rewrite = TableCopySpecs::new(sortition_copy_specs(true)); + assert_eq!( - spec_set, required_set, - "every REQUIRED_TABLES entry must have exactly one copy spec (and vice versa); \ - a cloned-but-uncopied table would be present-but-empty in the squash" + plain.table_names(), + rewrite.table_names(), + "boundary must not change the set of copied tables" ); + + let memo_tables: HashSet<&str> = ["stacks_chain_tips", "stacks_chain_tips_by_burn_view"] + .into_iter() + .collect(); + for (p, r) in plain.iter().zip(rewrite.iter()) { + assert_eq!( + p.table, r.table, + "spec order must be stable across boundary" + ); + let (TableCopySource::Sql(p_sql), TableCopySource::Sql(r_sql)) = (&p.source, &r.source) + else { + panic!("sortition specs are all SQL-sourced"); + }; + if memo_tables.contains(p.table) { + assert_ne!( + p_sql, r_sql, + "{} must use a boundary-specific template", + p.table + ); + } else { + assert_eq!( + p_sql, r_sql, + "{} SQL must not depend on the boundary", + p.table + ); + } + } } /// Runtime guard: a source DB carrying a table the copy lists don't classify diff --git a/stackslib/src/chainstate/stacks/db/snapshot/tests/spv.rs b/stackslib/src/chainstate/stacks/db/snapshot/tests/spv.rs index 1bc0bc714e..a26e299ee2 100644 --- a/stackslib/src/chainstate/stacks/db/snapshot/tests/spv.rs +++ b/stackslib/src/chainstate/stacks/db/snapshot/tests/spv.rs @@ -24,9 +24,8 @@ use stacks_common::deps_common::bitcoin::network::encodable::VarInt; use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash; use tempfile::tempdir; -use super::super::spv::{ - assert_source_tables_classified, copy_spv_headers, spv_copy_specs, REQUIRED_TABLES, -}; +use super::super::common::TableCopySpecs; +use super::super::spv::{assert_source_tables_classified, copy_spv_headers, spv_copy_specs}; use crate::burnchains::bitcoin::spv::{SpvClient, BLOCK_DIFFICULTY_CHUNK_SIZE, SPV_DB_VERSION}; use crate::burnchains::bitcoin::BitcoinNetworkType; use crate::chainstate::stacks::index::Error; @@ -45,22 +44,23 @@ fn test_no_unclassified_spv_tables() { .expect("fresh production SPV schema must be fully classified"); } -/// Every cloned table ([`REQUIRED_TABLES`]) must have a row-copy spec and vice versa, -/// else it would be cloned but never populated (present-but-empty). +/// Copy-spec coverage guard: no duplicate spec tables, and every SPV spec +/// row-copies (none accidentally schema-only). The `known_*` set derives from +/// the specs, so a dropped spec is caught by the unclassified-table guard. #[test] -fn test_spv_copy_specs_match_required_tables() { - let spec_tables: Vec<&str> = spv_copy_specs(100).iter().map(|s| s.table).collect(); - let spec_set: HashSet<&str> = spec_tables.iter().copied().collect(); +fn test_spv_copy_specs_well_formed() { + let tables: Vec<&str> = spv_copy_specs().iter().map(|s| s.table).collect(); + let spec_set: HashSet<&str> = tables.iter().copied().collect(); assert_eq!( - spec_tables.len(), + tables.len(), spec_set.len(), "duplicate table in spv_copy_specs" ); - let required_set: HashSet<&str> = REQUIRED_TABLES.iter().copied().collect(); - assert_eq!( - spec_set, required_set, - "every REQUIRED_TABLES entry must have exactly one copy spec (and vice versa); \ - a cloned-but-uncopied table would be present-but-empty in the squash" + assert!( + TableCopySpecs::new(spv_copy_specs()) + .schema_only() + .is_empty(), + "spv has no schema-only tables; every spec must row-copy" ); }