[backport release-mainnet-1.2.0-rc] Hash id migration

Cargo.toml

@@ -357,6 +357,7 @@ snafu = "0.8"
 snow = { version = "0.10.0", features = ["ring-accelerated"] }
 sqlx = { version = "=0.8.3", features = [
     "bit-vec",
+    "chrono",
     "postgres",
     "runtime-tokio",
     "tls-native-tls",

crates/espresso/node/api/database.toml

@@ -1,3 +1,13 @@
+[route.get_migration_status]
+PATH = ["migration-status"]
+METHOD = "GET"
+DOC = """
+Get the status of all deferred background migrations.
+
+Returns a list of migrations with their name, when they started, when they completed (if done),
+and the last processed offset (useful for estimating progress).
+"""
+
 [route.get_table_sizes]
 PATH = ["table-sizes"]
 METHOD = "GET"

crates/espresso/node/api/migrations/postgres/V1501__hash_bigint_expand.sql

@@ -0,0 +1,47 @@
+-- Expand phase: create new BIGINT-keyed tables alongside the existing ones.
+-- Old tables (hash, fee_merkle_tree, block_merkle_tree) are left untouched and
+-- serve as the read fallback during the backfill window.
+-- The contract phase (dropping old tables and renaming *_bigint to canonical names)
+-- is a separate follow-up migration.
+
+-- Drop reward merkle tree tables — unused and always empty across all deployments.
+DROP TABLE reward_merkle_tree;
+DROP TABLE reward_merkle_tree_v2;
+
+-- New hash table with BIGSERIAL.
+CREATE TABLE hash_bigint (
+    id    BIGSERIAL PRIMARY KEY,
+    value BYTEA NOT NULL UNIQUE
+);
+
+-- Seed the sequence above the old max so new auto-IDs never collide
+-- with rows we backfill from hash (which preserve their original INT ids).
+SELECT setval(
+    pg_get_serial_sequence('hash_bigint', 'id'),
+    GREATEST(COALESCE((SELECT MAX(id) FROM hash), 1), 1)
+);
+
+-- New Merkle-tree tables with BIGINT FK.
+CREATE TABLE fee_merkle_tree_bigint (
+    path            JSONB        NOT NULL,
+    created         BIGINT       NOT NULL,
+    hash_id         BIGINT       NOT NULL REFERENCES hash_bigint(id),
+    children        JSONB,
+    children_bitvec BIT VARYING,
+    idx             JSONB,
+    entry           JSONB,
+    PRIMARY KEY (path, created)
+);
+CREATE INDEX fee_merkle_tree_bigint_created ON fee_merkle_tree_bigint (created);
+
+CREATE TABLE block_merkle_tree_bigint (
+    path            JSONB        NOT NULL,
+    created         BIGINT       NOT NULL,
+    hash_id         BIGINT       NOT NULL REFERENCES hash_bigint(id),
+    children        JSONB,
+    children_bitvec BIT VARYING,
+    idx             JSONB,
+    entry           JSONB,
+    PRIMARY KEY (path, created)
+);
+CREATE INDEX block_merkle_tree_bigint_created ON block_merkle_tree_bigint (created);

crates/espresso/node/api/migrations/postgres/V1502__deferred_migrations.sql

@@ -0,0 +1,8 @@
+-- Progress-tracking table for background DataBackfill migrations.
+CREATE TABLE deferred_migrations (
+    name         TEXT        PRIMARY KEY,
+    started_at   TIMESTAMPTZ NOT NULL,
+    completed_at TIMESTAMPTZ,
+    error        TEXT,
+    last_offset  BIGINT
+);

crates/espresso/node/api/migrations/sqlite/V1501__hash_bigint_expand.sql

@@ -0,0 +1,9 @@
+-- Rename tables to *_bigint for consistency with the postgres schema.
+-- SQLite integers are already 64-bit so this is a naming-only change.
+ALTER TABLE hash              RENAME TO hash_bigint;
+ALTER TABLE fee_merkle_tree   RENAME TO fee_merkle_tree_bigint;
+ALTER TABLE block_merkle_tree RENAME TO block_merkle_tree_bigint;
+
+-- Drop reward merkle tree tables — unused and always empty across all deployments.
+DROP TABLE IF EXISTS reward_merkle_tree;
+DROP TABLE IF EXISTS reward_merkle_tree_v2;

crates/espresso/node/api/migrations/sqlite/V1502__deferred_migrations.sql

@@ -0,0 +1,8 @@
+-- Progress-tracking table for background DataBackfill migrations.
+CREATE TABLE deferred_migrations (
+    name         TEXT     PRIMARY KEY,
+    started_at   TEXT     NOT NULL,
+    completed_at TEXT,
+    error        TEXT,
+    last_offset  INTEGER
+);

crates/espresso/node/api/public-env-vars.toml

@@ -53,6 +53,7 @@ variables = [
     "ESPRESSO_NODE_API_PEERS",
     "ESPRESSO_NODE_API_PORT",
     "ESPRESSO_NODE_ARCHIVE",
+    "ESPRESSO_NODE_BACKFILL_BATCH_DELAY_MS",
     "ESPRESSO_NODE_BOOTSTRAP_EPOCH_CATCHUP_TIMEOUT",
     "ESPRESSO_NODE_CATCHUP_BACKOFF_FACTOR",
     "ESPRESSO_NODE_CATCHUP_BACKOFF_JITTER",

crates/espresso/node/src/api.rs

@@ -837,6 +837,10 @@ where
     async fn get_table_sizes(&self) -> anyhow::Result<Vec<data_source::TableSize>> {
         self.inner().get_table_sizes().await
     }
+
+    async fn get_migration_status(&self) -> anyhow::Result<Vec<data_source::MigrationStatus>> {
+        self.inner().get_migration_status().await
+    }
 }
 
 impl<N: ConnectedNetwork<PubKey>, P: SequencerPersistence, D: CatchupStorage + Send + Sync>
@@ -6021,7 +6025,6 @@ mod test {
     }
 
     #[rstest]
-    #[case(POS_V3)]
     #[case(POS_V4)]
     #[test_log::test(tokio::test(flavor = "multi_thread"))]
     async fn test_state_reconstruction(#[case] upgrade: Upgrade) -> anyhow::Result<()> {
@@ -7655,7 +7658,6 @@ mod test {
     }
 
     #[rstest]
-    #[case(POS_V3)]
     #[case(POS_V4)]
     #[test_log::test(tokio::test(flavor = "multi_thread"))]
     async fn test_reward_proof_endpoint(#[case] upgrade: Upgrade) -> anyhow::Result<()> {

crates/espresso/node/src/api/data_source.rs

@@ -388,12 +388,26 @@ pub struct TableSize {
     pub total_size_bytes: Option<i64>,
 }
 
+/// Status of a single deferred background migration.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct MigrationStatus {
+    pub name: String,
+    pub started_at: chrono::DateTime<chrono::Utc>,
+    pub completed_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub last_offset: Option<i64>,
+}
+
 /// Data source for database metadata and statistics.
 ///
 /// This trait is only implemented by SQL-based storage backends (PostgreSQL and SQLite).
 pub(crate) trait DatabaseMetadataSource {
     /// Get the sizes of all tables in the database.
     fn get_table_sizes(&self) -> impl Send + Future<Output = anyhow::Result<Vec<TableSize>>>;
+
+    /// Get the status of all deferred background migrations.
+    fn get_migration_status(
+        &self,
+    ) -> impl Send + Future<Output = anyhow::Result<Vec<MigrationStatus>>>;
 }
 
 // ============================================================================

crates/espresso/node/src/api/endpoints.rs

@@ -754,6 +754,17 @@ where
                     .map_err(|err| Error::internal(format!("failed to get table sizes: {err:#}")))
             }
             .boxed()
+        })?
+        .at("get_migration_status", |_req, state| {
+            async move {
+                state
+                    .read(|state| state.get_migration_status().boxed())
+                    .await
+                    .map_err(|err| {
+                        Error::internal(format!("failed to get migration status: {err:#}"))
+                    })
+            }
+            .boxed()
         })?;
 
     Ok(api)