Demo upgrade to New Protocol + Fixes (#4406)

* add upgrade test for new proto

* fix test

* fix cutover

* harden upgrade

* reduce comments

Author: bfish713

.env

@@ -170,7 +170,9 @@ ESPRESSO_BUILDER_WEBSERVER_RESPONSE_TIMEOUT_DURATION=1500ms
 ESPRESSO_BUILDER_BUFFER_VIEW_NUM_COUNT=50
 
 # Load generator
-ESPRESSO_SUBMIT_TRANSACTIONS_DELAY=2s
+# Mean submit delay defaults to 2s (set per-process in process-compose.yaml as
+# ${ESPRESSO_SUBMIT_TRANSACTIONS_DELAY:-2s} so tests can override it). Keeping the
+# default out of this file avoids a .env-vs-env precedence ambiguity for the override.
 ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT=24010
 ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT=24020
 ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_FALLBACK_PORT=24030

.github/workflows/test.yml

@@ -360,6 +360,7 @@ jobs:
           - test-name: test_native_demo_drb_header_upgrade
           - test-name: test_native_demo_fee_to_drb_header_upgrade
           - test-name: test_native_demo_epoch_reward_upgrade
+          - test-name: test_native_demo_new_protocol_upgrade
           - test-name: test_native_demo_da_committee
           - test-name: test_native_demo_ff_base
 

crates/espresso/node/src/consensus_handle.rs

@@ -11,7 +11,7 @@ use hotshot_new_protocol::{
     coordinator::{Coordinator, CoordinatorOutput, error::Severity},
     cutover::{
         CutoverGate, extract_pre_cutover_seed, forward_legacy_epoch_changes,
-        forward_legacy_timeout_votes,
+        forward_legacy_high_qc, forward_legacy_timeout_votes,
     },
     network::Network,
     state::UpdateLeaf,
@@ -155,6 +155,10 @@ where
             legacy_event_rx.clone(),
             client_api.clone(),
         ));
+        spawn(forward_legacy_high_qc(
+            legacy_event_rx.clone(),
+            client_api.clone(),
+        ));
         spawn(forward_legacy_epoch_changes(
             legacy_event_rx.clone(),
             client_api.clone(),

crates/espresso/node/src/context.rs

@@ -206,15 +206,7 @@ where
             .membership_coordinator(membership_coordinator.clone())
             .network(coordinator_network)
             .initializer(&initializer_for_coordinator)
-            .upgrade_lock({
-                // TODO: The Coordinator and HotShot each create their own UpgradeLock
-                // from the same inputs. They need to share a single lock so that upgrade
-                // certificate updates are visible to both.
-                UpgradeLock::from_certificate(
-                    upgrade,
-                    &initializer_for_coordinator.decided_upgrade_certificate,
-                )
-            })
+            .upgrade_lock(handle.hotshot.upgrade_lock.clone())
             .public_key(validator_config.public_key)
             .private_key(validator_config.private_key.clone())
             .state_private_key(validator_config.state_private_key.clone())

crates/hotshot/new-protocol/src/client.rs

@@ -5,6 +5,7 @@ use committable::Commitment;
 use hotshot_types::{
     data::{EpochNumber, Leaf2, ViewNumber},
     message::Proposal as SignedProposal,
+    simple_certificate::QuorumCertificate2,
     simple_vote::TimeoutVote2,
     traits::{leaf_fetcher_network::LeafFetcherNetwork, node_implementation::NodeType},
     utils::StateAndDelta,
@@ -123,6 +124,14 @@ impl<T: NodeType> ClientApi<T> {
             .await
     }
 
+    /// Forward the last legacy view's QC so the first new-protocol leader can
+    /// propose on it even if the cutover seed was snapshotted before it formed.
+    pub async fn submit_legacy_high_qc(&self, qc: QuorumCertificate2<T>) -> Result<(), QueryError> {
+        let (respond, rx) = oneshot::channel();
+        self.call(ClientRequest::SubmitLegacyHighQc { qc, respond }, rx)
+            .await
+    }
+
     /// Refresh the coordinator network's peer set for `epoch`.
     pub async fn bump_network_epoch(&self, epoch: EpochNumber) -> Result<(), QueryError> {
         let (respond, rx) = oneshot::channel();
@@ -226,6 +235,10 @@ pub(crate) enum ClientRequest<T: NodeType> {
         vote: TimeoutVote2<T>,
         respond: oneshot::Sender<()>,
     },
+    SubmitLegacyHighQc {
+        qc: QuorumCertificate2<T>,
+        respond: oneshot::Sender<()>,
+    },
     BumpNetworkEpoch {
         epoch: EpochNumber,
         respond: oneshot::Sender<()>,

crates/hotshot/new-protocol/src/consensus.rs

@@ -304,16 +304,21 @@ impl<T: NodeType> Consensus<T> {
         let view = seed.decided_anchor.view_number();
         if view > self.last_decided_view {
             self.last_decided_view = view;
-            self.last_decided_leaf = seed.decided_anchor;
+            self.last_decided_leaf = seed.decided_anchor.clone();
         }
 
+        let mut highest_seeded_block: u64 = seed.decided_anchor.block_header().block_number();
+
         for leaf in seed.undecided {
             let view = leaf.view_number();
             let justify_qc = leaf.justify_qc().clone();
             self.register_legacy_qc(&justify_qc);
 
             let block_number = leaf.block_header().block_number();
             let epoch = EpochNumber::new(epoch_from_block_number(block_number, *self.epoch_height));
+            if block_number > highest_seeded_block {
+                highest_seeded_block = block_number;
+            }
 
             let view_change_evidence = leaf.view_change_evidence.clone().and_then(|e| match e {
                 ViewChangeEvidence2::Timeout(tc) => Some(tc),
@@ -355,6 +360,13 @@ impl<T: NodeType> Consensus<T> {
         if last_pre_cutover > self.current_view {
             self.current_view = last_pre_cutover;
         }
+        let seeded_epoch = EpochNumber::new(epoch_from_block_number(
+            highest_seeded_block,
+            *self.epoch_height,
+        ));
+        if self.current_epoch.is_none_or(|cur| cur < seeded_epoch) {
+            self.current_epoch = Some(seeded_epoch);
+        }
     }
 
     /// Register `justify_qc` as Cert1 for its parent view (idempotent)

crates/hotshot/new-protocol/src/coordinator.rs

@@ -1315,6 +1315,41 @@ where
                 }
                 let _ = respond.send(());
             },
+            ClientRequest::SubmitLegacyHighQc { qc, respond } => {
+                // QC certifies the last legacy view; cutover view is the next.
+                // Register idempotently so the smooth-start precondition holds
+                // regardless of arrival order vs. the cutover seed.
+                let qc_view = qc.view_number();
+                let cutover_view = qc_view + 1;
+                self.consensus.register_legacy_qc(&qc);
+
+                // Still parked on the last legacy view (seed landed without this
+                // QC, waiting out the timer) and not yet skipped via TC2: propose
+                // the cutover view on the real QC now. Self-idempotent — once
+                // started, `cur_view` advances past `qc_view` and `maybe_propose`
+                // dedups by `proposed_views`.
+                let cur_view = self.consensus.current_view();
+                if cur_view == qc_view
+                    && self.consensus.timeout_cert_at(cutover_view).is_none()
+                    && self.consensus.cert1_at(qc_view).is_some()
+                    && self.consensus.proposal_at(qc_view).is_some()
+                {
+                    tracing::info!(
+                        %cutover_view,
+                        "bridged late legacy high QC; proposing cutover view on it (no timeout)"
+                    );
+                    self.start();
+                    while let Some(output) = self.outbox.pop_front() {
+                        if let Err(err) = self.process_consensus_output(output) {
+                            tracing::warn!(
+                                %err,
+                                "error processing bridged-high-qc bootstrap output"
+                            );
+                        }
+                    }
+                }
+                let _ = respond.send(());
+            },
             ClientRequest::BumpNetworkEpoch { epoch, respond } => {
                 if let Err(err) = self
                     .network

crates/hotshot/new-protocol/src/cutover.rs

@@ -159,6 +159,23 @@ pub async fn forward_legacy_timeout_votes<T: NodeType>(
     }
 }
 
+/// Forward the last legacy view's QC into the coordinator, so the cutover-view
+/// leader can propose on it instead of waiting out a timeout when the cutover
+/// seed was snapshotted before the QC formed.
+pub async fn forward_legacy_high_qc<T: NodeType>(
+    legacy_event_rx: InactiveReceiver<Event<T>>,
+    client_api: ClientApi<T>,
+) {
+    let mut rx = legacy_event_rx.activate_cloned();
+    while let Some(event) = rx.next().await {
+        if let EventType::LegacyHighQcFormed { qc } = event.event
+            && let Err(err) = client_api.submit_legacy_high_qc(qc).await
+        {
+            tracing::warn!(%err, "failed to forward legacy high QC to new-protocol coordinator");
+        }
+    }
+}
+
 /// Forward legacy epoch transitions into `bump_network_epoch`.
 /// `epoch_height == 0` disables forwarding.
 pub async fn forward_legacy_epoch_changes<T: NodeType>(

crates/hotshot/new-protocol/src/tests/legacy_cutover.rs

@@ -51,7 +51,7 @@ use versions::{NEW_PROTOCOL_VERSION, Upgrade, version};
 use crate::{
     consensus::ConsensusOutput,
     coordinator::{Coordinator, CoordinatorOutput, error::Severity, timer::Timer},
-    cutover::{CutoverGate, forward_legacy_timeout_votes},
+    cutover::{CutoverGate, forward_legacy_high_qc, forward_legacy_timeout_votes},
     helpers::test_upgrade_lock,
     network::cliquenet::Cliquenet,
     outbox::Outbox,
@@ -421,11 +421,14 @@ async fn spawn_node(
     let legacy_event_rx = legacy.read().await.event_stream_known_impl().deactivate();
     bg_handles.push(
         tokio::spawn(forward_legacy_timeout_votes(
-            legacy_event_rx,
+            legacy_event_rx.clone(),
             client_api.clone(),
         ))
         .abort_handle(),
     );
+    bg_handles.push(
+        tokio::spawn(forward_legacy_high_qc(legacy_event_rx, client_api.clone())).abort_handle(),
+    );
 
     let (decision_tx, decision_rx) = mpsc::unbounded_channel::<DecisionEvent>();
     let runner_abort = tokio::spawn(run_cutover_node(
@@ -670,14 +673,16 @@ const PREDICTED_CUTOVER_VIEW: u64 = UPGRADE_VIEW + 20;
 /// subset of `{V-2, V-1, V, V+1, V+2}`.
 const V: u64 = PREDICTED_CUTOVER_VIEW - 1;
 
-/// Happy path. `cutover_view - 1` reliably has no QC at cutover, so
-/// the new protocol skips it via TC2.
+/// Happy path. The QC for `cutover_view - 1` forms in the legacy protocol and
+/// rides into the new protocol via the cutover seed's `high_qc`, so the first
+/// leader proposes directly on it (no TC2 skip, no timer wait) and every view
+/// is decided.
 #[tokio::test(flavor = "multi_thread")]
 async fn legacy_runs_upgrade_then_new_protocol_takes_over() {
     run_cutover_test(
         4,
         6,
-        views([PREDICTED_CUTOVER_VIEW - 1]),
+        BTreeSet::new(),
         Duration::from_secs(180),
         DEFAULT_NEW_PROTO_VIEW_TIMEOUT,
         Vec::new(),

crates/hotshot/task-impls/src/consensus/mod.rs

@@ -14,7 +14,7 @@ use hotshot_types::{
     consensus::OuterConsensus,
     data::{EpochNumber, ViewNumber},
     epoch_membership::EpochMembershipCoordinator,
-    event::Event,
+    event::{Event, EventType},
     message::UpgradeLock,
     simple_certificate::{NextEpochQuorumCertificate2, QuorumCertificate2, TimeoutCertificate2},
     simple_vote::{HasEpoch, NextEpochQuorumVote2, QuorumVote2, TimeoutVote2},
@@ -35,7 +35,7 @@ use self::handlers::{
 };
 use crate::{
     events::HotShotEvent,
-    helpers::{broadcast_view_change, validate_qc_and_next_epoch_qc},
+    helpers::{broadcast_event, broadcast_view_change, validate_qc_and_next_epoch_qc},
     vote_collection::{EpochRootVoteCollectorsMap, VoteCollectorsMap},
 };
 
@@ -261,6 +261,32 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>> ConsensusTaskState<TYPES, I>
                     .await;
                 }
             },
+            HotShotEvent::Qc2Formed(either::Left(qc))
+                if self.upgrade_lock.new_protocol_active(self.cur_view)
+                    && !self.upgrade_lock.new_protocol_active(qc.view_number()) =>
+            {
+                // Cutover boundary only: the gated proposal path won't land this
+                // last-legacy QC in `high_qc`, so capture it here for
+                // `extract_pre_cutover_seed` to carry across. `update_high_qc` is
+                // monotone, so this is a no-op outside the cutover window.
+                let mut consensus_writer = self.consensus.write().await;
+                let _ = consensus_writer.update_high_qc(qc.clone());
+                drop(consensus_writer);
+                if let Err(e) = self.storage.update_high_qc2(qc.clone()).await {
+                    tracing::warn!("Failed to persist boundary high QC: {e}");
+                }
+                // Forward to the espresso bridge -> new-protocol coordinator, in case the
+                // cutover seed was snapshotted before this QC finished assembling. Only the
+                // cutover-view leader reaches this arm, so it lands exactly where needed.
+                broadcast_event(
+                    Event {
+                        view_number: qc.view_number(),
+                        event: EventType::LegacyHighQcFormed { qc: qc.clone() },
+                    },
+                    &self.output_event_stream,
+                )
+                .await;
+            },
             _ => {},
         }
 
@@ -279,7 +305,24 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>> TaskState for ConsensusTaskS
         _receiver: &Receiver<Arc<Self::Event>>,
     ) -> Result<()> {
         if self.upgrade_lock.new_protocol_active(self.cur_view) {
-            return Ok(());
+            // Past cutover: still admit votes/QCs for strictly-pre-cutover views so
+            // the cutover leader can finish the last legacy QC; everything else
+            // (proposing, voting, view changes) stays shut down.
+            let admit = match event.as_ref() {
+                HotShotEvent::QuorumVoteRecv(vote) => {
+                    !self.upgrade_lock.new_protocol_active(vote.view_number())
+                },
+                HotShotEvent::EpochRootQuorumVoteRecv(vote) => {
+                    !self.upgrade_lock.new_protocol_active(vote.view_number())
+                },
+                HotShotEvent::Qc2Formed(either::Left(qc)) => {
+                    !self.upgrade_lock.new_protocol_active(qc.view_number())
+                },
+                _ => false,
+            };
+            if !admit {
+                return Ok(());
+            }
         }
         self.handle(event, sender.clone()).await
     }