apollo_network: backoff bootstrap re-dial on connection close (#13941)

Author: sirandreww-starkware

crates/apollo_network/src/discovery/behaviours/bootstrapping/bootstrap_peer.rs

@@ -84,7 +84,14 @@ impl BootstrapPeerEventStream {
             }) if peer_id == self.peer_id && remaining_established == 0 => {
                 self.dial_mode = DialMode::Disconnected;
                 self.should_add_peer_to_kad_routing_table = true;
-                self.time_for_next_bootstrap_dial = now;
+                let delta_duration = self
+                    .dial_retry_strategy
+                    .next()
+                    .expect("Dial sleep strategy ended even though it's an infinite iterator.");
+                self.time_for_next_bootstrap_dial = now + delta_duration;
+                // Drop any sleeper armed for the previous deadline so poll_next builds a fresh
+                // one against the new time_for_next_bootstrap_dial.
+                self.sleeper = None;
                 self.wake_if_needed();
             }
             FromSwarm::AddressChange(AddressChange { peer_id, .. }) if peer_id == self.peer_id => {

crates/apollo_network/src/discovery/behaviours/bootstrapping/bootstrap_test.rs

@@ -337,6 +337,7 @@ async fn bootstrapping_redials_when_all_connections_closed(
     let (bootstrap_peers, mut behaviour) = make_and_connect_bootstrap_nodes(peer_count).await;
 
     close_all_connections(&mut behaviour, &bootstrap_peers, 0);
+    assert_no_event_happens_before_duration(&mut behaviour, BOOTSTRAP_DIAL_SLEEP_BASE).await;
     consume_dial_events(&mut behaviour, bootstrap_peers.clone()).await;
     assert_no_event_happens_before_duration(&mut behaviour, BOOTSTRAP_DIAL_SLEEP_MAX * 2).await;
 }
@@ -374,14 +375,17 @@ async fn bootstrapping_redials_in_accordance_with_strategy_when_all_connections_
     assert_no_event_happens_before_duration(&mut behaviour, BOOTSTRAP_DIAL_SLEEP_MAX * 2).await;
 
     close_all_connections(&mut behaviour, &bootstrap_peers, 0);
+    assert_no_event_happens_before_duration(&mut behaviour, BOOTSTRAP_DIAL_SLEEP_BASE).await;
     consume_dial_events(&mut behaviour, bootstrap_peers.clone()).await;
-    assert_no_event_happens_before_duration(&mut behaviour, BOOTSTRAP_DIAL_SLEEP_MAX * 2).await;
+    assert_no_event(&mut behaviour);
 
     for i in 1..=NUMBER_OF_DIAL_RETRIES {
         fail_all_dial_attempts(&mut behaviour, &bootstrap_peers);
+        // ConnectionClosed already consumed one strategy step, so iteration `i` of this loop
+        // observes BOOTSTRAP_DIAL_SLEEP_BASE_MILLIS.pow(i + 1).
         let current_retry_duration = min(
             BOOTSTRAP_DIAL_SLEEP_MAX,
-            Duration::from_millis(BOOTSTRAP_DIAL_SLEEP_BASE_MILLIS.pow(i)),
+            Duration::from_millis(BOOTSTRAP_DIAL_SLEEP_BASE_MILLIS.pow(i + 1)),
         );
         assert_no_event_happens_before_duration(&mut behaviour, current_retry_duration).await;
         consume_dial_events(&mut behaviour, bootstrap_peers.clone()).await;

crates/apollo_network/src/discovery/mod.rs

@@ -200,7 +200,7 @@ pub struct RetryConfig {
 
 impl Default for RetryConfig {
     fn default() -> Self {
-        Self { base_delay_millis: 2, max_delay_seconds: Duration::from_secs(5), factor: 5 }
+        Self { base_delay_millis: 1000, max_delay_seconds: Duration::from_secs(5), factor: 5 }
     }
 }
 

crates/apollo_node/resources/config_schema.json

@@ -2537,7 +2537,7 @@
   "consensus_manager_config.network_config.discovery_config.bootstrap_dial_retry_config.base_delay_millis": {
     "description": "The base delay in milliseconds for the exponential backoff strategy.",
     "privacy": "Public",
-    "value": 2
+    "value": 1000
   },
   "consensus_manager_config.network_config.discovery_config.bootstrap_dial_retry_config.factor": {
     "description": "The factor for the exponential backoff strategy.",
@@ -3177,7 +3177,7 @@
   "mempool_p2p_config.network_config.discovery_config.bootstrap_dial_retry_config.base_delay_millis": {
     "description": "The base delay in milliseconds for the exponential backoff strategy.",
     "privacy": "Public",
-    "value": 2
+    "value": 1000
   },
   "mempool_p2p_config.network_config.discovery_config.bootstrap_dial_retry_config.factor": {
     "description": "The factor for the exponential backoff strategy.",
@@ -3477,7 +3477,7 @@
   "state_sync_config.static_config.network_config.discovery_config.bootstrap_dial_retry_config.base_delay_millis": {
     "description": "The base delay in milliseconds for the exponential backoff strategy.",
     "privacy": "Public",
-    "value": 2
+    "value": 1000
   },
   "state_sync_config.static_config.network_config.discovery_config.bootstrap_dial_retry_config.factor": {
     "description": "The factor for the exponential backoff strategy.",