Skip to content

Commit 45c33d1

Browse files
committed
fix(raft): deduplicate votes by peer ID and make election timeout configurable
Use a HashSet keyed by peer ID in handle_request_vote_response so that duplicate grants from the same peer cannot inflate the vote count and cause premature leader promotion. Add election_timeout_min/max fields to ClusterConfig and propagate them through bootstrap, join, and restart paths. Expose matching fields in ClusterTransportTuning (default 2–5 s) and wire them into MultiRaft so every startup path honours the configured window rather than using a hardcoded constant. Adjust network tuning defaults from 60/120 s to 2/5 s to match real-world cluster behaviour.
1 parent 89071f5 commit 45c33d1

9 files changed

Lines changed: 31 additions & 11 deletions

File tree

nodedb-cluster/src/bootstrap/bootstrap_fn.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ pub(super) fn bootstrap(config: &ClusterConfig, catalog: &ClusterCatalog) -> Res
3535
);
3636

3737
// Create MultiRaft with all groups (single-node, no peers).
38-
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone());
38+
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone())
39+
.with_election_timeout(config.election_timeout_min, config.election_timeout_max);
3940
for group_id in routing.group_ids() {
4041
multi_raft.add_group(group_id, vec![])?;
4142
}
@@ -81,6 +82,7 @@ fn generate_cluster_id() -> u64 {
8182
mod tests {
8283
use super::*;
8384
use crate::catalog::ClusterCatalog;
85+
use std::time::Duration;
8486

8587
fn temp_catalog() -> (tempfile::TempDir, ClusterCatalog) {
8688
let dir = tempfile::tempdir().unwrap();
@@ -102,6 +104,8 @@ mod tests {
102104
force_bootstrap: false,
103105
join_retry: Default::default(),
104106
swim_udp_addr: None,
107+
election_timeout_min: Duration::from_millis(150),
108+
election_timeout_max: Duration::from_millis(300),
105109
};
106110

107111
let state = bootstrap(&config, &catalog).unwrap();

nodedb-cluster/src/bootstrap/config.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,10 @@ pub struct ClusterConfig {
9191
/// [`crate::spawn_swim`] after the cluster is up and feed the
9292
/// seed list from `seed_nodes`.
9393
pub swim_udp_addr: Option<SocketAddr>,
94+
/// Raft election timeout range. Controls how long a follower waits
95+
/// before starting an election after losing contact with the leader.
96+
pub election_timeout_min: Duration,
97+
pub election_timeout_max: Duration,
9498
}
9599

96100
/// Result of cluster startup — everything needed to run the Raft loop.

nodedb-cluster/src/bootstrap/join.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,8 @@ fn apply_join_response(
288288
// learners). A learner-started group boots in the `Learner`
289289
// role and will not run an election until a subsequent
290290
// `PromoteLearner` conf change is applied.
291-
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone());
291+
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone())
292+
.with_election_timeout(config.election_timeout_min, config.election_timeout_max);
292293
for g in &resp.groups {
293294
let is_voter = g.members.contains(&config.node_id);
294295
let is_learner = g.learners.contains(&config.node_id);
@@ -450,6 +451,8 @@ mod tests {
450451
force_bootstrap: false,
451452
join_retry: Default::default(),
452453
swim_udp_addr: None,
454+
election_timeout_min: Duration::from_millis(150),
455+
election_timeout_max: Duration::from_millis(300),
453456
};
454457
let state1 = bootstrap(&config1, &catalog1).unwrap();
455458

@@ -499,6 +502,8 @@ mod tests {
499502
force_bootstrap: false,
500503
join_retry: Default::default(),
501504
swim_udp_addr: None,
505+
election_timeout_min: Duration::from_millis(150),
506+
election_timeout_max: Duration::from_millis(300),
502507
};
503508

504509
let lifecycle = ClusterLifecycleTracker::new();

nodedb-cluster/src/bootstrap/probe.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,8 @@ mod tests {
223223
force_bootstrap: false,
224224
join_retry: Default::default(),
225225
swim_udp_addr: None,
226+
election_timeout_min: Duration::from_millis(150),
227+
election_timeout_max: Duration::from_millis(300),
226228
}
227229
}
228230

nodedb-cluster/src/bootstrap/restart.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ pub(super) fn restart(
3535
// as a learner on restart; dropping the group entirely would
3636
// leave the node permanently without any copy of it and
3737
// silently broken.
38-
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone());
38+
let mut multi_raft = MultiRaft::new(config.node_id, routing.clone(), config.data_dir.clone())
39+
.with_election_timeout(config.election_timeout_min, config.election_timeout_max);
3940
for (group_id, info) in routing.group_members() {
4041
let is_voter = info.members.contains(&config.node_id);
4142
let is_learner = info.learners.contains(&config.node_id);
@@ -91,6 +92,7 @@ mod tests {
9192
use super::super::bootstrap_fn::bootstrap;
9293
use super::*;
9394
use crate::catalog::ClusterCatalog;
95+
use std::time::Duration;
9496

9597
fn temp_catalog() -> (tempfile::TempDir, ClusterCatalog) {
9698
let dir = tempfile::tempdir().unwrap();
@@ -112,6 +114,8 @@ mod tests {
112114
force_bootstrap: false,
113115
join_retry: Default::default(),
114116
swim_udp_addr: None,
117+
election_timeout_min: Duration::from_millis(150),
118+
election_timeout_max: Duration::from_millis(300),
115119
};
116120

117121
// Bootstrap first.

nodedb-cluster/src/multi_raft/core.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ impl MultiRaft {
7777
node_id,
7878
groups: HashMap::new(),
7979
routing,
80-
election_timeout_min: Duration::from_millis(150),
81-
election_timeout_max: Duration::from_millis(300),
80+
election_timeout_min: Duration::from_secs(2),
81+
election_timeout_max: Duration::from_secs(5),
8282
heartbeat_interval: Duration::from_millis(50),
8383
data_dir,
8484
}

nodedb-raft/src/node/core.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
//! replication) live in [`super::internal`]. RPC handlers live in
66
//! [`super::rpc`].
77
8+
use std::collections::HashSet;
89
use std::time::Instant;
910

1011
use crate::error::{RaftError, Result};
@@ -61,7 +62,7 @@ pub struct RaftNode<S: LogStorage> {
6162
/// When the next heartbeat should be sent (leader only).
6263
pub(super) heartbeat_deadline: Instant,
6364
/// Votes received in current election.
64-
pub(super) votes_received: Vec<u64>,
65+
pub(super) votes_received: HashSet<u64>,
6566
/// Pending ready output.
6667
pub(super) ready: Ready,
6768
/// Known leader ID (0 = unknown).
@@ -89,7 +90,7 @@ impl<S: LogStorage> RaftNode<S> {
8990
leader_state: None,
9091
election_deadline: now + config.election_timeout_max,
9192
heartbeat_deadline: now,
92-
votes_received: Vec::new(),
93+
votes_received: HashSet::new(),
9394
ready: Ready::default(),
9495
leader_id: 0,
9596
config,

nodedb-raft/src/node/rpc.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ impl<S: LogStorage> RaftNode<S> {
166166
}
167167

168168
/// Handle RequestVote response (candidate only).
169-
pub fn handle_request_vote_response(&mut self, _peer: u64, resp: &RequestVoteResponse) {
169+
pub fn handle_request_vote_response(&mut self, peer: u64, resp: &RequestVoteResponse) {
170170
if resp.term > self.hard_state.current_term {
171171
self.become_follower(resp.term);
172172
return;
@@ -177,7 +177,7 @@ impl<S: LogStorage> RaftNode<S> {
177177
}
178178

179179
if resp.vote_granted {
180-
self.votes_received.push(resp.term);
180+
self.votes_received.insert(peer);
181181
let vote_count = self.votes_received.len() + 1; // +1 for self-vote
182182

183183
if vote_count >= self.config.quorum() {

nodedb-types/src/config/tuning/network.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,10 @@ fn default_raft_tick_interval_ms() -> u64 {
223223
10
224224
}
225225
fn default_election_timeout_min_secs() -> u64 {
226-
60
226+
2
227227
}
228228
fn default_election_timeout_max_secs() -> u64 {
229-
120
229+
5
230230
}
231231
fn default_rpc_timeout_secs() -> u64 {
232232
5

0 commit comments

Comments
 (0)