Skip to content

Commit 65121d5

Browse files
committed
chore(cluster): log descriptor version and drain lifecycle details
A persistent descriptor mismatch signals a planner/leaseholder disagreement rather than the transient race the retry path assumes, but the expected/actual versions were dropped before reaching the gateway error. Log them at both the exec receiver and the gateway, and trace drain install/release so a stuck drain is diagnosable from logs instead of only from its TTL expiry.
1 parent 2c97001 commit 65121d5

3 files changed

Lines changed: 33 additions & 3 deletions

File tree

nodedb/src/control/exec_receiver/executor.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,12 @@ impl LocalPlanExecutor {
146146
stored.descriptor_version
147147
};
148148
if actual != entry.version {
149+
tracing::debug!(
150+
collection = %entry.collection,
151+
expected_version = entry.version,
152+
actual_version = actual,
153+
"exec receiver: descriptor version mismatch"
154+
);
149155
return Err(TypedClusterError::DescriptorMismatch {
150156
collection: entry.collection.clone(),
151157
expected_version: entry.version,

nodedb/src/control/gateway/dispatcher.rs

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,26 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) ->
321321
leader_node: leader_node_id.unwrap_or(0),
322322
leader_addr: leader_addr.unwrap_or_default(),
323323
},
324-
TypedClusterError::DescriptorMismatch { collection, .. } => Error::RetryableSchemaChanged {
325-
descriptor: collection,
326-
},
324+
TypedClusterError::DescriptorMismatch {
325+
collection,
326+
expected_version,
327+
actual_version,
328+
} => {
329+
// The versions are the whole diagnosis for this error: a mismatch
330+
// that keeps repeating means the planner and the leaseholder
331+
// disagree persistently, which is a bug rather than the transient
332+
// race the retry assumes. Dropping them leaves "retryable schema
333+
// change" with nothing to act on.
334+
tracing::debug!(
335+
%collection,
336+
expected_version,
337+
actual_version,
338+
"gateway: descriptor version mismatch at leaseholder"
339+
);
340+
Error::RetryableSchemaChanged {
341+
descriptor: collection,
342+
}
343+
}
327344
TypedClusterError::DeadlineExceeded { .. } => Error::DeadlineExceeded {
328345
request_id: crate::types::RequestId::new(0),
329346
},

nodedb/src/control/lease/drain.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ impl DescriptorDrainTracker {
6868
/// Called by the metadata applier on every node when a
6969
/// `DescriptorDrainStart` raft entry commits.
7070
pub fn install_start(&self, id: DescriptorId, up_to_version: u64, expires_at: Hlc) {
71+
tracing::debug!(
72+
?id,
73+
up_to_version,
74+
expires_wall_ns = expires_at.wall_ns,
75+
"drain: install_start"
76+
);
7177
let mut map = self.active.write().unwrap_or_else(|p| p.into_inner());
7278
map.insert(
7379
id,
@@ -83,6 +89,7 @@ impl DescriptorDrainTracker {
8389
/// raft entries AND on the implicit clear path that runs
8490
/// after a successful `Put*` apply.
8591
pub fn install_end(&self, id: &DescriptorId) {
92+
tracing::debug!(?id, "drain: install_end");
8693
let mut map = self.active.write().unwrap_or_else(|p| p.into_inner());
8794
map.remove(id);
8895
}

0 commit comments

Comments
 (0)