Skip to content

Commit f75e705

Browse files
committed
feat(data): add per-core write-version index keyed by WAL LSN
Thread the WAL LSN the Control Plane allocates at wal-dispatch time through the Request envelope and ExecutionTask onto every Data Plane apply chokepoint (point/bulk KV, document, graph, columnar family, transaction handlers). Each chokepoint records the committed write into a new per-core WriteVersionIndex, tracking the last write LSN per key and per collection, with horizon GC bounding the per-key map. This is the shard-local write-version substrate an upcoming optimistic-concurrency commit path will validate transaction read-sets against; nothing reads the index yet.
1 parent f5d1c2c commit f75e705

101 files changed

Lines changed: 1589 additions & 208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ impl TestClusterNode {
5555
user_id: None,
5656
statement_digest: None,
5757
txn_id: None,
58+
wal_lsn: None,
5859
};
5960

6061
// Register for response routing before dispatching, then submit through
@@ -124,6 +125,7 @@ impl TestClusterNode {
124125
user_id: None,
125126
statement_digest: None,
126127
txn_id: None,
128+
wal_lsn: None,
127129
};
128130

129131
let mut rx = self.shared.tracker.register(request_id);

nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ impl TestClusterNode {
5555
user_id: None,
5656
statement_digest: None,
5757
txn_id: None,
58+
wal_lsn: None,
5859
};
5960

6061
let mut rx = self.shared.tracker.register(request_id);
@@ -115,6 +116,7 @@ impl TestClusterNode {
115116
user_id: None,
116117
statement_digest: None,
117118
txn_id: None,
119+
wal_lsn: None,
118120
};
119121

120122
let mut rx = self.shared.tracker.register(request_id);

nodedb-test-support/src/tx_batch_helpers.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ pub fn make_request(plan: PhysicalPlan) -> Request {
5656
user_id: None,
5757
statement_digest: None,
5858
txn_id: None,
59+
wal_lsn: None,
5960
}
6061
}
6162

nodedb/src/bridge/dispatch.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,7 @@ mod tests {
509509
user_id: None,
510510
statement_digest: None,
511511
txn_id: None,
512+
wal_lsn: None,
512513
}
513514
}
514515

@@ -537,6 +538,7 @@ mod tests {
537538
user_id: None,
538539
statement_digest: None,
539540
txn_id: None,
541+
wal_lsn: None,
540542
}
541543
}
542544

nodedb/src/bridge/envelope.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,17 @@ pub struct Request {
140140
/// keys the per-transaction staging overlay. `None` for autocommit /
141141
/// non-transactional / system requests.
142142
pub txn_id: Option<TxnId>,
143+
144+
/// WAL LSN the Control Plane allocated for this write at wal-dispatch time.
145+
/// The committed write-LSN is part of the cross-plane write contract: the
146+
/// Data Plane copies it onto the [`ExecutionTask`] so the apply chokepoint
147+
/// records the per-key / per-collection write version (see
148+
/// `data::executor::core_loop::write_index`). `None` for reads, control
149+
/// ops, and writes whose LSN is not (yet) threaded — the version index is
150+
/// skipped rather than advanced with a wrong value.
151+
///
152+
/// [`ExecutionTask`]: crate::data::executor::task::ExecutionTask
153+
pub wal_lsn: Option<Lsn>,
143154
}
144155

145156
/// Response envelope: Data Plane -> Control Plane.
@@ -379,6 +390,7 @@ mod tests {
379390
user_id: None,
380391
statement_digest: None,
381392
txn_id: None,
393+
wal_lsn: None,
382394
}
383395
}
384396

@@ -447,6 +459,7 @@ mod tests {
447459
user_id: None,
448460
statement_digest: None,
449461
txn_id: None,
462+
wal_lsn: None,
450463
};
451464
match req.plan {
452465
PhysicalPlan::Meta(MetaOp::Cancel { target_request_id }) => {

nodedb/src/control/array_sync/raft_apply.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,7 @@ fn build_array_request(
424424
user_id: None,
425425
statement_digest: None,
426426
txn_id: None,
427+
wal_lsn: None,
427428
}
428429
}
429430

nodedb/src/control/catalog_entry/post_apply/async_dispatch/continuous_aggregate.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ async fn dispatch_meta(
102102
user_id: None,
103103
statement_digest: None,
104104
txn_id: None,
105+
wal_lsn: None,
105106
};
106107
let rx = shared.tracker.register(request_id);
107108
if d.dispatch_to_core(core_id, request).is_err() {

nodedb/src/control/catalog_entry/post_apply/async_dispatch/materialized_view.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ pub async fn delete_async(tenant_id: u64, name: String, shared: Arc<SharedState>
5050
user_id: None,
5151
statement_digest: None,
5252
txn_id: None,
53+
wal_lsn: None,
5354
};
5455
let rx = shared.tracker.register(request_id);
5556
if d.dispatch_to_core(core_id, request).is_err() {

nodedb/src/control/checkpoint_manager.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ pub async fn run_checkpoint_cycle(
124124
user_id: None,
125125
statement_digest: None,
126126
txn_id: None,
127+
wal_lsn: None,
127128
};
128129

129130
let rx = tracker.register(request_id);

nodedb/src/control/clone/copyup.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ pub async fn perform_kv_clone_copyup(params: KvCopyUpParams<'_>) -> crate::Resul
9595
user_id: None,
9696
statement_digest: None,
9797
txn_id: None,
98+
wal_lsn: None,
9899
};
99100

100101
let mut rx = state.tracker.register(req_id);
@@ -248,6 +249,7 @@ pub async fn perform_clone_copyup(params: CopyUpParams<'_>) -> crate::Result<Sur
248249
user_id: None,
249250
statement_digest: None,
250251
txn_id: None,
252+
wal_lsn: None,
251253
};
252254

253255
let mut rx = state.tracker.register(req_id);

0 commit comments

Comments
 (0)