Skip to content

Commit 31afe38

Browse files
authored
Merge pull request #27 from mkhairi/fix/allocate-page-quadratic-scan
perf(btree): decide page-reuse eligibility at free time, not per allocation
2 parents 8217994 + b76708f commit 31afe38

11 files changed

Lines changed: 436 additions & 116 deletions

File tree

src/btree/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@ pub(crate) mod tree;
1212

1313
// `pub` here is crate-scoped in effect: the `btree` module itself is
1414
// `pub(crate)`, so this does not escape the crate.
15-
pub use tree::BTree;
15+
pub use tree::{BTree, ConsumedPages, PageSource};

src/btree/tree/core.rs

Lines changed: 256 additions & 87 deletions
Large diffs are not rendered by default.

src/btree/tree/flush.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,10 +100,13 @@ impl<V: Vfs> BTree<V> {
100100

101101
/// Drain and return all `page_ids` that were freed during this tree's
102102
/// mutation session, plus any leaf pages scheduled for freeing at the
103-
/// upcoming `flush`. After this call, both `self.freed` and
104-
/// `self.scheduled_frees` are empty.
103+
/// upcoming `flush`. After this call, `self.freed`, `self.freed_reusable`
104+
/// and `self.scheduled_frees` are all empty — a page eligible for
105+
/// in-session reuse that no allocation claimed is still a page this session
106+
/// freed, and the deferred-free queue must hear about it.
105107
pub fn drain_freed(&mut self) -> Vec<u64> {
106108
let mut out = std::mem::take(&mut self.freed);
109+
out.append(&mut self.freed_reusable);
107110
out.append(&mut self.scheduled_frees);
108111
out
109112
}

src/btree/tree/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ pub(crate) mod core;
55
pub(crate) mod flush;
66
pub(crate) mod maintenance;
77
pub(crate) mod navigate;
8+
pub(crate) mod page_source;
89
pub(crate) mod read;
910
pub(crate) mod scan;
1011
pub(crate) mod write;
1112

1213
pub use core::BTree;
14+
pub use page_source::{ConsumedPages, PageSource};

src/btree/tree/page_source.rs

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
//! Where a write session's pages come from, supplied to a tree as one value.
2+
//!
3+
//! Reuse eligibility is decided when a page is freed, not when one is
4+
//! allocated (see [`BTree::free_page`](super::core::BTree::free_page)). That is
5+
//! only sound while every input to the decision is already in place at the
6+
//! first free and cannot move afterwards, so the inputs travel together in
7+
//! [`PageSource`] and are handed over once, at
8+
//! [`BTree::open_session`](super::core::BTree::open_session). There is no
9+
//! setter for any of them: a tree either has a session's page source for its
10+
//! whole life or has none and bump-allocates.
11+
//!
12+
//! [`ConsumedPages`] carries the other half of that guarantee. A page recorded
13+
//! there is eligible for in-session reuse, and pages already banked on the
14+
//! strength of that record cannot be un-banked — so the sink must not shrink
15+
//! while a session is running. Trees therefore hold a [`ConsumedHandle`], which
16+
//! can record and test but cannot empty; emptying is [`ConsumedPages::take`],
17+
//! reachable only from whoever owns the sink, at a transaction boundary.
18+
19+
use std::sync::Arc;
20+
21+
use rustc_hash::FxHashSet;
22+
23+
/// Page ids the allocator drew from the shared free-page cache and reused this
24+
/// session. The commit path removes them from the durable free-list — they now
25+
/// hold live committed data — and [`BTree::free_page`] consults them: a page
26+
/// drawn from the cache is free as of the last durable header regardless of
27+
/// what this session wrote to it, so it may be recycled again in-session even
28+
/// from below the reuse threshold.
29+
///
30+
/// A set, not a list: membership is tested once per freed page, and the size is
31+
/// bounded by the free-list window (`WINDOW_PAGES × chain_capacity(page_size)`)
32+
/// rather than by anything small — a linear test would keep the per-free cost
33+
/// proportional to that window. Nothing reads insertion order or needs
34+
/// duplicates.
35+
///
36+
/// [`BTree::free_page`]: super::core::BTree::free_page
37+
#[derive(Debug, Default)]
38+
pub struct ConsumedPages(parking_lot::Mutex<FxHashSet<u64>>);
39+
40+
impl ConsumedPages {
41+
/// An empty sink, ready to be shared with a session's trees via
42+
/// [`Self::handle`].
43+
#[must_use]
44+
pub fn new() -> Arc<Self> {
45+
Arc::new(Self::default())
46+
}
47+
48+
/// A tree's view of this sink: record and test, never empty. Handing trees
49+
/// this instead of the `Arc` is what makes the sink's monotonicity within a
50+
/// session a property of the types rather than of caller discipline.
51+
#[must_use]
52+
pub fn handle(self: &Arc<Self>) -> ConsumedHandle {
53+
ConsumedHandle(Arc::clone(self))
54+
}
55+
56+
/// Empty the sink and return what it held.
57+
///
58+
/// A transaction-boundary operation, and the only way to empty it. Calling
59+
/// this while a session's trees are still allocating would strip the record
60+
/// that made already-banked pages eligible, leaving them banked on a reason
61+
/// that no longer exists — which is why no [`ConsumedHandle`] can reach it.
62+
pub fn take(&self) -> FxHashSet<u64> {
63+
std::mem::take(&mut *self.0.lock())
64+
}
65+
}
66+
67+
/// A tree's handle on a session's [`ConsumedPages`]. Clone-shared across the
68+
/// trees of one transaction.
69+
#[derive(Clone, Debug)]
70+
pub struct ConsumedHandle(Arc<ConsumedPages>);
71+
72+
impl ConsumedHandle {
73+
pub(crate) fn record(&self, page_id: u64) {
74+
self.0.0.lock().insert(page_id);
75+
}
76+
77+
pub(crate) fn contains(&self, page_id: u64) -> bool {
78+
self.0.0.lock().contains(&page_id)
79+
}
80+
}
81+
82+
/// The pages a write session may allocate from, beyond bumping its own cursor.
83+
///
84+
/// Constructed once per session and cloned to each of its trees, so a tree can
85+
/// never observe a partly-wired source — a cache without the sink that records
86+
/// draws from it, or a threshold arriving after pages have already been
87+
/// classified against the default.
88+
#[derive(Clone, Debug)]
89+
pub struct PageSource {
90+
/// Minimum `page_id` that may be recycled from within this session. Pages
91+
/// below it existed before the session began: they are still referenced by
92+
/// the last durable header (a copy-on-write free does not unreference them
93+
/// on disk until the *next* header lands) and may also be pinned by reader
94+
/// snapshots, so their bytes must survive until the header that frees them
95+
/// is durable. They re-enter circulation through the deferred-free queue.
96+
///
97+
/// Callers pass the session-start `next_page_id`, so only pages
98+
/// bump-allocated within the session are recyclable immediately. Zero means
99+
/// every freed page is (the commit-history tree, whose pages no reader
100+
/// snapshot names).
101+
pub(crate) reuse_threshold: u64,
102+
/// Cross-commit pool of reusable page ids, shared across the main, catalog
103+
/// and history trees of one `Db`. Loaded at `begin_write` with *only*
104+
/// free-list pages below the reclamation floor — pages no live reader and
105+
/// no retained-history root can observe — so drawing from it is safe
106+
/// regardless of `reuse_threshold`.
107+
///
108+
/// Draw-only: nothing may push into it. The commit deletes a recycled id
109+
/// from the chain by finding it in the scanned window, so an id pushed in
110+
/// from elsewhere would be handed out while the unscanned tail still named
111+
/// it — the same page id given to two live structures.
112+
pub(crate) cache: Arc<parking_lot::Mutex<Vec<u64>>>,
113+
/// Where draws from `cache` are recorded.
114+
pub(crate) consumed: ConsumedHandle,
115+
}
116+
117+
impl PageSource {
118+
/// Bind a session's allocation state to one value. `reuse_threshold` is the
119+
/// session-start `next_page_id` for trees a reader may hold a snapshot of,
120+
/// and zero for those none can.
121+
#[must_use]
122+
pub fn new(
123+
reuse_threshold: u64,
124+
cache: Arc<parking_lot::Mutex<Vec<u64>>>,
125+
consumed: &Arc<ConsumedPages>,
126+
) -> Self {
127+
Self {
128+
reuse_threshold,
129+
cache,
130+
consumed: consumed.handle(),
131+
}
132+
}
133+
}

src/txn/db/catalog.rs

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,6 @@ impl<V: Vfs + Clone> Db<V> {
232232
readers.iter().map(|r| r.commit_id.0).min()
233233
};
234234

235-
let mut hist_tree = BTree::open(
236-
self.pager.clone(),
237-
self.realm_id,
238-
state.commit_history_root_page_id,
239-
state.next_page_id,
240-
self.page_size,
241-
);
242235
// The commit-history tree is not part of any reader's pinned snapshot
243236
// (readers track the data and catalog roots, never the history root), so
244237
// every page its copy-on-write/prune frees is immediately reusable
@@ -250,9 +243,18 @@ impl<V: Vfs + Clone> Db<V> {
250243
// pushes into it. Its own frees leave through `drain_freed` into the
251244
// commit's entry set instead, so no page reaches an allocator without
252245
// the window entry that names it having been located first.
253-
hist_tree.set_reuse_threshold(0);
254-
hist_tree.set_free_page_cache(self.free_page_cache.clone());
255-
hist_tree.set_free_page_consumed(self.free_page_consumed.clone());
246+
let mut hist_tree = BTree::open_session(
247+
self.pager.clone(),
248+
self.realm_id,
249+
state.commit_history_root_page_id,
250+
state.next_page_id,
251+
self.page_size,
252+
crate::btree::PageSource::new(
253+
0,
254+
self.free_page_cache.clone(),
255+
&self.free_page_consumed,
256+
),
257+
);
256258

257259
// Insert the new entry.
258260
let key = new_commit_id.to_be_bytes().to_vec();

src/txn/db/core.rs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,10 +231,15 @@ pub struct Db<V: Vfs + Clone> {
231231
/// history) draw from the same pool. Cleared by `compact_now`'s full repack,
232232
/// which relocates pages and invalidates every cached id.
233233
pub(crate) free_page_cache: Arc<parking_lot::Mutex<Vec<u64>>>,
234-
/// Per-txn sink (cleared at `begin_write`) recording page ids the allocator
234+
/// Per-txn sink (emptied at `begin_write`) recording page ids the allocator
235235
/// drew from `free_page_cache`. The commit path removes them from the
236236
/// durable free-list — they now hold live committed data.
237-
pub(crate) free_page_consumed: Arc<parking_lot::Mutex<Vec<u64>>>,
237+
///
238+
/// The trees see it through a handle that cannot empty it: a page banked
239+
/// for in-session reuse on the strength of a record here must not have that
240+
/// record withdrawn under it, so emptying belongs to the transaction
241+
/// boundaries alone.
242+
pub(crate) free_page_consumed: Arc<crate::btree::ConsumedPages>,
238243
#[cfg(test)]
239244
pub(crate) visibility_test_hook: parking_lot::Mutex<Option<Arc<VisibilityTestHook>>>,
240245
#[cfg(test)]

src/txn/db/open/create.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ impl<V: Vfs + Clone> Db<V> {
130130
})),
131131
poisoned_commit: parking_lot::Mutex::new(None),
132132
free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())),
133-
free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())),
133+
free_page_consumed: crate::btree::ConsumedPages::new(),
134134
#[cfg(test)]
135135
visibility_test_hook: parking_lot::Mutex::new(None),
136136
#[cfg(test)]

src/txn/db/open/existing.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ impl<V: Vfs + Clone> Db<V> {
321321
})),
322322
poisoned_commit: parking_lot::Mutex::new(None),
323323
free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())),
324-
free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())),
324+
free_page_consumed: crate::btree::ConsumedPages::new(),
325325
#[cfg(test)]
326326
visibility_test_hook: parking_lot::Mutex::new(None),
327327
#[cfg(test)]

src/txn/write/commit.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,9 @@ impl<V: Vfs + Clone> WriteTxn<'_, V> {
144144
.collect();
145145
// Pages the allocator reused from the cache this txn — they were free
146146
// before, now hold live committed data, so they leave the free-list.
147-
let consumed: HashSet<u64> = std::mem::take(&mut *self.db.free_page_consumed.lock())
148-
.into_iter()
149-
.collect();
147+
// Emptied here rather than mid-session: the trees have all drained, so
148+
// nothing can still be banking pages against it.
149+
let consumed = self.db.free_page_consumed.take();
150150

151151
// The scanned window after this commit: its entries minus the ones
152152
// reused, kept with their original freeing-commit tag. Every id in

0 commit comments

Comments
 (0)