fix(txn): scope spill keys to the open handle and reclaim orphaned scratch

The transaction sequence restarts at 1 on every open and doubles as the
spill nonce, so a spill key derived only from the durable file id and
that sequence would repeat across two opens of one store at the same
sequence, and the nonce built from it would repeat with it. Mix a
random per-open epoch into the derivation so the key is distinct per
open, leaving the sequence free to serve as the nonce.

Pair this with reclaiming the scratch such keys ever protected: a
transaction dropped without commit or abort cannot await the removal,
so it now hands its exact path to the next begin_write, and whatever
still survives at the next open is swept outright, since a spill file
is never read back across handles.

Author: farhan-syah

src/crypto/kdf.rs

@@ -57,14 +57,35 @@ pub fn derive_hk(mk: &MasterKey) -> Result<DerivedKey> {
 
 /// Derive a transient per-WriteTxn spill key from the master key.
 ///
-/// `info` = `"pagedb/spill/" ‖ file_id[16] ‖ txn_seq.to_le_bytes()[8]` (13 + 16 + 8 = 37 bytes).
-/// The key is transient: the tmp file it protects is discarded at commit/abort.
-pub fn derive_spill_key(mk: &MasterKey, file_id: &[u8; 16], txn_seq: u64) -> Result<DerivedKey> {
+/// `info` = `"pagedb/spill/" ‖ file_id[16] ‖ spill_epoch[16] ‖ txn_seq.to_le_bytes()[8]`
+/// (13 + 16 + 16 + 8 = 53 bytes).
+///
+/// `spill_epoch` is fresh per open handle. `file_id` and the master key are
+/// durable and `txn_seq` restarts at 1 on every open, so those three alone
+/// would repeat the same key across two opens of one store — and the spill
+/// nonce, built from that same restarting sequence, would repeat with it. The
+/// epoch is what keeps each open's spill key distinct, and therefore what makes
+/// the sequence a sound nonce source without a durable anchor.
+///
+/// Nothing reads a spill file written by an earlier open, so there is no
+/// durability cost to a per-open key: the scratch a crash leaves behind is
+/// swept at the next open, never decrypted.
+pub fn derive_spill_key(
+    mk: &MasterKey,
+    file_id: &[u8; 16],
+    spill_epoch: &[u8; 16],
+    txn_seq: u64,
+) -> Result<DerivedKey> {
     const PREFIX: &[u8] = b"pagedb/spill/";
-    let mut info = [0u8; PREFIX.len() + 16 + 8];
-    info[..PREFIX.len()].copy_from_slice(PREFIX);
-    info[PREFIX.len()..PREFIX.len() + 16].copy_from_slice(file_id);
-    info[PREFIX.len() + 16..].copy_from_slice(&txn_seq.to_le_bytes());
+    let mut info = [0u8; PREFIX.len() + 16 + 16 + 8];
+    let mut off = 0;
+    info[off..off + PREFIX.len()].copy_from_slice(PREFIX);
+    off += PREFIX.len();
+    info[off..off + 16].copy_from_slice(file_id);
+    off += 16;
+    info[off..off + 16].copy_from_slice(spill_epoch);
+    off += 16;
+    info[off..off + 8].copy_from_slice(&txn_seq.to_le_bytes());
     expand(mk.as_bytes(), &info)
 }
 
@@ -114,6 +135,36 @@ mod tests {
         assert_ne!(a.as_bytes(), b.as_bytes());
     }
 
+    #[test]
+    fn spill_key_isolates_per_open_at_the_same_sequence() {
+        // The transaction sequence restarts at 1 on every open and also builds
+        // the spill nonce, so two opens of one store must not share a key at
+        // one sequence.
+        let mk = derive_mk(&fixed_kek(), &[0; 16], 0).unwrap();
+        let file_id = [0x5A; 16];
+        let a = derive_spill_key(&mk, &file_id, &[0x01; 16], 1).unwrap();
+        let b = derive_spill_key(&mk, &file_id, &[0x02; 16], 1).unwrap();
+        assert_ne!(a.as_bytes(), b.as_bytes());
+    }
+
+    #[test]
+    fn spill_key_isolates_per_sequence_within_one_open() {
+        let mk = derive_mk(&fixed_kek(), &[0; 16], 0).unwrap();
+        let epoch = [0x01; 16];
+        let a = derive_spill_key(&mk, &[0x5A; 16], &epoch, 1).unwrap();
+        let b = derive_spill_key(&mk, &[0x5A; 16], &epoch, 2).unwrap();
+        assert_ne!(a.as_bytes(), b.as_bytes());
+    }
+
+    #[test]
+    fn spill_key_isolates_per_store() {
+        let mk = derive_mk(&fixed_kek(), &[0; 16], 0).unwrap();
+        let epoch = [0x01; 16];
+        let a = derive_spill_key(&mk, &[0x5A; 16], &epoch, 1).unwrap();
+        let b = derive_spill_key(&mk, &[0xA5; 16], &epoch, 1).unwrap();
+        assert_ne!(a.as_bytes(), b.as_bytes());
+    }
+
     #[test]
     fn dek_isolates_per_realm() {
         let mk = derive_mk(&fixed_kek(), &[0; 16], 0).unwrap();

src/crypto/random/identity.rs

@@ -25,6 +25,18 @@ pub(crate) fn segment_id() -> Result<[u8; 16]> {
     random_nonzero_identity()
 }
 
+/// Identity partitioning the spill nonce space of one open handle.
+///
+/// The per-transaction spill key is derived from the durable `file_id` and the
+/// transaction sequence, and the spill nonce is built from that same sequence.
+/// The sequence restarts at 1 on every open, so without a per-open component
+/// two handles over the same store would encrypt different scratch payloads
+/// under one key at one nonce. Mixing this in makes the key distinct per open,
+/// which is what leaves the sequence free to serve as the nonce.
+pub(crate) fn spill_epoch() -> Result<[u8; 16]> {
+    random_nonzero_identity()
+}
+
 // Sole caller is `txn::db::snapshot::apply_incremental`, which is native-only
 // (`#![cfg(not(target_arch = "wasm32"))]`); gate the function the same way.
 #[cfg(not(target_arch = "wasm32"))]
@@ -41,5 +53,11 @@ mod tests {
         assert_ne!(database_identity().unwrap().0, [0u8; 16]);
         assert_ne!(segment_id().unwrap(), [0u8; 16]);
         assert_ne!(journal_id().unwrap(), [0u8; 16]);
+        assert_ne!(spill_epoch().unwrap(), [0u8; 16]);
+    }
+
+    #[test]
+    fn spill_epochs_do_not_repeat_across_opens() {
+        assert_ne!(spill_epoch().unwrap(), spill_epoch().unwrap());
     }
 }

src/crypto/random/mod.rs

@@ -4,4 +4,4 @@ mod identity;
 
 #[cfg(not(target_arch = "wasm32"))]
 pub(crate) use identity::journal_id;
-pub(crate) use identity::{database_identity, segment_id};
+pub(crate) use identity::{database_identity, segment_id, spill_epoch};

src/recovery/mod.rs

@@ -1,10 +1,11 @@
 //! Open-flow recovery: apply-journal replay, catalog reconciliation,
-//! tombstone GC.
+//! tombstone GC, spill-scratch reclamation.
 
 pub(crate) mod deep_walk;
 pub(crate) mod gc;
 pub(crate) mod journal;
 pub(crate) mod reconcile;
+pub(crate) mod scratch;
 #[cfg(test)]
 mod tests;
 

src/recovery/scratch.rs

@@ -0,0 +1,74 @@
+//! Reclamation of write-transaction spill scratch.
+//!
+//! A spill file is scratch for one live write transaction and is never read
+//! back by a later one: its key is scoped to the open handle that wrote it. So
+//! every scratch file present at open belongs to a handle that is already gone,
+//! and is reclaimable without inspection.
+
+use crate::Result;
+use crate::vfs::Vfs;
+
+/// Directory holding per-transaction spill scratch.
+const SCRATCH_DIR: &str = "tmp";
+
+/// Prefix of a spill scratch file name, completed by the transaction sequence.
+const SCRATCH_PREFIX: &str = "scratch-";
+
+/// Delete the spill scratch left by handles that closed without cleaning up.
+///
+/// Callers must hold write authority over the store: this removes files, and a
+/// handle that only observes must not. Returns the number of files deleted.
+pub async fn delete_orphaned_scratch<V: Vfs + Clone>(vfs: &V) -> Result<u64> {
+    let entries = vfs.list_dir(SCRATCH_DIR).await?;
+    let mut count: u64 = 0;
+    for name in entries {
+        // Only the names this crate writes. `tmp/` is a shared directory in the
+        // store layout, and a sweep that removed everything under it would
+        // reclaim files belonging to whatever is added there next.
+        if !name.starts_with(SCRATCH_PREFIX) {
+            continue;
+        }
+        vfs.remove(&format!("{SCRATCH_DIR}/{name}")).await?;
+        count += 1;
+    }
+    if count > 0 {
+        // Durable directory entries: a crash between here and the next open
+        // would otherwise present the same scratch to be swept again.
+        vfs.sync_dir(SCRATCH_DIR).await?;
+    }
+    Ok(count)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::vfs::VfsFile;
+    use crate::vfs::memory::MemVfs;
+    use crate::vfs::types::OpenMode;
+
+    async fn write(vfs: &MemVfs, path: &str) {
+        vfs.mkdir_all(SCRATCH_DIR).await.unwrap();
+        let mut f = vfs.open(path, OpenMode::CreateNew).await.unwrap();
+        f.write_at(0, b"scratch").await.unwrap();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn sweeps_scratch_and_spares_unrelated_names() {
+        let vfs = MemVfs::new();
+        write(&vfs, "tmp/scratch-1").await;
+        write(&vfs, "tmp/scratch-9").await;
+        write(&vfs, "tmp/unrelated").await;
+
+        assert_eq!(delete_orphaned_scratch(&vfs).await.unwrap(), 2);
+
+        assert!(vfs.open("tmp/scratch-1", OpenMode::Read).await.is_err());
+        assert!(vfs.open("tmp/scratch-9", OpenMode::Read).await.is_err());
+        assert!(vfs.open("tmp/unrelated", OpenMode::Read).await.is_ok());
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn absent_scratch_directory_is_not_an_error() {
+        let vfs = MemVfs::new();
+        assert_eq!(delete_orphaned_scratch(&vfs).await.unwrap(), 0);
+    }
+}

src/txn/db/core.rs

@@ -177,6 +177,17 @@ pub struct Db<V: Vfs + Clone> {
     /// Each txn gets `txn_seq.fetch_add(1, Relaxed)` (first txn gets 1 since
     /// we start at 0 and add-then-use the pre-increment value + 1).
     pub(crate) txn_seq: AtomicU64,
+    /// Random per-open identity mixed into every spill key derivation. The
+    /// counter above restarts at 1 on each open and also supplies the spill
+    /// nonce, so this is what stops two opens of one store from encrypting
+    /// different scratch under the same key at the same nonce.
+    pub(crate) spill_epoch: [u8; 16],
+    /// Scratch paths belonging to write transactions that were dropped without
+    /// `commit` or `abort`. `Drop` cannot await the VFS removal, so it records
+    /// the exact path here and the next `begin_write` — which holds the writer
+    /// lock and can await — removes it. Anything still listed when the handle
+    /// closes is swept by the next open.
+    pub(crate) orphaned_spill_paths: parking_lot::Mutex<Vec<String>>,
     /// The mode this handle was opened with (Standalone, Follower, `ReadOnly`, Observer).
     pub(crate) mode: DbMode,
     /// Set of reader `entry_id`s that have been aborted by `AbortOldest` stall

src/txn/db/open/create.rs

@@ -42,7 +42,7 @@ struct FreshDbState<V: Vfs + Clone> {
 }
 
 impl<V: Vfs + Clone> Db<V> {
-    fn assemble_fresh(state: FreshDbState<V>) -> Self {
+    fn assemble_fresh(state: FreshDbState<V>) -> Result<Self> {
         let FreshDbState {
             pager,
             realm_id,
@@ -83,7 +83,7 @@ impl<V: Vfs + Clone> Db<V> {
                 seq: initial.seq,
             },
         );
-        Self {
+        Ok(Self {
             pager: Arc::new(pager),
             realm_id,
             page_size,
@@ -109,6 +109,8 @@ impl<V: Vfs + Clone> Db<V> {
             mmap_bytes_in_use: Arc::new(AtomicU64::new(0)),
             spill_bytes_in_use: AtomicU64::new(0),
             txn_seq: AtomicU64::new(0),
+            spill_epoch: crate::crypto::random::spill_epoch()?,
+            orphaned_spill_paths: parking_lot::Mutex::new(Vec::new()),
             mode: DbMode::Standalone,
             aborted_readers: parking_lot::Mutex::new(std::collections::HashSet::new()),
             sentinel_locks: Vec::new(),
@@ -132,7 +134,7 @@ impl<V: Vfs + Clone> Db<V> {
             visibility_test_hook: parking_lot::Mutex::new(None),
             #[cfg(test)]
             rekey_test_fault: parking_lot::Mutex::new(None),
-        }
+        })
     }
 
     /// Bootstrap a fresh database. Creates `main.db`, writes an initial A/B
@@ -278,7 +280,7 @@ impl<V: Vfs + Clone> Db<V> {
         let vfs_for_pager = V::clone(&*vfs_arc);
         let pager = Pager::open(vfs_for_pager, mk, cfg).await?;
 
-        Ok(Self::assemble_fresh(FreshDbState {
+        Self::assemble_fresh(FreshDbState {
             pager,
             realm_id: realm,
             page_size,
@@ -291,6 +293,6 @@ impl<V: Vfs + Clone> Db<V> {
             file_id,
             kek_salt,
             initial,
-        }))
+        })
     }
 }

src/txn/db/open/existing.rs

@@ -274,6 +274,8 @@ impl<V: Vfs + Clone> Db<V> {
             mmap_bytes_in_use: Arc::new(AtomicU64::new(0)),
             spill_bytes_in_use: AtomicU64::new(0),
             txn_seq: AtomicU64::new(0),
+            spill_epoch: crate::crypto::random::spill_epoch()?,
+            orphaned_spill_paths: parking_lot::Mutex::new(Vec::new()),
             mode,
             aborted_readers: parking_lot::Mutex::new(std::collections::HashSet::new()),
             sentinel_locks: Vec::new(),

src/txn/db/open/recovery.rs

@@ -32,6 +32,14 @@ pub(super) async fn recover_open_state<V: Vfs + Clone>(
             Err(PagedbError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {}
             Err(error) => return Err(error),
         }
+
+        // Write-transaction spill scratch is keyed to the handle that wrote it,
+        // so anything still present belongs to a handle that is gone and is
+        // reclaimable outright. Without this the scratch of every transaction
+        // that died with its process would accumulate for the life of the
+        // store. Gated with the removals above because it needs write
+        // authority.
+        crate::recovery::scratch::delete_orphaned_scratch(&*db.vfs).await?;
     }
 
     // An incremental apply assembles its target beside `main.db` and renames it

src/txn/write/spill.rs

@@ -39,17 +39,22 @@ pub(crate) struct SpillSegmentMeta {
 /// storage in a per-transaction tmp file.
 ///
 /// The tmp file lives at `tmp/scratch-<txn_seq>`. It is created lazily on the
-/// first `append` call and removed at `commit` or `abort` (best-effort).
+/// first `append` call and removed at `commit` or `abort`. A transaction
+/// dropped without either records its path for the next `begin_write` to
+/// remove, and whatever a crash leaves behind is swept at the next open.
 ///
 /// Nonce scheme: `txn_seq_le6 ‖ segment_index_le6`.
 /// - First 6 bytes: the low 6 bytes of `txn_seq` in little-endian order.
 /// - Last 6 bytes: the low 6 bytes of the per-append segment index in
 ///   little-endian order.
 ///
-/// This guarantees uniqueness across all appends within a txn and across
-/// independent txns (different `txn_seq`), without requiring a durable
-/// nonce anchor (the tmp file is discarded before any subsequent txn can
-/// reuse the same `txn_seq`).
+/// Uniqueness under one key comes from `txn_seq` being strictly increasing
+/// within an open handle and `segment_index` within a transaction. It does not
+/// come from the file being deleted: `txn_seq` restarts at 1 on every open, so
+/// deletion alone would leave two opens writing one nonce. The spill key is
+/// instead derived per open, from a random epoch this handle generates and
+/// never persists, so a repeated sequence lands under a different key and no
+/// durable nonce anchor is needed.
 pub struct SpillScope<'scope, 'db, V: Vfs + Clone> {
     pub(super) txn: &'scope mut WriteTxn<'db, V>,
 }
@@ -212,7 +217,12 @@ impl<'db, V: Vfs + Clone> WriteTxn<'db, V> {
     pub(crate) fn ensure_spill_cipher(&mut self) -> Result<&Cipher> {
         if self.spill_cipher.is_none() {
             let pager_mk = self.db.pager.mk()?;
-            let key = derive_spill_key(&pager_mk, &self.db.file_id, self.txn_seq)?;
+            let key = derive_spill_key(
+                &pager_mk,
+                &self.db.file_id,
+                &self.db.spill_epoch,
+                self.txn_seq,
+            )?;
             self.spill_cipher = Some(Cipher::new_aes_gcm(&key));
         }
         self.spill_cipher.as_ref().ok_or(PagedbError::NotFound)
@@ -238,10 +248,10 @@ impl<'db, V: Vfs + Clone> WriteTxn<'db, V> {
     /// Build the per-append nonce deterministically from `(txn_seq, segment_index)`.
     ///
     /// Layout: `txn_seq_le6 ‖ segment_index_le6` (6 + 6 = 12 bytes).
-    /// Uniqueness: nonces are unique per-txn (distinct `segment_index`) and
-    /// across txns (distinct `txn_seq` in first 6 bytes). No durable anchor
-    /// is needed because the tmp file is discarded before any key reuse
-    /// could matter.
+    /// Uniqueness within one key: distinct `segment_index` within a
+    /// transaction, distinct `txn_seq` across the transactions of one open
+    /// handle. `txn_seq` repeats across opens, which is why the key is scoped
+    /// to an open rather than to the store; see the type-level docs.
     pub(crate) fn next_spill_nonce(&self, segment_index: u64) -> Nonce {
         let mut bytes = [0u8; 12];
         let seq_le = self.txn_seq.to_le_bytes();