perf: incremental corpus counters, sort-index inheritance, and steadier graph cache [user-impact-checked]

Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: XNLLLLH <XNLLLLH@users.noreply.github.com>

Author: 3 people

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.1] — 2026-07-10
+
+### Changed
+
+- **Lower idle CPU.** Corpus counters now update incrementally instead of
+  rescanning, fresh readers inherit the writer's sort index instead of
+  rebuilding it, and the graph cache no longer re-streams the whole corpus on
+  ambient captures — cutting background rebuild churn substantially.
+
 ## [2.0.0] — 2026-07-10
 
 ### Changed

crates/lilliengine/src/conn.rs

@@ -94,6 +94,12 @@ pub struct PersistedTableIndex {
     /// admits a built one into `id_caches`.
     #[serde(default)]
     pub id_index: IdIndex,
+    /// The built ORDER BY index, persisted under the SAME fingerprint so a cold
+    /// process serves its first ordered scan from the sidecar instead of paying
+    /// the population scan. Unbuilt when the writer never ran an ordered query
+    /// on the table; the load path only admits a built one into `ordered_caches`.
+    #[serde(default)]
+    pub ordered_index: OrderedColIndex,
 }
 
 /// A transferable snapshot of built index pairs (Arc-backed — cloning copies
@@ -102,7 +108,7 @@ pub struct PersistedTableIndex {
 /// by [`Connection::adopt_built_indexes`] on a freshly-opened reader.
 #[derive(Debug, Clone, Default)]
 pub struct IndexSnapshot {
-    tables: Vec<(String, i64, ColIndex, IdIndex)>,
+    tables: Vec<(String, i64, ColIndex, IdIndex, OrderedColIndex)>,
 }
 
 /// A single result column's description, mirroring the DB-API `description`
@@ -399,7 +405,8 @@ impl Connection {
         // `open_read_only` (both route through `from_store`), so the daemon-down
         // RO recall reader loads-not-rebuilds for free. The id-index rides the
         // same gate, so a warm boot serves `WHERE id = ?` with no build scan.
-        let (mut col_caches, mut id_caches) = load_col_indexes(&store, &meta, &catalog, &root_map);
+        let (mut col_caches, mut id_caches, ordered_caches) =
+            load_col_indexes(&store, &meta, &catalog, &root_map);
         // A read-write connection maintains its indexes per committed write, so
         // it must own private postings: materialize the copy-on-write clone HERE,
         // once, on the open (boot) path — never lazily on the first awake-path
@@ -459,7 +466,7 @@ impl Connection {
             id_caches,
             conflict_caches: HashMap::new(),
             col_caches,
-            ordered_caches: HashMap::new(),
+            ordered_caches,
             bare_count_cache: HashMap::new(),
             parse_cache: HashMap::new(),
         })
@@ -528,7 +535,15 @@ impl Connection {
                 .get(table)
                 .cloned()
                 .unwrap_or_default();
-            tables.push((table.clone(), generation, cidx.clone(), iidx));
+            // The ordered index rides the same generation stamp: rebuilt from
+            // scratch by every fresh reader otherwise (its population scan is
+            // the dominant reopen cost after the col/id indexes are adopted).
+            let oidx = self
+                .ordered_caches
+                .get(table)
+                .cloned()
+                .unwrap_or_default();
+            tables.push((table.clone(), generation, cidx.clone(), iidx, oidx));
         }
         IndexSnapshot { tables }
     }
@@ -540,7 +555,7 @@ impl Connection {
     /// table is skipped (the lazy build stays the always-correct fallback);
     /// an already-built local index is never overwritten.
     pub fn adopt_built_indexes(&mut self, snapshot: &IndexSnapshot) {
-        for (table, generation, cidx, iidx) in &snapshot.tables {
+        for (table, generation, cidx, iidx, oidx) in &snapshot.tables {
             let already_built = self
                 .col_caches
                 .get(table)
@@ -568,9 +583,27 @@ impl Connection {
             {
                 self.id_caches.insert(table.clone(), iidx.clone());
             }
+            if oidx.is_built() && !self
+                .ordered_caches
+                .get(table)
+                .map(|o| o.is_built())
+                .unwrap_or(false)
+            {
+                self.ordered_caches.insert(table.clone(), oidx.clone());
+            }
         }
     }
 
+    /// True when `table`'s ORDER BY index is resident and built — exposed so
+    /// integration tests can assert a sidecar load or a snapshot adoption
+    /// admitted it without paying the population scan.
+    pub fn ordered_index_is_built(&self, table: &str) -> bool {
+        self.ordered_caches
+            .get(table)
+            .map(|o| o.is_built())
+            .unwrap_or(false)
+    }
+
     /// Parse `sql` into its AST, reusing a cached parse when one exists.
     ///
     /// The AST depends only on the SQL text, so a hit returns a clone of the
@@ -702,6 +735,15 @@ impl Connection {
             } else {
                 IdIndex::new()
             };
+            // The ordered index is persisted only when the writer already
+            // built one (it exists per ORDER BY usage, never force-built) —
+            // an unbuilt default round-trips as "nothing to adopt".
+            let ordered_snapshot = self
+                .ordered_caches
+                .get(&table)
+                .filter(|o| o.is_built())
+                .cloned()
+                .unwrap_or_default();
             let generation = self.meta.col_generation(&self.store, &table)?;
             let row_count = tree.count_cells().map_err(open_err)?;
             tables.push(PersistedTableIndex {
@@ -711,6 +753,7 @@ impl Connection {
                 row_count,
                 index: col_snapshot,
                 id_index: id_snapshot,
+                ordered_index: ordered_snapshot,
             });
         }
 
@@ -1801,8 +1844,12 @@ fn load_col_indexes(
     meta: &MetaTable,
     catalog: &Catalog,
     root_map: &BTreeMap<String, u32>,
-) -> (HashMap<String, ColIndex>, HashMap<String, IdIndex>) {
-    let empty = || (HashMap::new(), HashMap::new());
+) -> (
+    HashMap<String, ColIndex>,
+    HashMap<String, IdIndex>,
+    HashMap<String, OrderedColIndex>,
+) {
+    let empty = || (HashMap::new(), HashMap::new(), HashMap::new());
     let sidecar = col_index_sidecar_path(store.path());
     let Some(envelope) = cached_sidecar_envelope(&sidecar) else {
         return empty();
@@ -1812,6 +1859,7 @@ fn load_col_indexes(
     }
     let mut loaded: HashMap<String, ColIndex> = HashMap::new();
     let mut loaded_ids: HashMap<String, IdIndex> = HashMap::new();
+    let mut loaded_ordered: HashMap<String, OrderedColIndex> = HashMap::new();
     for entry in &envelope.tables {
         let Some(&root) = root_map.get(&entry.table) else {
             continue;
@@ -1843,9 +1891,15 @@ fn load_col_indexes(
         if entry.id_index.is_built() {
             loaded_ids.insert(entry.table.clone(), entry.id_index.clone());
         }
+        // The ordered index rides the SAME fingerprint gate; only a built one
+        // (the writer had run an ordered query on the table) is admitted, so a
+        // cold process serves its first ORDER BY from the sidecar scan-free.
+        if entry.ordered_index.is_built() {
+            loaded_ordered.insert(entry.table.clone(), entry.ordered_index.clone());
+        }
         loaded.insert(entry.table.clone(), entry.index.clone());
     }
-    (loaded, loaded_ids)
+    (loaded, loaded_ids, loaded_ordered)
 }
 
 /// How many distinct sidecar paths the process-wide decoded-envelope cache

crates/lilliengine/src/eval.rs

@@ -608,7 +608,7 @@ const CLASS_BLOB: u8 = 3;
 
 /// A within-class sort key that orders numbers, text, and blobs correctly while
 /// remaining `Ord` (numbers compare by value, including int↔real).
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub enum SortKey {
     /// The NULL placeholder; never participates in a within-class compare.
     Null,

crates/lilliengine/src/exec.rs

@@ -2308,7 +2308,7 @@ type OrderedTag = (u8, SortKey);
 /// decodes unrelated columns such as the embedding BLOB), maintained
 /// incrementally on insert/update/delete (no-op when unbuilt), cleared on
 /// rollback and tainted commit.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)]
 pub struct OrderedColIndex {
     /// The single column this index covers (e.g. `"ts"` for events).
     column: String,

crates/lilliengine/tests/boot_perf.rs

@@ -825,3 +825,56 @@ fn persisted_id_index_serves_warm_boot_without_scan() {
         );
     }
 }
+
+// ---------------------------------------------------------------------------
+// persisted_ordered_index_loads_built
+//
+// Build the ORDER BY index lazily via one ordered read, persist, reopen: the
+// fresh connection must hold the ordered index pre-built (no population scan)
+// and serve the same rows as the connection that built it.
+// ---------------------------------------------------------------------------
+#[test]
+fn persisted_ordered_index_loads_built() {
+    fn probe_ordered(conn: &mut Connection) -> Vec<String> {
+        let mut cur = conn
+            .execute("SELECT src FROM edges ORDER BY src LIMIT 10", vec![])
+            .unwrap();
+        cur.fetchall()
+            .iter()
+            .map(|r| match r.get_index(0) {
+                Some(Value::Text(s)) => s.clone(),
+                other => panic!("expected text src, got {other:?}"),
+            })
+            .collect()
+    }
+
+    let f = EdgeFixture::seeded(50);
+    let expected: Vec<String>;
+    {
+        let mut conn = f.open();
+        expected = probe_ordered(&mut conn);
+        assert!(
+            conn.ordered_index_is_built("edges"),
+            "the ordered probe must have built the ORDER BY index"
+        );
+        conn.persist_col_indexes().unwrap();
+    }
+
+    let mut loaded = f.open();
+    assert!(
+        loaded.ordered_index_is_built("edges"),
+        "a fresh open over the sidecar must admit the ordered index pre-built"
+    );
+    assert_eq!(
+        probe_ordered(&mut loaded),
+        expected,
+        "the loaded ordered index must serve the same rows as the builder"
+    );
+
+    let mut loaded_ro = f.open_ro();
+    assert!(
+        loaded_ro.ordered_index_is_built("edges"),
+        "a read-only open must admit the persisted ordered index too"
+    );
+    assert_eq!(probe_ordered(&mut loaded_ro), expected);
+}

src/iai_mcp/daemon/_watchdog.py

@@ -64,7 +64,7 @@ def _pkg():
 WATCHDOG_FAILURE_DEBOUNCE_N: int = int(
     os.environ.get("IAI_MCP_WATCHDOG_FAILURE_DEBOUNCE_N", "3"),
 )
-# Watchdog ceiling (4.0 GiB). Calibrated against the current
+# Watchdog ceiling (4.0 GiB). Recalibrated 2026-06-19 against the post-v9.3
 # corpus (25k+ episodic records, 270 MB SQLite, 17+ MB hnsw, plus the steady-state
 # warm-on-WAKE cache footprint and the deferred-capture drain loop). The previous
 # 2.5 GiB cap was set in mid-June when the corpus + warm caches sat near 1.72 GiB;
@@ -140,16 +140,34 @@ async def _hippea_cascade_loop(
     store, shutdown: asyncio.Event, *, _clock=time.monotonic,
 ) -> None:
     from iai_mcp import retrieve
-    from iai_mcp.daemon_state import load_state, save_state
+    from iai_mcp.daemon_state import daemon_state_path, load_state, save_state
     from iai_mcp.hippea_cascade import _install_warm, compute_and_fetch_warm
     from iai_mcp.lock_protocol import check_consolidation_intent
     # late import so the package attribute is re-fetched and a monkeypatch stays visible
     from iai_mcp.daemon import write_event
 
+    # mtime short-circuit: this loop polls a small JSON file around the clock
+    # and the file is idle almost always — skip the read+parse when it has not
+    # changed since the last tick and that tick carried no pending request.
+    _last_state_mtime_ns: "int | None" = None
+    _last_had_pending = False
     while not shutdown.is_set():
         try:
-            state = await asyncio.to_thread(load_state)
-            req = state.get("hippea_cascade_request") or {}
+            try:
+                _mtime_ns = daemon_state_path().stat().st_mtime_ns
+            except OSError:
+                _mtime_ns = None
+            if (
+                _mtime_ns is not None
+                and _mtime_ns == _last_state_mtime_ns
+                and not _last_had_pending
+            ):
+                req = {}
+            else:
+                _last_state_mtime_ns = _mtime_ns
+                state = await asyncio.to_thread(load_state)
+                req = state.get("hippea_cascade_request") or {}
+                _last_had_pending = bool(req.get("pending"))
             if req.get("pending"):
                 elapsed = _clock() - _pkg()._last_cascade_completed_at
                 if elapsed < HIPPEA_CASCADE_MIN_INTERVAL_SEC: