Merge pull request #447 from ScriptedAlchemy/fix/session-catchup-integrity

perf(sessions): coalesce and bulk-route catch-ups

Author: ScriptedAlchemy

.changeset/session-catchup-hook-trust.md

@@ -0,0 +1,5 @@
+---
+"tracedecay": patch
+---
+
+Coalesce transcript catch-ups, route Hermes history through one shared source sweep, harden branch database recovery markers, emit Codex hook trust state in the exact form required for non-interactive approval, and reduce test-profile link overhead while preserving line-table backtraces.

Cargo.toml

@@ -183,6 +183,12 @@ jsonschema = { version = "0.46.8", default-features = false }
 name = "large_repos"
 harness = false
 
+# Full debuginfo makes each large integration-test binary 500-700 MB and
+# dominates local/CI link time. Level 1 preserves source line tables for
+# useful panic backtraces while substantially shrinking linker input/output.
+[profile.test]
+debug = 1
+
 # Compile the SQLite stack with optimizations even in dev/test builds. The
 # bundled SQLite C sources in libsql-ffi are otherwise built at -O0 (cc honors
 # cargo's OPT_LEVEL), and nearly every test creates databases and runs many

src/agents/codex.rs

@@ -1041,10 +1041,38 @@ fn sync_codex_hook_trust(home: &Path, tracedecay_bin: &str) -> Result<CodexHookT
         trusted += 1;
     }
 
-    write_toml_file(&config_path, &config)?;
+    write_codex_hook_trust_config(&config_path, &config)?;
     Ok(CodexHookTrustSyncOutcome { trusted, skipped })
 }
 
+/// Codex's hook loader requires the parent table to be explicit on disk. The
+/// `toml` serializer otherwise emits only `[hooks.state."..."]` child tables,
+/// which parses equivalently but still triggers Codex's hook-review prompt.
+fn write_codex_hook_trust_config(config_path: &Path, config: &toml::Value) -> Result<()> {
+    super::backup_file(config_path)?;
+    let contents = toml::to_string_pretty(config).map_err(|error| TraceDecayError::Config {
+        message: format!("failed to serialize {}: {error}", config_path.display()),
+    })?;
+    let Some(child_offset) = contents.find("[hooks.state.\"") else {
+        return Err(TraceDecayError::Config {
+            message: "Codex hook trust state serialized without hook entries".to_string(),
+        });
+    };
+    let mut updated = String::with_capacity(contents.len() + "[hooks.state]\n\n".len());
+    updated.push_str(&contents[..child_offset]);
+    updated.push_str("[hooks.state]\n\n");
+    updated.push_str(&contents[child_offset..]);
+    std::fs::write(config_path, updated).map_err(|error| TraceDecayError::Config {
+        message: format!("failed to write {}: {error}", config_path.display()),
+    })?;
+    eprintln!("\x1b[32m✔\x1b[0m Wrote {}", config_path.display());
+    Ok(())
+}
+
+fn codex_hook_state_table_is_explicit(contents: &str) -> bool {
+    contents.lines().any(|line| line.trim() == "[hooks.state]")
+}
+
 /// Auto-trust the installed plugin's hooks, printing a concise confirmation on
 /// full success and falling back to [`print_hook_trust_guidance`] whenever a
 /// hook is skipped by the safety valve or the config could not be written.
@@ -1855,8 +1883,17 @@ fn doctor_check_hooks(
     let entries = codex_hook_trust_entries_for_marketplace(&hooks, marketplace_name);
     match load_toml_file(config_path) {
         Ok(config) => match codex_plugin_hook_trust_state(&config, &entries) {
-            CodexHookTrustState::Trusted => dc.pass(&format!(
-                "Codex hook trust entries recorded and current in {}",
+            CodexHookTrustState::Trusted
+                if std::fs::read_to_string(config_path)
+                    .is_ok_and(|contents| codex_hook_state_table_is_explicit(&contents)) =>
+            {
+                dc.pass(&format!(
+                    "Codex hook trust entries recorded and current in {}",
+                    config_path.display()
+                ));
+            }
+            CodexHookTrustState::Trusted => dc.warn(&format!(
+                "Codex hook trust records in {} lack an explicit [hooks.state] table, so Codex still requests review; run `tracedecay update-plugin` to repair and auto-trust them",
                 config_path.display()
             )),
             CodexHookTrustState::Missing(missing) => dc.info(&format!(

src/agents/codex/tests.rs

@@ -263,10 +263,33 @@ trusted_hash = "sha256:foreign"
 "#,
     )
     .unwrap();
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        std::fs::set_permissions(&config_path, std::fs::Permissions::from_mode(0o600)).unwrap();
+    }
 
     let outcome = sync_codex_hook_trust(home.path(), TEST_BIN).unwrap();
     assert_eq!(outcome.trusted, CODEX_MANAGED_HOOKS.len());
     assert!(outcome.skipped.is_empty());
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        assert_eq!(
+            std::fs::metadata(&config_path)
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777,
+            0o600
+        );
+    }
+
+    let config_text = std::fs::read_to_string(&config_path).unwrap();
+    assert!(
+        config_text.lines().any(|line| line == "[hooks.state]"),
+        "Codex requires an explicit [hooks.state] parent table before trusting child records"
+    );
 
     let entries = managed_entries(TEST_BIN);
     let config = load_toml_file(&config_path).unwrap();

src/db/connection.rs

@@ -256,13 +256,45 @@ impl Database {
     /// This ensures all committed transactions are merged into the main DB
     /// before the process exits, preventing a stale WAL file on next startup.
     pub async fn checkpoint(&self) -> Result<()> {
-        self.conn
-            .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")
+        let mut rows = self
+            .conn
+            .query("PRAGMA wal_checkpoint(TRUNCATE);", ())
             .await
             .map_err(|e| TraceDecayError::Database {
                 message: format!("failed to checkpoint WAL: {e}"),
                 operation: "checkpoint".to_string(),
             })?;
+        let row = rows
+            .next()
+            .await
+            .map_err(|e| TraceDecayError::Database {
+                message: format!("failed to read WAL checkpoint status: {e}"),
+                operation: "checkpoint".to_string(),
+            })?
+            .ok_or_else(|| TraceDecayError::Database {
+                message: "WAL checkpoint returned no status row".to_string(),
+                operation: "checkpoint".to_string(),
+            })?;
+        let busy: i64 = row.get(0).map_err(|e| TraceDecayError::Database {
+            message: format!("failed to read WAL checkpoint busy status: {e}"),
+            operation: "checkpoint".to_string(),
+        })?;
+        let log_frames: i64 = row.get(1).map_err(|e| TraceDecayError::Database {
+            message: format!("failed to read WAL checkpoint frame count: {e}"),
+            operation: "checkpoint".to_string(),
+        })?;
+        let checkpointed_frames: i64 = row.get(2).map_err(|e| TraceDecayError::Database {
+            message: format!("failed to read WAL checkpoint completion count: {e}"),
+            operation: "checkpoint".to_string(),
+        })?;
+        if busy != 0 || checkpointed_frames < log_frames {
+            return Err(TraceDecayError::Database {
+                message: format!(
+                    "WAL checkpoint incomplete: busy={busy}, log_frames={log_frames}, checkpointed_frames={checkpointed_frames}"
+                ),
+                operation: "checkpoint".to_string(),
+            });
+        }
         Ok(())
     }
 

src/doctor.rs

@@ -309,7 +309,10 @@ fn database_recovery_guidance(db_path: &Path) -> String {
     let wal_path = db_path.with_extension("db-wal");
     let shm_path = db_path.with_extension("db-shm");
     let data_root = db_path.parent().unwrap_or_else(|| Path::new("."));
-    let dirty_path = data_root.join("dirty");
+    let mut graph_dirty = db_path.as_os_str().to_os_string();
+    graph_dirty.push(".dirty");
+    let graph_dirty = PathBuf::from(graph_dirty);
+    let legacy_dirty = data_root.join("dirty");
     let sessions_path = data_root.join(crate::storage::SESSIONS_DB_FILENAME);
 
     format!(
@@ -318,15 +321,17 @@ fn database_recovery_guidance(db_path: &Path) -> String {
          DB: {}\n\
          WAL: {}\n\
          SHM: {}\n\
-         dirty sentinel: {}\n\
+         graph dirty sentinel: {}\n\
+         legacy dirty sentinel (if present): {}\n\
          `sessions.db` is separate and must not be removed: {}\n\
          Facts are stored in the graph database; automatic rebuild is intentionally blocked because it cannot preserve them generically.\n\
          Do not run `tracedecay init`, `tracedecay sync --force`, or `tracedecay wipe` until that recovery set is safely copied.\n\
          Report the preserved set at https://github.com/ScriptedAlchemy/tracedecay/issues for offline recovery.",
         db_path.display(),
         wal_path.display(),
         shm_path.display(),
-        dirty_path.display(),
+        graph_dirty.display(),
+        legacy_dirty.display(),
         sessions_path.display(),
     )
 }

src/doctor/tests.rs

@@ -215,10 +215,15 @@ fn database_recovery_guidance_names_the_preserved_recovery_set() {
     let db_path = PathBuf::from("/profile/projects/proj_test/tracedecay.db");
     let guidance = database_recovery_guidance(&db_path);
 
-    assert!(guidance.contains("/profile/projects/proj_test/tracedecay.db"));
-    assert!(guidance.contains("/profile/projects/proj_test/tracedecay.db-wal"));
-    assert!(guidance.contains("/profile/projects/proj_test/tracedecay.db-shm"));
-    assert!(guidance.contains("/profile/projects/proj_test/dirty"));
+    for path in [
+        db_path.clone(),
+        db_path.with_extension("db-wal"),
+        db_path.with_extension("db-shm"),
+        PathBuf::from(format!("{}.dirty", db_path.display())),
+        db_path.parent().unwrap().join("dirty"),
+    ] {
+        assert!(guidance.contains(&path.display().to_string()));
+    }
     assert!(guidance.contains("stop all TraceDecay daemon and MCP processes"));
     assert!(
         guidance.contains(

src/global_db.rs

@@ -971,20 +971,38 @@ impl GlobalDb {
     /// in-process and retried briefly to also cover a racing *external*
     /// process (e.g. two MCP servers starting simultaneously).
     pub async fn open_at(db_path: &std::path::Path) -> Option<Self> {
+        Self::open_at_with_backfill(db_path, true).await
+    }
+
+    /// Opens and ensures a writable session store without starting detached
+    /// structured backfill. Bulk multi-store catch-up uses this to avoid
+    /// launching one competing backfill task per registered project.
+    pub async fn open_at_without_structured_backfill(db_path: &std::path::Path) -> Option<Self> {
+        Self::open_at_with_backfill(db_path, false).await
+    }
+
+    async fn open_at_with_backfill(
+        db_path: &std::path::Path,
+        spawn_structured_backfill: bool,
+    ) -> Option<Self> {
         static OPEN_ENSURE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
         let _guard = OPEN_ENSURE_LOCK.lock().await;
         for attempt in 0..3_u64 {
             if attempt > 0 {
                 tokio::time::sleep(std::time::Duration::from_millis(50 * attempt)).await;
             }
-            if let Some(db) = Self::open_at_unsynchronized(db_path).await {
+            if let Some(db) = Self::open_at_unsynchronized(db_path, spawn_structured_backfill).await
+            {
                 return Some(db);
             }
         }
         None
     }
 
-    async fn open_at_unsynchronized(db_path: &std::path::Path) -> Option<Self> {
+    async fn open_at_unsynchronized(
+        db_path: &std::path::Path,
+        spawn_structured_backfill: bool,
+    ) -> Option<Self> {
         if let Some(parent) = db_path.parent() {
             std::fs::create_dir_all(parent).ok()?;
         }
@@ -1204,7 +1222,9 @@ impl GlobalDb {
         // runs on every open (per hook event, per CLI/MCP invocation), so it
         // must not block: schedule it on a detached background task rather than
         // synchronously reading and re-parsing a batch of multi-MB transcripts.
-        db.spawn_structured_backfill();
+        if spawn_structured_backfill {
+            db.spawn_structured_backfill();
+        }
 
         Some(db)
     }
@@ -3477,10 +3497,17 @@ impl GlobalDb {
                 return false;
             }
         }
-        if !self
-            .set_parse_offset_in_existing_tx(parse_offset_path, parse_offset)
-            .await
-        {
+        let cursor_set = match mode {
+            TranscriptWriteMode::Full => {
+                self.set_parse_offset_in_existing_tx(parse_offset_path, parse_offset)
+                    .await
+            }
+            TranscriptWriteMode::ProjectionOnly => {
+                self.set_parse_offset_monotonic_in_existing_tx(parse_offset_path, parse_offset)
+                    .await
+            }
+        };
+        if !cursor_set {
             let _ = self.conn.execute("ROLLBACK", ()).await;
             return false;
         }
@@ -4851,6 +4878,39 @@ impl GlobalDb {
         let _ = self.set_parse_offset_in_existing_tx(path, offset).await;
     }
 
+    /// Advances a row-style parse cursor without allowing an overlapping,
+    /// older sweep to move it backwards.
+    pub async fn advance_parse_offset(&self, path: &str, offset: ParseOffset) {
+        let _ = self
+            .set_parse_offset_monotonic_in_existing_tx(path, offset)
+            .await;
+    }
+
+    async fn set_parse_offset_monotonic_in_existing_tx(
+        &self,
+        path: &str,
+        offset: ParseOffset,
+    ) -> bool {
+        self.conn
+            .execute(
+                "INSERT INTO parse_offsets (file_path, byte_offset, mtime, file_id)
+                 VALUES (?1, ?2, ?3, ?4)
+                 ON CONFLICT(file_path) DO UPDATE SET
+                    byte_offset = excluded.byte_offset,
+                    mtime = excluded.mtime,
+                    file_id = excluded.file_id
+                 WHERE excluded.byte_offset >= parse_offsets.byte_offset",
+                params![
+                    path,
+                    offset.byte_offset as i64,
+                    offset.mtime as i64,
+                    offset.file_id as i64
+                ],
+            )
+            .await
+            .is_ok()
+    }
+
     async fn set_parse_offset_in_existing_tx(&self, path: &str, offset: ParseOffset) -> bool {
         if self
             .conn