fix: Parallelize git & indexing

Author: dmtrKovalenko

Cargo.lock

@@ -1233,9 +1233,12 @@ impl FilePicker {
         }
     }
 
-    pub(crate) fn set_bigram_index(&mut self, index: BigramFilter, overlay: BigramOverlay) {
+    pub(crate) fn set_bigram_index(&mut self, index: BigramFilter) {
         self.sync_data.bigram_index = Some(Arc::new(index));
-        self.sync_data.bigram_overlay = Some(Arc::new(parking_lot::RwLock::new(overlay)));
+        // once the index is reset automatically reset the overaly
+        self.sync_data.bigram_overlay = Some(Arc::new(parking_lot::RwLock::new(
+            BigramOverlay::new(self.sync_data.indexable_count),
+        )));
     }
 
     pub(crate) fn scan_signals(&self) -> crate::scan::ScanSignals {
@@ -1720,14 +1723,15 @@ impl FileSync {
         std::thread::spawn(move || {
             GitStatusCache::read_git_status(
                 Some(git_workdir.as_path()),
-                &mut crate::git::default_status_options(),
+                &mut crate::git::initial_scan_status_options(),
             )
         })
     }
 
     /// Returns files immediately (searchable) and a handle to the in-progress
     /// git status computation. This avoids blocking on `git status` which can
     /// take 10+ seconds on very large repos (e.g. chromium).
+    #[tracing::instrument(skip_all, name = "walk_filesystem", level = Level::INFO)]
     pub(crate) fn walk_filesystem(
         base_path: &Path,
         git_workdir: Option<PathBuf>,
@@ -1767,6 +1771,7 @@ impl FileSync {
         // no chunking, no HashMap, just Vec::push under the Mutex.
         let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new());
 
+        let walker_span = tracing::info_span!("walker_run").entered();
         walker.run(|| {
             let pairs = &pairs;
             let counter = Arc::clone(synced_files_count);
@@ -1800,6 +1805,7 @@ impl FileSync {
                 ignore::WalkState::Continue
             })
         });
+        drop(walker_span);
 
         let mut pairs = pairs.into_inner();
         info!(

crates/fff-core/src/file_picker.rs

@@ -15,6 +15,20 @@ pub(crate) fn default_status_options() -> StatusOptions {
     opts
 }
 
+/// Status options for the initial scan / rescan.
+///
+/// Skips `include_unmodified` because every `FileItem` starts with
+/// `git_status: None` (== clean), so a missing cache entry already means
+/// "clean" — no need to ask libgit2 to enumerate every tracked path.
+/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries).
+pub(crate) fn initial_scan_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .exclude_submodules(true);
+    opts
+}
+
 #[derive(Debug, Clone, Default)]
 pub(crate) struct GitStatusCache(AHashMap<PathBuf, Status>);
 

crates/fff-core/src/git.rs

@@ -27,6 +27,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
+use libc::unsetenv;
 use tracing::{error, info};
 
 use crate::FileSync;
@@ -58,7 +59,7 @@ pub(crate) struct ScanSignals {
 }
 
 /// Which optional phases a scan should run.
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub(crate) struct ScanConfig {
     pub(crate) warmup: bool,
     pub(crate) content_indexing: bool,
@@ -232,26 +233,26 @@ impl ScanJob {
             None
         };
 
-        // 3. Apply git status + frecency off-lock (joins the git thread).
-        // Done BEFORE the bigram/warmup pass so `FileItem::git_status` is
-        // populated when the indexer decides which files to pin in the
-        // mmap cache (dirty files stay hot in the page cache on their own).
-        if !signals.cancelled.load(Ordering::Acquire)
-            && let Some(status_handle) = status_handle
-            && let Some(snap) = snapshot.as_mut()
-        {
-            apply_git_status_and_frecency(&shared_frecency, status_handle, mode, snap);
-        }
-
-        // 4. Post-scan warmup + bigram build.
+        // 3. Post-scan warmup + bigram build — runs in parallel with the
+        // git-status thread to overlap the two expensive phases.
         if (config.warmup || config.content_indexing)
             && !signals.cancelled.load(Ordering::Acquire)
             && let Some(snap) = snapshot.as_ref()
         {
             Self::run_post_scan(&shared_picker, &signals, &config, snap);
         }
 
-        drop(snapshot);
+        // 4. Join and git status, this HAS to be done after the post scan
+        if !signals.cancelled.load(Ordering::Acquire)
+            && let Some(status_handle) = status_handle
+            && let Some(snapshot) = snapshot.as_mut()
+        {
+            if let Ok(Some(git_status)) = status_handle.join() {
+                apply_git_status_and_frecency(git_status, &shared_frecency, mode, snapshot);
+            }
+        }
+
+        drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT
 
         // 5. Install filesystem watcher (initial scan only).
         if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) {
@@ -300,7 +301,7 @@ impl ScanJob {
         }
     }
 
-    #[tracing::instrument(skip_all)]
+    #[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))]
     fn run_post_scan(
         shared_picker: &SharedFilePicker,
         signals: &ScanSignals,
@@ -314,24 +315,27 @@ impl ScanJob {
             .unwrap_or(ArenaPtr::null());
         let budget: &ContentCacheBudget = &unsafe_snapshot.budget;
         let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
+        if !signals.cancelled.load(Ordering::Acquire) {
+            return;
+        }
 
         // unified bigram and warmup_mmaps in one go, it's important to reuse open files as much as possible
-        if config.content_indexing && !signals.cancelled.load(Ordering::Acquire) {
+        if config.content_indexing {
             let indexable_files = &files[..unsafe_snapshot.indexable_count.min(files.len())];
             let index = build_bigram_index(
                 indexable_files,
                 budget,
                 &unsafe_snapshot.base_path,
                 arena,
-                config.warmup,
+                config.warmup, // can be optionally skipped
             );
 
             if let Ok(mut guard) = shared_picker.write()
                 && let Some(picker) = guard.as_mut()
             {
-                picker.set_bigram_index(index, BigramOverlay::new(unsafe_snapshot.indexable_count));
+                picker.set_bigram_index(index);
             }
-        } else if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
+        } else if config.warmup {
             // Warmup-only: no bigram indexing, just fill the mmap cache.
             warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
         }
@@ -386,21 +390,17 @@ fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) {
     });
 }
 
+#[tracing::instrument(
+    level = "debug",
+    skip_all,
+    fields(file_count = tracing::field::Empty, dirty_count = tracing::field::Empty),
+)]
 fn apply_git_status_and_frecency(
+    git_cache: GitStatusCache,
     shared_frecency: &SharedFrecency,
-    git_handle: std::thread::JoinHandle<Option<GitStatusCache>>,
     mode: FFFMode,
     unsafe_snapshot: &mut crate::file_picker::PostScanUnsafeSnapshot,
 ) {
-    let git_cache = match git_handle.join() {
-        Ok(Some(cache)) => cache,
-        Ok(None) => return,
-        Err(_) => {
-            error!("Git status thread panicked");
-            return;
-        }
-    };
-
     let frecency = shared_frecency.read().ok();
     let frecency_ref = frecency.as_ref().and_then(|f| f.as_ref());
 
@@ -445,11 +445,7 @@ fn apply_git_status_and_frecency(
             }
         });
     });
-    drop(frecency);
 
-    info!(
-        "SCAN: Applied git status to {} files ({} dirty)",
-        unsafe_snapshot.base_count,
-        git_cache.statuses_len(),
-    );
+    let span = tracing::Span::current();
+    span.record("dirty_count", git_cache.statuses_len());
 }

crates/fff-core/src/scan.rs

@@ -93,6 +93,7 @@ smartstring = { version = "1.0.1", features = ["serde"] }
 criterion = { version = "0.5", features = ["html_reports"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [[bench]]
 name = "fuzzy_search"

crates/fff-nvim/Cargo.toml

@@ -3,10 +3,30 @@ use criterion::{Criterion, criterion_group, criterion_main};
 use fff::file_picker::{FFFMode, FilePicker};
 use fff::{FilePickerOptions, SharedFilePicker, SharedFrecency};
 use std::path::PathBuf;
+use std::sync::Once;
 use std::time::{Duration, Instant};
 
 const WAIT_TIMEOUT: Duration = Duration::from_secs(300);
 
+static TRACING_INIT: Once = Once::new();
+
+fn init_tracing() {
+    TRACING_INIT.call_once(|| {
+        use tracing_subscriber::EnvFilter;
+        use tracing_subscriber::fmt::format::FmtSpan;
+
+        let _ = tracing_subscriber::fmt()
+            .with_env_filter(
+                EnvFilter::try_from_default_env()
+                    .unwrap_or_else(|_| EnvFilter::new("warn,fff_search=info")),
+            )
+            .with_span_events(FmtSpan::CLOSE)
+            .with_target(true)
+            .with_writer(std::io::stderr)
+            .try_init();
+    });
+}
+
 fn resolve_repo() -> Option<PathBuf> {
     if let Ok(env_path) = std::env::var("FFF_BENCH_REPO") {
         let p = PathBuf::from(env_path);
@@ -15,8 +35,7 @@ fn resolve_repo() -> Option<PathBuf> {
         }
     }
     // Resolve relative to the workspace root (two levels up from this crate).
-    let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../..");
+    let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
     let default = workspace_root.join("big-repo");
     if default.exists() {
         return fff::path_utils::canonicalize(&default).ok();
@@ -107,6 +126,7 @@ fn cleanup(sp: SharedFilePicker) {
 }
 
 fn bench_full_init(c: &mut Criterion) {
+    init_tracing();
     let Some(repo) = resolve_repo() else {
         eprintln!("skip: set FFF_BENCH_REPO or clone a repo to ./big-repo");
         return;
@@ -136,6 +156,7 @@ fn bench_full_init(c: &mut Criterion) {
 }
 
 fn bench_post_scan_only(c: &mut Criterion) {
+    init_tracing();
     let Some(repo) = resolve_repo() else {
         return;
     };
@@ -164,6 +185,7 @@ fn bench_post_scan_only(c: &mut Criterion) {
 }
 
 fn bench_walk_only(c: &mut Criterion) {
+    init_tracing();
     let Some(repo) = resolve_repo() else {
         return;
     };