diff --git a/crates/fff-core/src/background_watcher.rs b/crates/fff-core/src/background_watcher.rs index 86e8263c..c419342b 100644 --- a/crates/fff-core/src/background_watcher.rs +++ b/crates/fff-core/src/background_watcher.rs @@ -37,6 +37,10 @@ pub struct BackgroundWatcher { const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(250); const MAX_PATHS_THRESHOLD: usize = 1024; +/// On macOS, each `watch()` call creates a separate FSEventStream. When the +/// number of directories exceeds this threshold we fall back to a single +/// recursive watch to avoid exhausting the per-process stream limit. +const MAX_MACOS_NONRECURSIVE_WATCHES: usize = 4096; /// Minimum seconds between frecency tracks of the same file in AI mode. /// Prevents score inflation from rapid burst edits by AI agents. const AI_MODE_COOLDOWN_SECS: u64 = 5 * 60; @@ -58,7 +62,8 @@ impl BackgroundWatcher { let (watch_tx, watch_rx) = mpsc::channel::(); - // Clone shared state for the owner thread + // Clone shared state for the owner thread (needed for injecting + // files that existed before a watch was registered on their directory). let owner_picker = shared_picker.clone(); let owner_git_workdir = git_workdir.clone(); @@ -95,9 +100,13 @@ impl BackgroundWatcher { warn!("Failed to watch new directory {}: {}", dir.display(), e); } } - // Files/dirs created before the watch was set up won't - // generate inotify events. Scan and inject them now. - scan_new_directory(&dir, &mut debouncer, &owner_picker, &owner_git_workdir); + + // Files created before the watch was registered don't + // generate events. Do a flat (non-recursive) read_dir + // to inject any files that already exist. Subdirectories + // are not descended — they get their own watches via + // future Create events from this directory's watch. + inject_existing_files(&dir, &owner_picker, &owner_git_workdir); } std::thread::park_timeout(Duration::from_secs(1)); } @@ -137,6 +146,11 @@ impl BackgroundWatcher { // our own grep calls and preview window rendering .with_event_kinds(EventKindMask::CORE); + // Decide the watching strategy up-front so the event handler closure + // knows whether it needs to request dynamic directory watches. + let use_recursive = + cfg!(target_os = "macos") && watch_dirs.len() > MAX_MACOS_NONRECURSIVE_WATCHES; + let git_workdir_for_handler = git_workdir.clone(); let mut debouncer = new_debouncer_opt( DEBOUNCE_TIMEOUT, @@ -144,37 +158,23 @@ impl BackgroundWatcher { { move |result: DebounceEventResult| match result { Ok(events) => { - // Detect newly created directories and request NonRecursive - // watches on them so we see files created inside. - // Skip gitignored directories (e.g. node_modules/) to - // avoid re-inflating the inotify watch set. - let repo = git_workdir_for_handler - .as_ref() - .and_then(|p| Repository::open(p).ok()); - for debounced_event in &events { - if matches!( - debounced_event.event.kind, - EventKind::Create(_) - | EventKind::Modify(notify::event::ModifyKind::Name(_)) - ) { - for path in &debounced_event.event.paths { - if path.is_dir() - && !is_git_file(path) - && !is_path_ignored(path, &repo) - { - let _ = watch_tx.send(path.clone()); - } - } - } - } - - handle_debounced_events( + let new_dirs = handle_debounced_events( events, &git_workdir_for_handler, &shared_picker, &shared_frecency, mode, ); + + // In NonRecursive mode, register watches for newly + // discovered directories so future file events in them + // are captured. In Recursive mode the single stream + // already covers new subdirectories. + if !use_recursive { + for dir in new_dirs { + let _ = watch_tx.send(dir); + } + } } Err(errors) => { error!("File watcher errors: {:?}", errors); @@ -189,37 +189,66 @@ impl BackgroundWatcher { config, )?; - // Watch all directories NonRecursively. The watch_dirs are derived from - // the already-scanned file list so they respect .gitignore at every depth. - // On Linux (inotify) RecursiveMode::Recursive creates one watch per subdirectory - // including gitignored ones like node_modules/, which wastes kernel resources. - // NonRecursive watches only the directories that actually contain indexed files. + // Watching strategy: + // + // For small-to-medium repos we watch each indexed directory individually + // (NonRecursive). This avoids receiving events for gitignored paths like + // node_modules/ and keeps the event volume low. + // + // On macOS, each `watch()` call creates a separate FSEventStream. Large + // repos (e.g. Chromium with 487K+ files) can have tens of thousands of + // directories, which exhausts the per-process FSEvents stream limit and + // causes "unable to start FSEvent stream" errors. When the directory + // count exceeds the threshold we fall back to a single Recursive watch + // on the base path. FSEvents handles this efficiently with one kernel + // stream for the entire subtree. Gitignored paths are already filtered + // in the event handler via `should_include_file()`. + // + // On Linux (inotify), RecursiveMode::Recursive creates one kernel watch + // per subdirectory *including* gitignored ones, wasting file descriptors. + // The per-directory NonRecursive approach is always used on Linux. // - // New directories created at runtime are detected via Create events on the - // parent and dynamically added by the owner thread via the watch_tx channel. - debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?; - - for dir in &watch_dirs { - match debouncer.watch(dir.as_path(), RecursiveMode::NonRecursive) { - Ok(()) => {} - Err(e) => { - // Non-fatal: directory may have been removed between discovery and watch - warn!("Failed to watch directory {}: {}", dir.display(), e); + // New directories created at runtime are detected via Create events on + // the parent and dynamically added by the owner thread via watch_tx. + + if use_recursive { + debouncer.watch(base_path.as_path(), RecursiveMode::Recursive)?; + info!( + "File watcher initialized with single recursive watch on {} \ + ({} directories exceeded threshold of {})", + base_path.display(), + watch_dirs.len(), + MAX_MACOS_NONRECURSIVE_WATCHES, + ); + } else { + debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?; + + for dir in &watch_dirs { + match debouncer.watch(dir.as_path(), RecursiveMode::NonRecursive) { + Ok(()) => {} + Err(e) => { + // Non-fatal: directory may have been removed between discovery and watch + warn!("Failed to watch directory {}: {}", dir.display(), e); + } } } + + info!( + "File watcher initialized for {} directories (NonRecursive) under {}", + watch_dirs.len(), + base_path.display() + ); } // The .git directory is excluded from the file list but we still need // to observe changes that affect git status (staging, unstaging, // committing, branch switches, merges, etc.). + // When using recursive mode the base watch already covers .git/, + // but these targeted watches are cheap (at most 3 extra streams) + // and ensure we catch status changes even if the recursive backend + // coalesces or delays .git events. watch_git_status_paths(&mut debouncer, git_workdir.as_ref()); - info!( - "File watcher initialized for {} directories (NonRecursive) under {}", - watch_dirs.len(), - base_path.display() - ); - Ok(debouncer) } @@ -250,13 +279,14 @@ fn handle_debounced_events( shared_picker: &SharedPicker, shared_frecency: &SharedFrecency, mode: FFFMode, -) { +) -> Vec { // this will be called very often, we have to minimiy the lock time for file picker let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok()); let mut need_full_rescan = false; let mut need_full_git_rescan = false; let mut paths_to_remove = Vec::new(); let mut paths_to_add_or_modify = Vec::new(); + let mut new_dirs_to_watch = Vec::new(); let mut affected_paths_count = 0usize; for debounced_event in &events { @@ -319,6 +349,13 @@ fn handle_debounced_events( if is_removal || !path.exists() { paths_to_remove.push(path.as_path()); + } else if path.is_dir() { + // New directory — collect it so the caller can register a + // watcher. No filesystem scanning: files that arrive later + // will be handled by the newly registered watch. + if !is_path_ignored(path, &repo) { + new_dirs_to_watch.push(path.to_path_buf()); + } } else { // For additions/modifications, still filter gitignored files. if should_include_file(path, &repo) { @@ -346,7 +383,7 @@ fn handle_debounced_events( if need_full_rescan { info!(?affected_paths_count, "Triggering full rescan"); trigger_full_rescan(shared_picker, shared_frecency); - return; + return Vec::new(); } // It's important to get the allocated sort @@ -356,9 +393,10 @@ fn handle_debounced_events( paths_to_add_or_modify.dedup_by(|a, b| a.as_os_str().eq(b.as_os_str())); info!( - "Event processing summary: {} to remove, {} to add/modify", + "Event processing summary: {} to remove, {} to add/modify, {} new dirs", paths_to_remove.len(), - paths_to_add_or_modify.len() + paths_to_add_or_modify.len(), + new_dirs_to_watch.len() ); // Apply file index updates (add/remove) unconditionally — these must @@ -396,11 +434,11 @@ fn handle_debounced_events( let Ok(mut guard) = shared_picker.write() else { error!("Failed to acquire file picker write lock"); - return; + return new_dirs_to_watch; }; let Some(ref mut picker) = *guard else { error!("File picker not initialized"); - return; + return new_dirs_to_watch; }; apply_changes(picker) } else { @@ -455,7 +493,7 @@ fn handle_debounced_events( // Git status updates require a repository. let Some(repo) = repo.as_ref() else { debug!("No git repo available, skipping git status updates"); - return; + return new_dirs_to_watch; }; if need_full_git_rescan { @@ -465,7 +503,7 @@ fn handle_debounced_events( if let Err(e) = result { error!("Failed to refresh git status: {:?}", e); } - return; + return new_dirs_to_watch; } if !files_to_update_git_status.is_empty() { @@ -478,7 +516,7 @@ fn handle_debounced_events( Ok(status) => status, Err(e) => { tracing::error!(?e, "Failed to query git status"); - return; + return new_dirs_to_watch; } }; @@ -494,6 +532,8 @@ fn handle_debounced_events( error!("Failed to acquire picker lock for git status update"); } } + + new_dirs_to_watch } fn trigger_full_rescan(shared_picker: &SharedPicker, shared_frecency: &SharedFrecency) { @@ -523,41 +563,24 @@ fn trigger_full_rescan(shared_picker: &SharedPicker, shared_frecency: &SharedFre picker.spawn_post_rescan_rebuild(shared_picker.clone()); } -fn should_include_file(path: &Path, repo: &Option) -> bool { - // Directories are not indexed — only regular files (and symlinks to files). - if path.is_dir() { - return false; - } - - match repo.as_ref() { - Some(repo) => repo.is_path_ignored(path) != Ok(true), - None => { - // No git repo — apply basic sanity filters. - // Hidden directories are skipped by the watcher setup (hidden(true)), - // but events can still arrive for files in known non-code directories. - !is_non_code_directory(path) - } - } -} - -fn is_non_code_directory(path: &Path) -> bool { - crate::ignore::is_non_code_directory(path) -} +/// After registering a watch on a newly created directory, list its +/// immediate children and add any files to the picker. +fn inject_existing_files(dir: &Path, shared_picker: &SharedPicker, git_workdir: &Option) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; -/// After adding a NonRecursive watch on a newly created directory, scan it for -/// files and subdirectories that were created before the watch was set up. -/// This closes the race where `mkdir foo && echo > foo/bar.txt` both happen -/// before the owner thread adds a watch on `foo/`. -fn scan_new_directory( - dir: &Path, - debouncer: &mut Debouncer, - shared_picker: &SharedPicker, - git_workdir: &Option, -) { let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok()); let mut files_to_add = Vec::new(); - collect_new_entries(dir, &repo, debouncer, &mut files_to_add); + for entry in entries.flatten() { + if entry.file_type().is_ok_and(|ft| ft.is_file()) { + let path = entry.path(); + if should_include_file(&path, &repo) { + files_to_add.push(path); + } + } + } if files_to_add.is_empty() { return; @@ -573,40 +596,35 @@ fn scan_new_directory( for path in &files_to_add { picker.on_create_or_modify(path); } - info!( - "Scanned new directory {}: added {} files", - dir.display(), + + debug!( + "Injected {} existing files from new directory {}", files_to_add.len(), + dir.display(), ); } -fn collect_new_entries( - dir: &Path, - repo: &Option, - debouncer: &mut Debouncer, - files: &mut Vec, -) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - - for entry in entries.flatten() { - let path = entry.path(); - let Ok(file_type) = entry.file_type() else { - continue; - }; +fn should_include_file(path: &Path, repo: &Option) -> bool { + // Directories are not indexed — only regular files (and symlinks to files). + if path.is_dir() { + return false; + } - if file_type.is_dir() { - if !is_git_file(&path) && !is_path_ignored(&path, repo) { - let _ = debouncer.watch(&path, RecursiveMode::NonRecursive); - collect_new_entries(&path, repo, debouncer, files); - } - } else if file_type.is_file() && should_include_file(&path, repo) { - files.push(path); + match repo.as_ref() { + Some(repo) => repo.is_path_ignored(path) != Ok(true), + None => { + // No git repo — apply basic sanity filters. + // Hidden directories are skipped by the watcher setup (hidden(true)), + // but events can still arrive for files in known non-code directories. + !is_non_code_directory(path) } } } +fn is_non_code_directory(path: &Path) -> bool { + crate::ignore::is_non_code_directory(path) +} + #[inline] fn is_path_ignored(path: &Path, repo: &Option) -> bool { match repo.as_ref() { diff --git a/crates/fff-core/tests/new_directory_watcher_test.rs b/crates/fff-core/tests/new_directory_watcher_test.rs new file mode 100644 index 00000000..23e3419f --- /dev/null +++ b/crates/fff-core/tests/new_directory_watcher_test.rs @@ -0,0 +1,534 @@ +//! Integration test: verifying that the background watcher dynamically detects +//! newly created directories and picks up files written inside them. +//! +//! This covers the NonRecursive watching behavior where: +//! 1. The watcher starts with watches on directories discovered during the +//! initial scan. +//! 2. A brand-new subdirectory is created at runtime (after the scan). +//! 3. The watcher's event handler detects the directory Create event, +//! collects it, and sends it to the owner thread via `watch_tx`. +//! 4. The owner thread adds a NonRecursive watch on the new directory and +//! does a flat (non-recursive) read_dir to inject files that already +//! exist (race-window coverage). +//! 5. Files created *after* the watch is established are picked up via +//! normal event delivery. +//! +//! The test uses the real `BackgroundWatcher` (via `watch: true`) and polls +//! the picker until the expected files appear or a timeout expires. + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{FilePickerOptions, PaginationArgs, QueryParser, SharedFrecency, SharedPicker}; + +// ═══════════════════════════════════════════════════════════════════════ +// Helpers +// ═══════════════════════════════════════════════════════════════════════ + +fn git_run(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .output() + .unwrap_or_else(|e| panic!("git {:?} failed: {}", args, e)); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git_init_and_commit(dir: &Path) { + git_run(dir, &["init", "-b", "main"]); + git_run(dir, &["add", "-A"]); + git_run(dir, &["commit", "-m", "initial"]); +} + +fn make_watched_picker(base: &Path) -> (SharedPicker, SharedFrecency) { + let shared_picker = SharedPicker::default(); + let shared_frecency = SharedFrecency::noop(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + warmup_mmap_cache: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + (shared_picker, shared_frecency) +} + +/// Wait for the initial scan + watcher to be fully ready. +fn wait_ready(shared_picker: &SharedPicker) { + assert!( + shared_picker.wait_for_scan(Duration::from_secs(10)), + "Timed out waiting for initial scan" + ); + assert!( + shared_picker.wait_for_watcher(Duration::from_secs(10)), + "Timed out waiting for watcher" + ); +} + +/// Poll the picker until `predicate` returns true or timeout expires. +/// Returns the elapsed duration if successful, panics on timeout. +fn poll_until( + shared_picker: &SharedPicker, + timeout: Duration, + description: &str, + predicate: impl Fn(&FilePicker) -> bool, +) -> Duration { + let start = Instant::now(); + loop { + { + let guard = shared_picker.read().unwrap(); + if let Some(ref picker) = *guard { + if predicate(picker) { + return start.elapsed(); + } + } + } + if start.elapsed() >= timeout { + // One final attempt to give a useful error message. + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let file_count = picker.get_files().len(); + let paths: Vec = picker + .get_files() + .iter() + .map(|f| f.relative_path(picker)) + .collect(); + panic!( + "Timed out after {:?} waiting for: {}\n\ + Current file count: {}\n\ + Current files: {:?}", + timeout, description, file_count, paths + ); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn grep_plain_count(picker: &FilePicker, query: &str) -> usize { + let parsed = parse_grep_query(query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + picker.grep(&parsed, &opts).matches.len() +} + +fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + fff_search::FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +/// Debounce timeout in the watcher is 250ms. Events need to propagate through +/// the debouncer, the owner thread park loop (1s), and the picker write lock. +/// We use a generous timeout for CI environments. +const WATCHER_TIMEOUT: Duration = Duration::from_secs(10); + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +/// Create a new directory and immediately write a file inside it. +/// The file is written before the watch is registered, so the flat +/// inject_existing_files scan in the owner thread must catch it. +#[test] +fn new_directory_and_file_detected_by_watcher() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + // Seed the repo with some initial files so the scan has something. + fs::create_dir_all(base.join("src")).unwrap(); + fs::write( + base.join("src/main.rs"), + "fn main() { println!(\"INITIAL_MARKER\"); }\n", + ) + .unwrap(); + fs::write(base.join("README.md"), "# Test project\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Sanity: initial file is indexed. + poll_until( + &shared_picker, + Duration::from_secs(5), + "initial file src/main.rs indexed", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("main.rs")) + }, + ); + + // Create a new directory and write a file into it immediately. + // The file exists before the watch is registered — inject_existing_files + // in the owner thread catches it via a flat read_dir. + let new_dir = base.join("src/components"); + fs::create_dir_all(&new_dir).unwrap(); + fs::write( + new_dir.join("button.rs"), + "pub struct Button;\nconst TOKEN: &str = \"NEW_DIR_BUTTON_TOKEN\";\n", + ) + .unwrap(); + + // Wait for the watcher to detect the new directory + file. + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "file src/components/button.rs in new directory", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("button.rs")) + }, + ); + eprintln!( + " New directory + file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Also verify via grep that the content is accessible. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds NEW_DIR_BUTTON_TOKEN", + |picker| grep_plain_count(picker, "NEW_DIR_BUTTON_TOKEN") >= 1, + ); + + // And via fuzzy search. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "fuzzy search finds button.rs", + |picker| { + let results = fuzzy_search_paths(picker, "button"); + results.iter().any(|p| p.contains("button.rs")) + }, + ); +} + +/// Create a new directory, then create files AFTER a delay to ensure the +/// watch was established on the directory. +#[test] +fn file_created_after_directory_watch_established() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::create_dir_all(base.join("lib")).unwrap(); + fs::write(base.join("lib/utils.rs"), "pub fn helper() {}\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create the directory first, wait for the watcher to register it. + let new_dir = base.join("lib/models"); + fs::create_dir(&new_dir).unwrap(); + + // Wait long enough for the debouncer to flush + owner thread to add watch. + std::thread::sleep(Duration::from_millis(2000)); + + // Now write a file into the already-watched directory. + fs::write( + new_dir.join("user.rs"), + "pub struct User { name: String }\nconst TOKEN: &str = \"POST_WATCH_USER_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "file lib/models/user.rs created after directory watch", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("user.rs")) + }, + ); + eprintln!( + " Post-watch file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Grep sanity. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds POST_WATCH_USER_TOKEN", + |picker| grep_plain_count(picker, "POST_WATCH_USER_TOKEN") >= 1, + ); +} + +/// Create a deeply nested directory tree all at once with create_dir_all +/// and write a file at the leaf. The watcher must detect the top-level +/// directory via the parent's watch, inject_existing_files finds the file +/// at the leaf (and intermediate dirs get their own watches from Create +/// events on subsequent levels). +#[test] +fn deeply_nested_new_directories_detected() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::write(base.join("root.txt"), "root file\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create each level one at a time, waiting for each watch to register. + // inject_existing_files is flat (non-recursive), so deeply nested dirs + // need each parent to be watched before we can see files at the leaf. + fs::create_dir(base.join("app")).unwrap(); + std::thread::sleep(Duration::from_millis(2000)); + + fs::create_dir(base.join("app/services")).unwrap(); + std::thread::sleep(Duration::from_millis(2000)); + + fs::create_dir(base.join("app/services/auth")).unwrap(); + // Write the file immediately — inject_existing_files catches it. + fs::write( + base.join("app/services/auth/jwt.rs"), + "pub fn verify_token() {}\nconst TOKEN: &str = \"DEEP_NESTED_JWT_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "deeply nested file app/services/auth/jwt.rs", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("jwt.rs")) + }, + ); + eprintln!( + " Deeply nested file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Verify content is grepable. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds DEEP_NESTED_JWT_TOKEN", + |picker| grep_plain_count(picker, "DEEP_NESTED_JWT_TOKEN") >= 1, + ); + + // Now create a sibling at the same depth — the parent (app/services) + // is already watched, so this just needs the flat inject. + let sibling_dir = base.join("app/services/database"); + fs::create_dir(&sibling_dir).unwrap(); + fs::write( + sibling_dir.join("pool.rs"), + "pub struct ConnectionPool;\nconst TOKEN: &str = \"SIBLING_POOL_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "sibling nested file app/services/database/pool.rs", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("pool.rs")) + }, + ); + eprintln!( + " Sibling nested file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds SIBLING_POOL_TOKEN", + |picker| grep_plain_count(picker, "SIBLING_POOL_TOKEN") >= 1, + ); +} + +/// Create a new directory and immediately burst-write multiple files. +/// inject_existing_files catches all of them in one flat read_dir. +#[test] +fn burst_file_creation_in_new_directory() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::create_dir_all(base.join("src")).unwrap(); + fs::write(base.join("src/lib.rs"), "// lib\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create a new directory and immediately write 5 files. + let batch_dir = base.join("src/batch"); + fs::create_dir(&batch_dir).unwrap(); + + let file_count = 5; + for i in 0..file_count { + fs::write( + batch_dir.join(format!("item_{i}.rs")), + format!("pub struct Item{i};\nconst TOKEN: &str = \"BATCH_ITEM_{i}\";\n"), + ) + .unwrap(); + } + + // Wait for ALL files to appear. + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + &format!("all {file_count} batch files in src/batch/"), + |picker| { + let batch_count = picker + .get_files() + .iter() + .filter(|f| f.relative_path(picker).starts_with("src/batch/")) + .count(); + batch_count >= file_count + }, + ); + eprintln!( + " All {} burst files detected in {:.0}ms", + file_count, + elapsed.as_secs_f64() * 1000.0 + ); + + // Verify each file's content is grepable. + for i in 0..file_count { + let token = format!("BATCH_ITEM_{i}"); + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + &format!("grep finds {token}"), + |picker| grep_plain_count(picker, &token) >= 1, + ); + } +} + +/// Verify that gitignored directories created at runtime are NOT watched +/// and their files do NOT appear in the index. +#[test] +fn gitignored_new_directory_excluded() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::write(base.join("main.rs"), "fn main() {}\n").unwrap(); + // Ignore the build/ directory. + fs::write(base.join(".gitignore"), "build/\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create a gitignored directory with files. + let ignored_dir = base.join("build"); + fs::create_dir(&ignored_dir).unwrap(); + fs::write( + ignored_dir.join("output.rs"), + "const TOKEN: &str = \"IGNORED_BUILD_TOKEN\";\n", + ) + .unwrap(); + + // Also create a non-ignored directory to confirm the watcher works. + let good_dir = base.join("src"); + fs::create_dir(&good_dir).unwrap(); + fs::write( + good_dir.join("app.rs"), + "const TOKEN: &str = \"GOOD_SRC_TOKEN\";\n", + ) + .unwrap(); + + // Wait for the non-ignored file to appear (proves watcher is working). + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "non-ignored file src/app.rs appears", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("app.rs")) + }, + ); + + // Give extra time for any straggler events from the ignored dir. + std::thread::sleep(Duration::from_secs(2)); + + // The gitignored file must NOT be in the index. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let has_ignored = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("output.rs")); + assert!( + !has_ignored, + "Gitignored file build/output.rs should NOT be in the index" + ); + + let grep_count = grep_plain_count(picker, "IGNORED_BUILD_TOKEN"); + assert_eq!(grep_count, 0, "Gitignored content should NOT be grepable"); + } +}