feat!: Revamp & optimize the way paths are stored

This is a very crucial redesign that actually required a lot of
consideration to move forward but now it allows us to save a lot of RAM,
acutally improves query time and gives an ability to query directories
for free

Author: dmtrKovalenko

Cargo.toml

@@ -34,7 +34,7 @@ mimalloc = "0.1.47"
 zlob = "1.3.0"
 
 mlua = { version = "0.11.1", features = ["module", "luajit"] }
-neo_frizbee = { version = "0.9.1", features = ["match_end_col"] }
+neo_frizbee = { version = "0.10.0", features = ["match_end_col"] }
 notify = { version = "9.0.0-rc.2" }
 notify-debouncer-full = { package="fff-notify-debouncer-full", version = "0.9.0" }
 once_cell = "1.20.2"

crates/fff-c/include/fff.h

@@ -64,7 +64,6 @@ typedef struct FffResult {
  * Free the entire result with `fff_free_search_result`.
  */
 typedef struct FffFileItem {
-  char *path;
   char *relative_path;
   char *file_name;
   char *git_status;
@@ -156,7 +155,6 @@ typedef struct FffMatchRange {
  * `FffGrepResult` with `fff_free_grep_result` to release everything.
  */
 typedef struct FffGrepMatch {
-  char *path;
   char *relative_path;
   char *file_name;
   char *git_status;
@@ -380,6 +378,17 @@ struct FffResult *fff_scan_files(void *fff_handle);
  */
 bool fff_is_scanning(void *fff_handle);
 
+/**
+ * Get the base path of the file picker.
+ *
+ * Returns an `FffResult` with a heap-allocated C string in the `handle`
+ * field. Free the string with `fff_free_string` after reading it.
+ *
+ * ## Safety
+ * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+ */
+struct FffResult *fff_get_base_path(void *fff_handle);
+
 /**
  * Get scan progress information.
  *

crates/fff-c/src/ffi_types.rs

@@ -7,13 +7,10 @@
 use std::ffi::{CString, c_char, c_void};
 use std::ptr;
 
+use fff::file_picker::FilePicker;
 use fff::git::format_git_status;
 use fff::{FileItem, GrepMatch, GrepResult, Location, Score, SearchResult};
 
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
 /// Allocate a heap CString from a `&str`, returning a raw pointer.
 fn cstring_new(s: &str) -> *mut c_char {
     CString::new(s).unwrap_or_default().into_raw()
@@ -64,7 +61,6 @@ unsafe fn free_cstring_array(arr: *mut *mut c_char, count: u32) {
 /// Free the entire result with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffFileItem {
-    pub path: *mut c_char,
     pub relative_path: *mut c_char,
     pub file_name: *mut c_char,
     pub git_status: *mut c_char,
@@ -77,11 +73,11 @@ pub struct FffFileItem {
 }
 
 impl FffFileItem {
-    pub fn from_item(item: &FileItem, arena: *const u8) -> Self {
+    pub fn from_item(item: &FileItem, picker: &FilePicker) -> Self {
+        let rel_path = picker.relative_path(item);
         FffFileItem {
-            path: cstring_new(&item.relative_path(arena)),
-            relative_path: cstring_new(&item.relative_path(arena)),
-            file_name: cstring_new(&item.file_name(arena)),
+            relative_path: cstring_new(&rel_path),
+            file_name: cstring_new(&picker.file_name(item)),
             git_status: cstring_new(format_git_status(item.git_status)),
             size: item.size,
             modified: item.modified,
@@ -98,9 +94,6 @@ impl FffFileItem {
     /// All string pointers must have been allocated by `CString::into_raw`.
     pub unsafe fn free_strings(&mut self) {
         unsafe {
-            if !self.path.is_null() {
-                drop(CString::from_raw(self.path));
-            }
             if !self.relative_path.is_null() {
                 drop(CString::from_raw(self.relative_path));
             }
@@ -232,11 +225,11 @@ pub struct FffSearchResult {
 
 impl FffSearchResult {
     /// Convert a core `SearchResult` into a heap-allocated `FffSearchResult`.
-    pub fn from_core(result: &SearchResult, arena: *const u8) -> *mut Self {
+    pub fn from_core(result: &SearchResult, picker: &FilePicker) -> *mut Self {
         let items: Vec<FffFileItem> = result
             .items
             .iter()
-            .map(|i| FffFileItem::from_item(i, arena))
+            .map(|i| FffFileItem::from_item(i, picker))
             .collect();
         let scores: Vec<FffScore> = result.scores.iter().map(FffScore::from).collect();
         let count = items.len() as u32;
@@ -273,7 +266,6 @@ pub struct FffMatchRange {
 #[repr(C)]
 pub struct FffGrepMatch {
     // -- pointers (8 bytes each) --
-    pub path: *mut c_char,
     pub relative_path: *mut c_char,
     pub file_name: *mut c_char,
     pub git_status: *mut c_char,
@@ -303,7 +295,7 @@ pub struct FffGrepMatch {
 }
 
 impl FffGrepMatch {
-    fn from_core_with_file(m: &GrepMatch, file: &FileItem, arena: *const u8) -> Self {
+    fn from_core_with_file(m: &GrepMatch, file: &FileItem, picker: &FilePicker) -> Self {
         let ranges: Vec<FffMatchRange> = m
             .match_byte_offsets
             .iter()
@@ -317,10 +309,10 @@ impl FffGrepMatch {
             None => (false, 0),
         };
 
+        let rel_path = picker.relative_path(file);
         FffGrepMatch {
-            path: cstring_new(&file.relative_path(arena)),
-            relative_path: cstring_new(&file.relative_path(arena)),
-            file_name: cstring_new(&file.file_name(arena)),
+            relative_path: cstring_new(&rel_path),
+            file_name: cstring_new(&picker.file_name(file)),
             git_status: cstring_new(format_git_status(file.git_status)),
             line_content: cstring_new(&m.line_content),
             match_ranges,
@@ -348,9 +340,6 @@ impl FffGrepMatch {
     /// All pointers must have been allocated by the corresponding `from_core`.
     pub unsafe fn free_fields(&mut self) {
         unsafe {
-            if !self.path.is_null() {
-                drop(CString::from_raw(self.path));
-            }
             if !self.relative_path.is_null() {
                 drop(CString::from_raw(self.relative_path));
             }
@@ -401,13 +390,13 @@ pub struct FffGrepResult {
 
 impl FffGrepResult {
     /// Convert a core `GrepResult` into a heap-allocated `FffGrepResult`.
-    pub fn from_core(result: &GrepResult, arena: *const u8) -> *mut Self {
+    pub fn from_core(result: &GrepResult, picker: &FilePicker) -> *mut Self {
         let items: Vec<FffGrepMatch> = result
             .matches
             .iter()
             .map(|m| {
                 let file = result.files[m.file_index];
-                FffGrepMatch::from_core_with_file(m, file, arena)
+                FffGrepMatch::from_core_with_file(m, file, picker)
             })
             .collect();
         let (items_ptr, count) = vec_to_raw(items);

crates/fff-c/src/lib.rs

@@ -294,9 +294,7 @@ pub unsafe extern "C" fn fff_search(
     let parser = QueryParser::default();
     let parsed = parser.parse(query_str);
 
-    let arena = picker.arena_base_ptr();
-    let results = FilePicker::fuzzy_search(
-        picker.get_files(),
+    let results = picker.fuzzy_search(
         &parsed,
         query_tracker_ref,
         FuzzySearchOptions {
@@ -310,10 +308,9 @@ pub unsafe extern "C" fn fff_search(
                 limit: page_size,
             },
         },
-        arena,
     );
 
-    let search_result = FffSearchResult::from_core(&results, arena);
+    let search_result = FffSearchResult::from_core(&results, picker);
     FffResult::ok_handle(search_result as *mut c_void)
 }
 
@@ -393,11 +390,11 @@ pub unsafe extern "C" fn fff_live_grep(
         after_context: after_context as usize,
         classify_definitions,
         trim_whitespace: false,
+        abort_signal: None,
     };
 
-    let arena = picker.arena_base_ptr();
     let result = picker.grep(&parsed, &options);
-    let grep_result = FffGrepResult::from_core(&result, arena);
+    let grep_result = FffGrepResult::from_core(&result, picker);
     FffResult::ok_handle(grep_result as *mut c_void)
 }
 
@@ -496,23 +493,11 @@ pub unsafe extern "C" fn fff_multi_grep(
         after_context: after_context as usize,
         classify_definitions,
         trim_whitespace: false,
+        abort_signal: None,
     };
 
-    let arena = picker.arena_base_ptr();
-    let overlay_guard = picker.bigram_overlay().map(|o| o.read());
-    let result = fff::multi_grep_search(
-        picker.get_files(),
-        &patterns,
-        constraint_refs,
-        &options,
-        picker.cache_budget(),
-        picker.bigram_index(),
-        overlay_guard.as_deref(),
-        None,
-        picker.base_path(),
-        arena,
-    );
-    let grep_result = FffGrepResult::from_core(&result, arena);
+    let result = picker.multi_grep(&patterns, constraint_refs, &options);
+    let grep_result = FffGrepResult::from_core(&result, picker);
     FffResult::ok_handle(grep_result as *mut c_void)
 }
 
@@ -561,6 +546,33 @@ pub unsafe extern "C" fn fff_is_scanning(fff_handle: *mut c_void) -> bool {
         .unwrap_or(false)
 }
 
+/// Get the base path of the file picker.
+///
+/// Returns an `FffResult` with a heap-allocated C string in the `handle`
+/// field. Free the string with `fff_free_string` after reading it.
+///
+/// ## Safety
+/// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_get_base_path(fff_handle: *mut c_void) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+
+    let guard = match inst.picker.read() {
+        Ok(g) => g,
+        Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
+    };
+
+    let picker = match guard.as_ref() {
+        Some(p) => p,
+        None => return FffResult::err("File picker not initialized"),
+    };
+
+    FffResult::ok_string(&picker.base_path().to_string_lossy())
+}
+
 /// Get scan progress information.
 ///
 /// ## Safety

crates/fff-core/Cargo.toml

@@ -10,6 +10,18 @@ description = "Faboulous & Fast File Finder - a fast and extremely correct file
 path = "src/lib.rs"
 crate-type = ["rlib", "staticlib", "cdylib"]
 
+[[bench]]
+name = "parse_bench"
+harness = false
+
+[[bench]]
+name = "bigram_bench"
+harness = false
+
+[[bench]]
+name = "memmem_bench"
+harness = false
+
 [features]
 default = []
 # Enable C FFI exports
@@ -27,7 +39,7 @@ rayon = { workspace = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
-fff-query-parser = { workspace = true  , version = "0.5.2" }
+fff-query-parser = { workspace = true }
 
 # External dependencies
 bindet = { workspace = true }
@@ -38,7 +50,7 @@ libc = "0.2"
 git2 = { workspace = true }
 glidesort = { workspace = true }
 globset = { workspace = true }
-fff-grep = { workspace = true  , version = "0.5.2" }
+fff-grep = { workspace = true }
 aho-corasick = "1"
 memchr = "2"
 heed = { workspace = true }
@@ -69,23 +81,3 @@ dunce = { workspace = true }
 criterion = { version = "0.5", features = ["html_reports"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"
-
-[[bench]]
-name = "parse_bench"
-harness = false
-
-[[bench]]
-name = "bigram_bench"
-harness = false
-
-[[bench]]
-name = "memmem_bench"
-harness = false
-
-[[bin]]
-name = "bench_segmented"
-path = "src/bin/bench_segmented.rs"
-
-[[bin]]
-name = "bench_chunked"
-path = "src/bin/bench_chunked.rs"

crates/fff-core/src/background_watcher.rs

@@ -621,7 +621,7 @@ fn is_git_file(path: &Path) -> bool {
         .any(|component| component.as_os_str() == ".git")
 }
 
-pub fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option<Repository>) -> bool {
+fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option<Repository>) -> bool {
     let Some(repo) = repo.as_ref() else {
         return false;
     };