feat: implement Git-based real-time incremental sync

CONTEXT.md

@@ -0,0 +1,29 @@
+# GrepTurbo Context
+
+GrepTurbo provides index-accelerated regex search that stays in sync with real-time code changes using Git.
+
+## Language
+
+**Baseline**:
+The immutable on-disk index tied to a specific Git commit.
+_Avoid_: Old index, Disk index, Saved index
+
+**Overlay**:
+The transient in-memory index of uncommitted (modified, added, or untracked) files detected during search.
+_Avoid_: Live index, Delta index, Fresh index
+
+**Commit Drift**:
+The state where the current Git HEAD differs from the commit hash stored in the **Baseline**.
+
+**Tombstone**:
+A list of file paths that have been deleted since the **Baseline** was built, used to filter out dead results.
+
+## Relationships
+
+- A **Search** merges results from the **Baseline** and the **Overlay**, while filtering out paths in the **Tombstone** set.
+- **Commit Drift** triggers an automatic rebuild of the **Baseline**.
+
+## Example dialogue
+
+> **Dev:** "If I just edited a file but haven't committed it, will the search find it?"
+> **Domain expert:** "Yes, the **Overlay** will index your unsaved changes and merge them with the **Baseline** results."

GEMINI.md

@@ -0,0 +1 @@
+AGENTS.md

cmd/grepturbo/main.go

@@ -86,7 +86,25 @@ func runSearch(idxDir, pattern string) error {
 
 	matches, err := query.Search(r, pattern)
 	if err != nil {
-		return fmt.Errorf("error searching: %w", err)
+		if drift, ok := err.(*query.ErrCommitDrift); ok {
+			fmt.Fprintf(os.Stderr, "Notice: %s. Rebuilding...\n", drift.Error())
+			r.Close() // close before rebuild
+			if err := runBuild(r.Meta.RootDir, idxDir, r.Meta.Skip); err != nil {
+				return err
+			}
+			// Re-open and try search again
+			r2, err := index.NewReader(idxDir)
+			if err != nil {
+				return err
+			}
+			defer r2.Close()
+			matches, err = query.Search(r2, pattern)
+			if err != nil {
+				return err
+			}
+		} else {
+			return fmt.Errorf("error searching: %w", err)
+		}
 	}
 
 	if len(matches) == 0 {

docs/adr/0001-git-sync-strategy.md

@@ -0,0 +1,18 @@
+# 0001-git-sync-strategy
+
+To ensure search results reflect real-time code changes without the overhead of a background daemon or full index rebuilds, we are implementing a Git-based on-demand synchronization strategy.
+
+### Context
+Users expect the search index to be "always fresh," but rebuilding a trigram index on every search is too slow. Approaches like background file watchers introduce complexity (daemons, locking), and content-hashing requires reading every file (defeating the index's performance).
+
+### Decision
+We will use `git status` to identify modified, deleted, and untracked files during every search.
+- **Baseline**: The on-disk index is tied to a specific Git commit hash stored in `metadata.json`.
+- **Overlay**: Dirty files (modified/untracked) are re-indexed in-memory on the fly and their results are merged with the Baseline.
+- **Tombstones**: Deleted files are filtered out of the Baseline results.
+- **Commit Drift**: If the current Git HEAD differs from the Baseline hash, an automatic full rebuild is triggered.
+
+### Consequences
+- **Trade-off**: Search depends on Git being present in the repository.
+- **Accuracy**: Search results are 100% accurate relative to the current working tree.
+- **Performance**: High-volume changes (thousands of uncommitted files) will increase search latency as they must be indexed in-memory, but we choose to prioritize accuracy over a "too many changes" threshold for now.

internal/index/builder.go

@@ -17,8 +17,10 @@ const maxFileSize = 1 << 20 // 1 MB — skip files larger than this
 // Builder walks a directory, extracts trigrams from each file,
 // and accumulates an in-memory posting list.
 type Builder struct {
-	Posts posting.List // trigram → sorted []fileID
-	Files []string     // fileID → filepath (index == fileID)
+	Posts   posting.List // trigram → sorted []fileID
+	Files   []string     // fileID → filepath (index == fileID)
+	RootDir string
+	Skip    []string
 }
 
 func NewBuilder() *Builder {
@@ -76,6 +78,13 @@ type extractResult struct {
 // Directories listed in skip are skipped entirely (e.g. "node_modules").
 // Directories and files that fail to read are silently skipped.
 func (b *Builder) Build(rootDir string, skip ...string) error {
+	absRoot, err := filepath.Abs(rootDir)
+	if err != nil {
+		return err
+	}
+	b.RootDir = absRoot
+	b.Skip = skip
+
 	skipSet := make(map[string]bool)
 	for k, v := range defaultSkipDirs {
 		skipSet[k] = v
@@ -126,7 +135,7 @@ func (b *Builder) Build(rootDir string, skip ...string) error {
 		close(done)
 	}()
 
-	err := filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
+	err = filepath.WalkDir(absRoot, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return nil
 		}

internal/index/metadata.go

@@ -0,0 +1,61 @@
+package index
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// Metadata contains information about the built Baseline index.
+type Metadata struct {
+	Commit  string   `json:"commit"`
+	RootDir string   `json:"root_dir"`
+	Skip    []string `json:"skip"`
+}
+
+// WriteMetadata saves index metadata to metadata.json in dir.
+func WriteMetadata(dir, rootDir string, skip []string) error {
+	commit, err := CurrentCommit(rootDir)
+	if err != nil {
+		commit = "unknown"
+	}
+
+	m := Metadata{
+		Commit:  commit,
+		RootDir: rootDir,
+		Skip:    skip,
+	}
+	data, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)
+}
+
+// ReadMetadata loads index metadata from metadata.json in dir.
+func ReadMetadata(dir string) (*Metadata, error) {
+	data, err := os.ReadFile(filepath.Join(dir, "metadata.json"))
+	if err != nil {
+		return nil, err
+	}
+
+	var m Metadata
+	if err := json.Unmarshal(data, &m); err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+// CurrentCommit returns the current Git HEAD commit hash in the given dir.
+func CurrentCommit(dir string) (string, error) {
+	cmd := exec.Command("git", "rev-parse", "HEAD")
+	cmd.Dir = dir
+	out, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}

internal/index/reader.go

@@ -15,14 +15,21 @@ import (
 // Reader holds the mmap'd lookup table and an open handle to postings.dat.
 // Use NewReader to open, and Close when done.
 type Reader struct {
-	table    []byte   // mmap'd contents of lookup.idx
-	numSlots uint32   // number of slots in the hash table
-	postings *os.File // open handle to postings.dat for random reads
-	Files    []string // fileID → filepath
+	table    []byte    // mmap'd contents of lookup.idx
+	numSlots uint32    // number of slots in the hash table
+	postings *os.File  // open handle to postings.dat for random reads
+	Files    []string  // fileID → filepath
+	Meta     *Metadata // index metadata (e.g. baseline commit)
 }
 
 // NewReader opens the index written by Write and mmap's the lookup table.
 func NewReader(dir string) (*Reader, error) {
+	// ── metadata.json ───────────────────────────────────────────────────────
+	meta, err := ReadMetadata(dir)
+	if err != nil {
+		return nil, fmt.Errorf("read metadata.json: %w", err)
+	}
+
 	// ── lookup.idx ──────────────────────────────────────────────────────────
 	lookupPath := filepath.Join(dir, "lookup.idx")
 	lf, err := os.Open(lookupPath)
@@ -77,6 +84,7 @@ func NewReader(dir string) (*Reader, error) {
 		numSlots: numSlots,
 		postings: pf,
 		Files:    files,
+		Meta:     meta,
 	}, nil
 }
 

internal/index/sync.go

@@ -0,0 +1,121 @@
+package index
+
+import (
+	"bufio"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+
+	"grepturbo/internal/posting"
+	"grepturbo/internal/trigram"
+)
+
+// GitStatus holds the lists of files that have changed since the baseline.
+type GitStatus struct {
+	Modified  []string
+	Untracked []string
+	Deleted   []string
+}
+
+// Overlay holds the transient in-memory index of dirty files.
+type Overlay struct {
+	Posts      posting.List
+	Files      []string        // fileID → filepath (starts from len(Baseline.Files))
+	Tombstones map[string]bool // paths that should be ignored from Baseline
+}
+
+// Sync performs a Git-based synchronization.
+func (r *Reader) Sync() (*Overlay, bool, error) {
+	current, err := CurrentCommit(r.Meta.RootDir)
+	if err != nil {
+		// Not in a git repo (or git not installed) — nothing to sync.
+		return &Overlay{
+			Posts:      make(posting.List),
+			Tombstones: make(map[string]bool),
+		}, false, nil
+	}
+
+	// Commit Drift detected
+	if r.Meta.Commit != current && r.Meta.Commit != "unknown" {
+		return nil, true, nil
+	}
+
+	status, err := GetGitStatus(r.Meta.RootDir)
+	if err != nil {
+		return nil, false, err
+	}
+
+	overlay := &Overlay{
+		Posts:      make(posting.List),
+		Tombstones: make(map[string]bool),
+	}
+
+	// Deleted files are Tombstones
+	for _, p := range status.Deleted {
+		overlay.Tombstones[filepath.Join(r.Meta.RootDir, p)] = true
+	}
+
+	// Modified files are both Tombstones (hide old version) and Indexed (show new version)
+	dirtyFiles := append(status.Modified, status.Untracked...)
+	for _, p := range status.Modified {
+		overlay.Tombstones[filepath.Join(r.Meta.RootDir, p)] = true
+	}
+
+	// Index dirty files in memory
+	for _, relPath := range dirtyFiles {
+		absPath := filepath.Join(r.Meta.RootDir, relPath)
+		data, err := os.ReadFile(absPath)
+		if err != nil {
+			continue // skip files we can't read
+		}
+		if !utf8.Valid(data) || len(data) > maxFileSize {
+			continue
+		}
+
+		fileID := uint32(len(r.Files) + len(overlay.Files))
+		overlay.Files = append(overlay.Files, absPath)
+
+		for _, t := range trigram.Extract(string(data)) {
+			overlay.Posts.AddBatch(t, []uint32{fileID})
+		}
+	}
+	overlay.Posts.Finalize()
+
+	return overlay, false, nil
+}
+
+// GetGitStatus runs 'git status --porcelain' in dir and returns the categorized files.
+func GetGitStatus(dir string) (*GitStatus, error) {
+	cmd := exec.Command("git", "status", "--porcelain")
+	cmd.Dir = dir
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	status := &GitStatus{}
+	scanner := bufio.NewScanner(strings.NewReader(string(out)))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) < 3 {
+			continue
+		}
+
+		// git status --porcelain format: "XY PATH"
+		xy := line[:2]
+		path := line[3:]
+
+		switch {
+		case xy == "??":
+			status.Untracked = append(status.Untracked, path)
+		case strings.Contains(xy, "D"):
+			status.Deleted = append(status.Deleted, path)
+		case strings.Contains(xy, "M") || strings.Contains(xy, "A"):
+			status.Modified = append(status.Modified, path)
+		}
+	}
+
+	return status, scanner.Err()
+}