fix: skip files ignored by gitignore (#78)

Fixes #62

- add pathspec dependency and load project .gitignore

- apply gitignore-aware matcher that handles nested rules

- add regression test ensuring ignored files stay out of the index

Co-authored-by: Clawdbot <bot@clawd.bot>

Author: zerone0x

pyproject.toml

@@ -31,6 +31,7 @@ dependencies = [
     "einops>=0.8.2",
     "typer>=0.9.0",
     "msgspec>=0.19.0",
+    "pathspec>=0.12.1",
     "pyyaml>=6.0",
 ]
 

src/cocoindex_code/indexer.py

@@ -2,16 +2,20 @@
 
 from __future__ import annotations
 
+from pathlib import Path, PurePath
+from collections.abc import Iterable
+
 import cocoindex as coco
 from cocoindex.connectors import localfs, sqlite
 from cocoindex.connectors.sqlite import Vec0TableDef
 from cocoindex.ops.text import RecursiveSplitter, detect_code_language
 from cocoindex.resources.chunk import Chunk
-from cocoindex.resources.file import PatternFilePathMatcher
+from cocoindex.resources.file import FilePathMatcher, PatternFilePathMatcher
 from cocoindex.resources.id import IdGenerator
+from pathspec import GitIgnoreSpec
 
 from .settings import PROJECT_SETTINGS
-from .shared import CODEBASE_DIR, EMBEDDER, EXT_LANG_OVERRIDE_MAP, SQLITE_DB, CodeChunk
+from .shared import CODEBASE_DIR, EMBEDDER, EXT_LANG_OVERRIDE_MAP, GITIGNORE_SPEC, SQLITE_DB, CodeChunk
 
 # Chunking configuration
 CHUNK_SIZE = 800
@@ -22,6 +26,107 @@
 splitter = RecursiveSplitter()
 
 
+def _normalize_gitignore_lines(lines: Iterable[str], directory: PurePath) -> list[str]:
+    """Normalize .gitignore lines to root-relative gitignore patterns."""
+    if directory in (PurePath("."), PurePath("")):
+        prefix = ""
+    else:
+        prefix = f"{directory.as_posix().rstrip('/')}/"
+
+    normalized: list[str] = []
+    for raw_line in lines:
+        line = raw_line.rstrip("\n\r")
+        if not line:
+            continue
+        stripped = line.lstrip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        if line.startswith("\\#") or line.startswith("\\!"):
+            line = line[1:]
+        negated = line.startswith("!")
+        if negated:
+            line = line[1:]
+        body = line.strip()
+        if not body:
+            continue
+        anchor = body.startswith("/")
+        if anchor:
+            body = body.lstrip("/")
+            pattern = f"{prefix}{body}" if prefix else body
+        else:
+            contains_slash = "/" in body
+            base = prefix
+            if contains_slash:
+                pattern = f"{base}{body}"
+            else:
+                if base:
+                    pattern = f"{base}**/{body}"
+                else:
+                    pattern = f"**/{body}"
+        if negated:
+            pattern = f"!{pattern}"
+        normalized.append(pattern)
+    return normalized
+
+
+class GitignoreAwareMatcher(FilePathMatcher):
+    """Wraps another matcher and applies .gitignore filtering."""
+
+    def __init__(
+        self,
+        delegate: FilePathMatcher,
+        root_spec: GitIgnoreSpec | None,
+        project_root: Path,
+    ) -> None:
+        self._delegate = delegate
+        self._root = project_root
+        self._spec_cache: dict[PurePath, GitIgnoreSpec | None] = {PurePath("."): root_spec}
+
+    def _spec_for(self, directory: PurePath) -> GitIgnoreSpec | None:
+        if directory in self._spec_cache:
+            return self._spec_cache[directory]
+
+        parent_dir = directory.parent if directory != PurePath(".") else PurePath(".")
+        parent_spec = self._spec_for(parent_dir)
+        spec = parent_spec
+
+        gitignore_path = (self._root / directory) / ".gitignore"
+        if gitignore_path.is_file():
+            try:
+                lines = gitignore_path.read_text().splitlines()
+            except (OSError, UnicodeDecodeError):
+                lines = []
+            normalized = _normalize_gitignore_lines(lines, directory)
+            if normalized:
+                new_spec = GitIgnoreSpec.from_lines(normalized)
+                spec = new_spec if spec is None else spec + new_spec
+
+        self._spec_cache[directory] = spec
+        return spec
+
+    def _is_ignored(self, path: PurePath, is_dir: bool) -> bool:
+        directory = path if is_dir else path.parent
+        if directory == PurePath(""):
+            directory = PurePath(".")
+        spec = self._spec_for(directory)
+        if spec is None:
+            return False
+        match_path = path.as_posix()
+        if is_dir and not match_path.endswith("/"):
+            match_path = f"{match_path}/"
+        return spec.match_file(match_path)
+
+    def is_dir_included(self, path: PurePath) -> bool:
+        if self._is_ignored(path, True):
+            return False
+        return self._delegate.is_dir_included(path)
+
+    def is_file_included(self, path: PurePath) -> bool:
+        if self._is_ignored(path, False):
+            return False
+        return self._delegate.is_file_included(path)
+
+
 @coco.fn(memo=True)
 async def process_file(
     file: localfs.File,
@@ -76,6 +181,8 @@ async def process(chunk: Chunk) -> None:
 async def indexer_main() -> None:
     """Main indexing function - walks files and processes each."""
     ps = coco.use_context(PROJECT_SETTINGS)
+    gitignore_spec = coco.use_context(GITIGNORE_SPEC)
+    project_root = coco.use_context(CODEBASE_DIR)
 
     table = await sqlite.mount_table_target(
         db=SQLITE_DB,
@@ -90,13 +197,16 @@ async def indexer_main() -> None:
         ),
     )
 
+    base_matcher = PatternFilePathMatcher(
+        included_patterns=ps.include_patterns,
+        excluded_patterns=ps.exclude_patterns,
+    )
+    matcher: FilePathMatcher = GitignoreAwareMatcher(base_matcher, gitignore_spec, project_root)
+
     files = localfs.walk_dir(
         CODEBASE_DIR,
         recursive=True,
-        path_matcher=PatternFilePathMatcher(
-            included_patterns=ps.include_patterns,
-            excluded_patterns=ps.exclude_patterns,
-        ),
+        path_matcher=matcher,
     )
 
     with coco.component_subpath(coco.Symbol("process_file")):

src/cocoindex_code/project.py

@@ -11,8 +11,8 @@
 
 from .indexer import indexer_main
 from .protocol import IndexingProgress
-from .settings import PROJECT_SETTINGS, ProjectSettings
-from .shared import CODEBASE_DIR, EMBEDDER, EXT_LANG_OVERRIDE_MAP, SQLITE_DB, Embedder
+from .settings import PROJECT_SETTINGS, ProjectSettings, load_gitignore_spec
+from .shared import CODEBASE_DIR, EMBEDDER, EXT_LANG_OVERRIDE_MAP, GITIGNORE_SPEC, SQLITE_DB, Embedder
 
 
 class Project:
@@ -88,6 +88,7 @@ async def create(
         target_sqlite_db_path = index_dir / "target_sqlite.db"
 
         settings = coco.Settings.from_env(cocoindex_db_path)
+        gitignore_spec = load_gitignore_spec(project_root)
 
         context = coco.ContextProvider()
         context.provide(CODEBASE_DIR, project_root)
@@ -98,6 +99,7 @@ async def create(
             EXT_LANG_OVERRIDE_MAP,
             {f".{lo.ext}": lo.lang for lo in project_settings.language_overrides},
         )
+        context.provide(GITIGNORE_SPEC, gitignore_spec)
 
         env = coco.Environment(settings, context_provider=context)
         app = coco.App(

src/cocoindex_code/settings.py

@@ -8,6 +8,7 @@
 
 import cocoindex as _coco
 import yaml as _yaml
+from pathspec import GitIgnoreSpec
 
 # ---------------------------------------------------------------------------
 # Default file patterns (moved from indexer.py)
@@ -189,6 +190,20 @@ def find_parent_with_marker(start: Path) -> Path | None:
         current = parent
 
 
+def load_gitignore_spec(project_root: Path) -> GitIgnoreSpec | None:
+    """Load a GitIgnoreSpec for the project's ``.gitignore`` if present."""
+    gitignore = project_root / ".gitignore"
+    if not gitignore.is_file():
+        return None
+    try:
+        lines = gitignore.read_text().splitlines()
+    except (OSError, UnicodeDecodeError):
+        return None
+    if not lines:
+        return None
+    return GitIgnoreSpec.from_lines(lines)
+
+
 # ---------------------------------------------------------------------------
 # Serialization helpers
 # ---------------------------------------------------------------------------

src/cocoindex_code/shared.py

@@ -9,6 +9,7 @@
 
 import cocoindex as coco
 from cocoindex.connectors import sqlite
+from pathspec import GitIgnoreSpec
 from numpy.typing import NDArray
 
 if TYPE_CHECKING:
@@ -31,6 +32,7 @@
 EMBEDDER = coco.ContextKey[Embedder]("embedder")
 SQLITE_DB = coco.ContextKey[sqlite.ManagedConnection]("index_db", tracked=False)
 CODEBASE_DIR = coco.ContextKey[pathlib.Path]("codebase", tracked=False)
+GITIGNORE_SPEC = coco.ContextKey[GitIgnoreSpec | None]("gitignore_spec", tracked=False)
 EXT_LANG_OVERRIDE_MAP = coco.ContextKey[dict[str, str]]("ext_lang_override_map")
 
 # Module-level variable — set by daemon at startup (needed for CodeChunk annotation).

tests/test_e2e.py

@@ -16,6 +16,7 @@
 import pytest
 from typer.testing import CliRunner
 
+from cocoindex.connectors import sqlite as coco_sqlite
 from cocoindex_code.cli import app
 from cocoindex_code.client import stop_daemon
 from cocoindex_code.settings import find_parent_with_marker
@@ -298,6 +299,37 @@ def test_session_reset_then_full_reinit(e2e_project: Path) -> None:
     assert "main.py" in result.output
 
 
+def test_session_respects_gitignore(e2e_project: Path) -> None:
+    """Indexing should skip files ignored by .gitignore while honoring negations."""
+    gitignore_path = e2e_project / ".gitignore"
+    gitignore_path.write_text("ignored.py\nignored_dir/\n!important.py\n")
+
+    (e2e_project / "ignored.py").write_text("IGNORED_TOKEN = True\n")
+    ignored_dir = e2e_project / "ignored_dir"
+    ignored_dir.mkdir()
+    (ignored_dir / "nested.py").write_text("NESTED_IGNORED = True\n")
+    (e2e_project / "important.py").write_text("IMPORTANT_TOKEN = True\n")
+
+    runner.invoke(app, ["init"], catch_exceptions=False)
+    result = runner.invoke(app, ["index"], catch_exceptions=False)
+    assert result.exit_code == 0, result.output
+
+    db_path = e2e_project / ".cocoindex_code" / "target_sqlite.db"
+    conn = coco_sqlite.connect(str(db_path), load_vec=True)
+    try:
+        with conn.readonly() as db:
+            file_paths = {
+                row[0]
+                for row in db.execute("SELECT DISTINCT file_path FROM code_chunks_vec")
+            }
+    finally:
+        conn.close()
+
+    assert "ignored.py" not in file_paths
+    assert "ignored_dir/nested.py" not in file_paths
+    assert "important.py" in file_paths
+
+
 @pytest.mark.usefixtures("e2e_project")
 def test_session_daemon_stop_and_auto_start() -> None:
     """Init → index → daemon stop → index auto-starts daemon → search works."""