✨ NEW: Replace gitignore-parser with ignore-python and add follow_links config

Agent-Logs-Url: https://github.com/useblocks/sphinx-codelinks/sessions/e9503678-3d44-4fee-9eb5-0d065cf4b5af

Co-authored-by: chrisjsewell <2997570+chrisjsewell@users.noreply.github.com>

Author: Copilot

pyproject.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 requires-python = ">= 3.12"
 dependencies = [
   "comment-parser>=1.2.4",
-  "gitignore-parser>=0.1.11",
+  "ignore-python>=0.3.3",
   "typer>=0.16.0",
   "click < 8.2",                # click 8.2.* produces empty errors if no args are given
   "jsonschema",

src/sphinx_codelinks/cmd.py

@@ -170,7 +170,7 @@ def analyse(  # noqa: PLR0912   # for CLI, so it needs the branches
 
 
 @app.command(no_args_is_help=True)
-def discover(
+def discover(  # noqa: PLR0913   # CLI command requires multiple parameters
     src_dir: Annotated[
         Path,
         typer.Argument(
@@ -203,9 +203,13 @@ def discover(
     gitignore: Annotated[
         bool,
         typer.Option(
-            help="Respect .gitignore in the given directory. Nested .gitignore Not supported"
+            help="Respect .gitignore files in the given directory and its parents"
         ),
     ] = True,
+    follow_links: Annotated[
+        bool,
+        typer.Option(help="Follow symbolic links during file discovery"),
+    ] = False,
     comment_type: Annotated[
         CommentType,
         typer.Option(
@@ -222,6 +226,7 @@ def discover(
         "exclude": exclude,
         "include": include,
         "gitignore": gitignore,
+        "follow_links": follow_links,
         "comment_type": comment_type,
     }
 

src/sphinx_codelinks/source_discover/config.py

@@ -30,6 +30,7 @@ class SourceDiscoverSectionConfigType(TypedDict, total=False):
     exclude: list[str]
     include: list[str]
     gitignore: bool
+    follow_links: bool
     comment_type: CommentType
 
 
@@ -40,6 +41,7 @@ class SourceDiscoverConfigType(TypedDict, total=False):
     exclude: list[str]
     include: list[str]
     gitignore: bool
+    follow_links: bool
     comment_type: CommentType
 
 
@@ -69,6 +71,9 @@ def field_names(cls) -> set[str]:
     gitignore: bool = field(default=True, metadata={"schema": {"type": "boolean"}})
     """Whether to respect .gitignore to exclude files."""
 
+    follow_links: bool = field(default=False, metadata={"schema": {"type": "boolean"}})
+    """Whether to follow symbolic links during file discovery."""
+
     comment_type: str = field(
         default="cpp",
         metadata={

src/sphinx_codelinks/source_discover/source_discover.py

@@ -1,11 +1,8 @@
-from collections.abc import Callable
-import fnmatch
 import os
 from pathlib import Path
 
-from gitignore_parser import (  # type: ignore[import-untyped]  # library has no stub
-    parse_gitignore,
-)
+from ignore import WalkBuilder
+from ignore.overrides import OverrideBuilder
 
 from sphinx_codelinks.source_discover.config import (
     COMMENT_FILETYPE,
@@ -17,51 +14,65 @@
 class SourceDiscover:
     def __init__(self, src_discover_config: SourceDiscoverConfig):
         self.src_discover_config = src_discover_config
-        # Only gitignore at source root is considered.
-        # TODO: Support nested gitignore files
-        gitignore_path = self.src_discover_config.src_dir / ".gitignore"
-        self.gitignore_matcher: Callable[[str], bool] | None = (
-            parse_gitignore(gitignore_path)
-            if self.src_discover_config.gitignore and gitignore_path.exists()
-            else None
-        )
         # normalize the file types to lower case with leading dot
         self.file_types = {
             f".{ext}" for ext in COMMENT_FILETYPE[src_discover_config.comment_type]
         }
 
         self.source_paths = self._discover()
 
+    def _build_overrides(self) -> OverrideBuilder | None:
+        """Build an OverrideBuilder for include/exclude patterns.
+
+        Include patterns are added as whitelist globs.
+        Exclude patterns are added as negated globs (prefixed with ``!``).
+        """
+        src_dir = str(self.src_discover_config.src_dir)
+        has_include = bool(self.src_discover_config.include)
+        has_exclude = bool(self.src_discover_config.exclude)
+
+        if not has_include and not has_exclude:
+            return None
+
+        ob = OverrideBuilder(src_dir)
+
+        if has_include:
+            for pattern in self.src_discover_config.include:
+                ob.add(pattern)
+
+        if has_exclude:
+            for pattern in self.src_discover_config.exclude:
+                ob.add(f"!{pattern}")
+
+        return ob
+
     def _discover(self) -> list[Path]:
         """Discover source files recursively in the given directory."""
+        src_dir = self.src_discover_config.src_dir
+        if not src_dir.is_dir():
+            return []
+
+        builder = WalkBuilder(str(src_dir))
+        builder.hidden(False)
+        builder.git_ignore(self.src_discover_config.gitignore)
+        builder.git_global(False)
+        builder.git_exclude(False)
+        builder.follow_links(self.src_discover_config.follow_links)
+
+        override_builder = self._build_overrides()
+        if override_builder is not None:
+            builder.overrides(override_builder.build())
+
         discovered_files = []
-        for filepath in self.src_discover_config.src_dir.rglob("*"):
-            if filepath.is_file():
-                if self.file_types and filepath.suffix.lower() not in self.file_types:
-                    continue
-                rel_filepath = str(
-                    filepath.relative_to(self.src_discover_config.src_dir)
-                )
-                if self.src_discover_config.include and self._matches_any(
-                    rel_filepath, self.src_discover_config.include
-                ):
-                    # "includes" has the highest priority over "gitignore" and "excludes"
-                    discovered_files.append(filepath)
-                    continue
-                if self.gitignore_matcher and self.gitignore_matcher(
-                    str(filepath.absolute())
-                ):
-                    continue
-                if self.src_discover_config.exclude and self._matches_any(
-                    rel_filepath, self.src_discover_config.exclude
-                ):
-                    continue
-                discovered_files.append(filepath)
+        for entry in builder.build():
+            filepath = entry.path()
+            if not filepath.is_file():
+                continue
+            if self.file_types and filepath.suffix.lower() not in self.file_types:
+                continue
+            discovered_files.append(filepath.resolve())
+
         sorted_filepaths = sorted(
             discovered_files, key=lambda x: os.path.normcase(os.path.normpath(x))
         )
         return sorted_filepaths
-
-    def _matches_any(self, rel_filepath: str, patterns: list[str]) -> bool:
-        """Check if the given file path matches any of the given patterns."""
-        return any(fnmatch.fnmatch(rel_filepath, pattern) for pattern in patterns)

src/sphinx_codelinks/sphinx_extension/directives/src_trace.py

@@ -213,6 +213,7 @@ def get_src_files(
                 gitignore=src_discover_config.gitignore,
                 include=src_discover_config.include,
                 exclude=src_discover_config.exclude,
+                follow_links=src_discover_config.follow_links,
                 comment_type=src_discover_config.comment_type,
             )
             source_discover = SourceDiscover(src_discover)

tests/test_source_discover.py

@@ -60,6 +60,15 @@
                 "Schema validation error in field 'comment_type': ['cpp', 'hpp'] is not of type 'string'"
             ],
         ),
+        (
+            {
+                "src_dir": "/path/to/root",
+                "follow_links": "not_a_bool",
+            },
+            [
+                "Schema validation error in field 'follow_links': 'not_a_bool' is not of type 'boolean'"
+            ],
+        ),
     ],
 )
 def test_schema_negative(config, msgs):
@@ -86,6 +95,10 @@ def test_schema_negative(config, msgs):
             "gitignore": True,
             "comment_type": "python",
         },
+        {
+            "src_dir": "/path/to/root",
+            "follow_links": True,
+        },
     ],
 )
 def test_schema_positive(config):
@@ -117,7 +130,7 @@ def test_schema_positive(config):
                 "exclude": ["charge/*.cpp"],
                 "include": ["**/*.cpp"],
             },
-            4,
+            2,
             "",
         ),
         (
@@ -174,3 +187,36 @@ def test_comment_filetype(
     )
     source_discover = SourceDiscover(config)
     assert len(source_discover.source_paths) == nums_files
+
+
+def test_follow_links(tmp_path: Path) -> None:
+    """Test that follow_links controls whether symbolic links are followed."""
+    # Create a real directory with a source file
+    real_dir = tmp_path / "real"
+    real_dir.mkdir()
+    (real_dir / "source.cpp").write_text("// test")
+
+    # Create a project directory with a symlink to the real directory
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "direct.cpp").write_text("// direct")
+    link = project_dir / "linked"
+    link.symlink_to(real_dir)
+
+    # Without follow_links, symlinked files should not be discovered
+    config_no_follow = SourceDiscoverConfig(
+        src_dir=project_dir, gitignore=False, follow_links=False
+    )
+    discover_no_follow = SourceDiscover(config_no_follow)
+    discovered_names = {p.name for p in discover_no_follow.source_paths}
+    assert "direct.cpp" in discovered_names
+    assert "source.cpp" not in discovered_names
+
+    # With follow_links, symlinked files should be discovered
+    config_follow = SourceDiscoverConfig(
+        src_dir=project_dir, gitignore=False, follow_links=True
+    )
+    discover_follow = SourceDiscover(config_follow)
+    discovered_names = {p.name for p in discover_follow.source_paths}
+    assert "direct.cpp" in discovered_names
+    assert "source.cpp" in discovered_names