fix(sync): ignore hidden paths relative to watched project (#815)

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/sync/watch_service.py

@@ -93,6 +93,7 @@ def __init__(
         self.status_path = app_config.data_dir_path / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
         self._ignore_patterns_cache: dict[Path, Set[str]] = {}
+        self._sorted_watch_filter_roots: tuple[Path, ...] | None = None
         self._sync_service_factory = sync_service_factory
         # When set (typically from BASIC_MEMORY_MCP_PROJECT), the watch cycle
         # only observes this project. Without it, each `basic-memory mcp --project X`
@@ -126,41 +127,54 @@ def _get_ignore_patterns(self, project_path: Path) -> Set[str]:
     async def _watch_projects_cycle(self, projects: Sequence[Project], stop_event: asyncio.Event):
         """Run one cycle of watching the given projects until stop_event is set."""
         project_paths = [project.path for project in projects]
+        previous_filter_roots = self._sorted_watch_filter_roots
+        self._sorted_watch_filter_roots = tuple(
+            sorted(
+                (Path(project.path).expanduser().resolve() for project in projects),
+                # Trigger: configured project roots can overlap.
+                # Why: an enclosing project's hidden directory should still hide descendants.
+                # Outcome: choose the outermost matching root when checking hidden path parts.
+                key=lambda project_path: len(project_path.parts),
+            )
+        )
 
-        async for changes in awatch(
-            *project_paths,
-            debounce=self.app_config.sync_delay,
-            watch_filter=self.filter_changes,
-            recursive=True,
-            stop_event=stop_event,
-        ):
-            # group changes by project and filter using ignore patterns
-            project_changes = defaultdict(list)
-            for change, path in changes:
-                for project in projects:
-                    if self.is_project_path(project, path):
-                        # Check if the file should be ignored based on gitignore patterns
-                        project_path = Path(project.path)
-                        file_path = Path(path)
-                        ignore_patterns = self._get_ignore_patterns(project_path)
-
-                        if should_ignore_path(file_path, project_path, ignore_patterns):
-                            logger.trace(
-                                f"Ignoring watched file change: {file_path.relative_to(project_path)}"
-                            )
-                            continue
-
-                        project_changes[project].append((change, path))
-                        break
+        try:
+            async for changes in awatch(
+                *project_paths,
+                debounce=self.app_config.sync_delay,
+                watch_filter=self.filter_changes,
+                recursive=True,
+                stop_event=stop_event,
+            ):
+                # group changes by project and filter using ignore patterns
+                project_changes = defaultdict(list)
+                for change, path in changes:
+                    for project in projects:
+                        if self.is_project_path(project, path):
+                            # Check if the file should be ignored based on gitignore patterns
+                            project_path = Path(project.path)
+                            file_path = Path(path)
+                            ignore_patterns = self._get_ignore_patterns(project_path)
+
+                            if should_ignore_path(file_path, project_path, ignore_patterns):
+                                logger.trace(
+                                    f"Ignoring watched file change: {file_path.relative_to(project_path)}"
+                                )
+                                continue
+
+                            project_changes[project].append((change, path))
+                            break
 
-            # create coroutines to handle changes
-            change_handlers = [
-                self.handle_changes(project, set(changes))
-                for project, changes in project_changes.items()
-            ]
+                # create coroutines to handle changes
+                change_handlers = [
+                    self.handle_changes(project, set(changes))
+                    for project, changes in project_changes.items()
+                ]
 
-            # process changes
-            await asyncio.gather(*change_handlers)
+                # process changes
+                await asyncio.gather(*change_handlers)
+        finally:
+            self._sorted_watch_filter_roots = previous_filter_roots
 
     async def _select_projects_to_watch(self) -> list[Project]:
         """Return the set of projects this watch cycle should observe.
@@ -267,15 +281,43 @@ async def run(self):  # pragma: no cover
             self.state.running = False
             await self.write_status()
 
-    def filter_changes(self, change: Change, path: str) -> bool:  # pragma: no cover
+    def filter_changes(self, change: Change, path: str) -> bool:
         """Filter to only watch non-hidden files and directories.
 
         Returns:
             True if the file should be watched, False if it should be ignored
         """
 
-        # Skip hidden directories and files
-        path_parts = Path(path).parts
+        path_obj = Path(path).expanduser().resolve()
+
+        project_paths = self._sorted_watch_filter_roots
+        if project_paths is None:
+            project_paths = tuple(
+                sorted(
+                    (
+                        Path(entry.path).expanduser().resolve()
+                        for entry in self.app_config.projects.values()
+                        if entry.path
+                    ),
+                    # Trigger: direct callers may not run inside a watch cycle.
+                    # Why: tests and one-off calls still need the same hidden-path semantics.
+                    # Outcome: compute the stable outermost-first order only for fallback calls.
+                    key=lambda project_path: len(project_path.parts),
+                )
+            )
+
+        relative_path = None
+        for project_path in project_paths:
+            try:
+                relative_path = path_obj.relative_to(project_path)
+                break
+            except ValueError:
+                continue
+
+        # Trigger: a project may live under a hidden parent such as ~/.claude.
+        # Why: only dotfiles and dot-directories inside the watched project should be ignored.
+        # Outcome: hidden parents outside the project root do not mute legitimate project changes.
+        path_parts = relative_path.parts if relative_path is not None else path_obj.parts
         for part in path_parts:
             if part.startswith("."):
                 return False

test-int/mcp/test_string_params_integration.py

@@ -28,6 +28,7 @@ async def test_search_notes_entity_types_as_string(mcp_server, app, test_project
             {
                 "project": test_project.name,
                 "query": "coercion",
+                "search_type": "text",
                 "entity_types": '["entity"]',
             },
         )
@@ -54,6 +55,7 @@ async def test_search_notes_note_types_as_string(mcp_server, app, test_project):
             {
                 "project": test_project.name,
                 "query": "coercion",
+                "search_type": "text",
                 "note_types": '["note"]',
             },
         )
@@ -81,6 +83,7 @@ async def test_search_notes_tags_as_string(mcp_server, app, test_project):
             {
                 "project": test_project.name,
                 "query": "tagged",
+                "search_type": "text",
                 "tags": '["alpha"]',
             },
         )
@@ -107,6 +110,7 @@ async def test_search_notes_metadata_filters_as_string(mcp_server, app, test_pro
             {
                 "project": test_project.name,
                 "query": "metadata",
+                "search_type": "text",
                 "metadata_filters": '{"type": "note"}',
             },
         )

tests/sync/test_watch_service_edge_cases.py

@@ -1,8 +1,12 @@
 """Test edge cases in the WatchService."""
 
+import builtins
+
 import pytest
 from watchfiles import Change
 
+from basic_memory.config import ProjectEntry
+
 
 def test_filter_changes_valid_path(watch_service, project_config):
     """Test the filter_changes method with valid non-hidden paths."""
@@ -21,6 +25,66 @@ def test_filter_changes_valid_path(watch_service, project_config):
     )
 
 
+def test_filter_changes_allows_project_under_hidden_parent(watch_service, tmp_path):
+    """Hidden parent directories outside the project root must not mute the watcher."""
+    project_home = tmp_path / ".claude" / "projects" / "memory"
+    project_home.mkdir(parents=True)
+    watch_service.app_config.projects["hidden-parent"] = ProjectEntry(path=str(project_home))
+    watch_service._sorted_watch_filter_roots = (project_home.resolve(),)
+
+    visible_note = project_home / "notes" / "visible.md"
+    hidden_note = project_home / "notes" / ".drafts" / "hidden.md"
+
+    assert watch_service.filter_changes(Change.added, str(visible_note)) is True
+    assert watch_service.filter_changes(Change.added, str(hidden_note)) is False
+
+
+def test_filter_changes_rejects_nested_project_inside_hidden_directory(watch_service, tmp_path):
+    """A nested project must not make its enclosing project's hidden path visible."""
+    outer_project = tmp_path / "outer"
+    nested_project = outer_project / ".private" / "subproject"
+    nested_project.mkdir(parents=True)
+    watch_service.app_config.projects["outer"] = ProjectEntry(path=str(outer_project))
+    watch_service.app_config.projects["nested"] = ProjectEntry(path=str(nested_project))
+    watch_service._sorted_watch_filter_roots = (
+        outer_project.resolve(),
+        nested_project.resolve(),
+    )
+
+    nested_note = nested_project / "notes" / "visible.md"
+
+    assert watch_service.filter_changes(Change.added, str(nested_note)) is False
+
+
+def test_filter_changes_uses_cached_sorted_roots_without_resorting(
+    monkeypatch,
+    watch_service,
+    tmp_path,
+):
+    """The watch callback hot path should not sort roots after the cycle cached them."""
+    project_home = tmp_path / "project"
+    project_home.mkdir()
+    watch_service._sorted_watch_filter_roots = (project_home.resolve(),)
+
+    def fail_if_sorted(*args, **kwargs):
+        raise AssertionError("cached watch roots should already be sorted")
+
+    monkeypatch.setattr(builtins, "sorted", fail_if_sorted)
+
+    assert watch_service.filter_changes(Change.added, str(project_home / "note.md")) is True
+
+
+def test_filter_changes_path_outside_all_projects(watch_service, tmp_path):
+    """Unmatched paths should still use full-path hidden filtering as a fallback."""
+    watch_service._sorted_watch_filter_roots = ()
+
+    unrelated = tmp_path / "unrelated" / "file.md"
+    hidden_unrelated = tmp_path / ".hidden" / "file.md"
+
+    assert watch_service.filter_changes(Change.added, str(unrelated)) is True
+    assert watch_service.filter_changes(Change.added, str(hidden_unrelated)) is False
+
+
 def test_filter_changes_hidden_path(watch_service, project_config):
     """Test the filter_changes method with hidden files/directories."""
     # Hidden file (starts with dot)