feat: add local daemon git layers

Author: rudironsoni

AGENTS.md

@@ -0,0 +1,48 @@
+This is built on top of [CocoIndex v1](https://cocoindex.io/docs-v1/llms.txt).
+
+
+## Build and Test Commands
+
+This project uses [uv](https://docs.astral.sh/uv/) for project management.
+
+```bash
+uv run mypy .           # Type check Python code
+uv run pytest tests/    # Run Python tests
+```
+
+## Code Conventions
+
+### Internal vs External Modules
+
+We distinguish between **internal modules** (under packages with `_` prefix, e.g. `_internal.*` or `connectors.*._source`) and **external modules** (which users can directly import).
+
+**External modules** (user-facing, e.g. `cocoindex/ops/sentence_transformers.py`):
+
+* Be strict about not leaking implementation details
+* Use `__all__` to explicitly list public exports
+* Prefix ALL non-public symbols with `_`, including:
+  * Standard library imports: `import threading as _threading`, `import typing as _typing`
+  * Third-party imports: `import numpy as _np`, `from numpy.typing import NDArray as _NDArray`
+  * Internal package imports: `from cocoindex.resources import schema as _schema`
+* Exception: `TYPE_CHECKING` imports for type hints don't need prefixing
+
+**Internal modules** (e.g. `cocoindex/_internal/component_ctx.py`):
+
+* Less strict since users shouldn't import these directly
+* Standard library and internal imports don't need underscore prefix
+* Only prefix symbols that are truly private to the module itself (e.g. `_context_var` for a module-private ContextVar)
+
+### General principles (also covered by `/review-changes`)
+
+- **Top-level imports.** Defer to in-function only for a real circular dependency or a heavy import that isn't always needed.
+- **Specific types over `Any`.** When a value enters as a weaker form (`str`, `Any`), convert to the strong type at the earliest point. Don't propagate the weak form.
+- **`NamedTuple`/small dataclass for multi-value returns.** Access fields by name at call sites.
+- **Single source of truth.** When the same value or logic appears in multiple places, consolidate it.
+- **Delete dead code and dead config.** When a change makes something unreachable, the code, the tests, and the knobs all go.
+- **Honest names.** The name describes what the code does today.
+
+### Testing Guidelines
+
+We prefer end-to-end tests on user-facing APIs, over unit tests on smaller internal functions. With this said, there're cases where unit tests are necessary, e.g. for internal logic with various situations and edge cases, in which case it's usually easier to cover various scenarios with unit tests.
+
+When tests fail, fix the underlying issue. Don't skip, ignore, or exclude to get a green result.

src/cocoindex_code/_daemon_paths.py

@@ -31,6 +31,23 @@ def daemon_runtime_dir() -> Path:
     return user_settings_dir()
 
 
+def daemon_state_dir() -> Path:
+    """Return the durable daemon-owned state directory.
+
+    This is separate from both project checkout state and daemon runtime files:
+    it stores shared layer metadata, materialized layer sources, and layer
+    databases.  ``COCOINDEX_CODE_STATE_DIR`` exists mostly for tests and
+    advanced users; otherwise we follow XDG data-home on Unix-like systems.
+    """
+    override = os.environ.get("COCOINDEX_CODE_STATE_DIR")
+    if override:
+        return Path(override)
+    xdg_data_home = os.environ.get("XDG_DATA_HOME")
+    if xdg_data_home:
+        return Path(xdg_data_home) / "cocoindex-code"
+    return Path.home() / ".local" / "share" / "cocoindex-code"
+
+
 def connection_family() -> str:
     """Return the multiprocessing connection family for this platform."""
     return "AF_PIPE" if sys.platform == "win32" else "AF_UNIX"

src/cocoindex_code/cli.py

@@ -1,4 +1,5 @@
 """CLI entry point for cocoindex-code (ccc command)."""
+# mypy: disable-error-code=untyped-decorator
 
 from __future__ import annotations
 
@@ -44,6 +45,8 @@
 
 daemon_app = _typer.Typer(name="daemon", help="Manage the daemon process.")
 app.add_typer(daemon_app, name="daemon")
+overlay_app = _typer.Typer(name="overlay", help="Inspect and prune Git index overlays.")
+app.add_typer(overlay_app, name="overlay")
 
 
 @app.callback()
@@ -99,6 +102,28 @@ def require_project_root() -> Path:
     return root
 
 
+def require_project_root_from(cwd: Path | None) -> Path:
+    """Find the initialized project root for *cwd* or the process CWD."""
+    if cwd is None:
+        return require_project_root()
+    gs_path = user_settings_path()
+    if not gs_path.is_file():
+        _typer.echo(
+            f"Error: Global settings not found: {format_path_for_display(gs_path)}\n"
+            "Run `ccc init` to create it with default settings.",
+            err=True,
+        )
+        raise _typer.Exit(code=1)
+    root = find_project_root(cwd)
+    if root is None:
+        _typer.echo(
+            f"Error: Not in an initialized project directory: {format_path_for_display(cwd)}",
+            err=True,
+        )
+        raise _typer.Exit(code=1)
+    return root
+
+
 _F = TypeVar("_F", bound=Callable[..., object])
 
 
@@ -181,7 +206,12 @@ def print_search_results(response: SearchResponse) -> None:
         _typer.echo(r.content)
 
 
-def _run_index_with_progress(project_root: str) -> None:
+def _run_index_with_progress(
+    project_root: str,
+    *,
+    cwd: str | None = None,
+    base_ref: str | None = None,
+) -> None:
     """Run indexing with streaming progress display. Exits on failure."""
     from rich.console import Console as _Console
     from rich.live import Live as _Live
@@ -208,7 +238,13 @@ def _on_progress(progress: IndexingProgress) -> None:
             live.update(_Spinner("dots", last_progress_line))
 
         try:
-            resp = _client.index(project_root, on_progress=_on_progress, on_waiting=_on_waiting)
+            resp = _client.index(
+                project_root,
+                cwd=cwd,
+                base_ref=base_ref,
+                on_progress=_on_progress,
+                on_waiting=_on_waiting,
+            )
         except RuntimeError as e:
             live.stop()
             # Let DaemonStartError propagate to the decorator for consistent handling.
@@ -229,6 +265,8 @@ def _on_progress(progress: IndexingProgress) -> None:
 def _search_with_wait_spinner(
     project_root: str,
     query: str,
+    cwd: str | None = None,
+    base_ref: str | None = None,
     languages: list[str] | None = None,
     paths: list[str] | None = None,
     limit: int = 10,
@@ -254,6 +292,8 @@ def _on_waiting() -> None:
         resp = _client.search(
             project_root=project_root,
             query=query,
+            cwd=cwd,
+            base_ref=base_ref,
             languages=languages,
             paths=paths,
             limit=limit,
@@ -533,13 +573,18 @@ def init(
 
 @app.command()
 @_catch_daemon_start_error
-def index() -> None:
+def index(
+    cwd: Path | None = _typer.Option(None, "--cwd", help="Workspace path to index"),
+    base_ref: str | None = _typer.Option(None, "--base", help="Git base ref"),
+) -> None:
     """Create/update index for the codebase."""
     from . import client as _client
 
-    project_root = str(require_project_root())
+    project_root_path = require_project_root_from(cwd.resolve() if cwd is not None else None)
+    project_root = str(project_root_path)
+    request_cwd = str(cwd.resolve()) if cwd is not None else None
     print_project_header(project_root)
-    _run_index_with_progress(project_root)
+    _run_index_with_progress(project_root, cwd=request_cwd, base_ref=base_ref)
     print_index_stats(_client.project_status(project_root))
 
 
@@ -552,13 +597,17 @@ def search(
     offset: int = _typer.Option(0, "--offset", help="Number of results to skip"),
     limit: int = _typer.Option(10, "--limit", help="Maximum results to return"),
     refresh: bool = _typer.Option(False, "--refresh", help="Refresh index before searching"),
+    cwd: Path | None = _typer.Option(None, "--cwd", help="Workspace path to search"),
+    base_ref: str | None = _typer.Option(None, "--base", help="Git base ref"),
 ) -> None:
     """Semantic search across the codebase."""
-    project_root = str(require_project_root())
+    project_root_path = require_project_root_from(cwd.resolve() if cwd is not None else None)
+    project_root = str(project_root_path)
+    request_cwd = str(cwd.resolve()) if cwd is not None else None
     query_str = " ".join(query)
 
     if refresh:
-        _run_index_with_progress(project_root)
+        _run_index_with_progress(project_root, cwd=request_cwd, base_ref=base_ref)
 
     # Default path filter from CWD
     paths: list[str] | None = None
@@ -572,6 +621,8 @@ def search(
     resp = _search_with_wait_spinner(
         project_root=project_root,
         query=query_str,
+        cwd=request_cwd,
+        base_ref=base_ref,
         languages=lang or None,
         paths=paths,
         limit=limit,
@@ -825,7 +876,7 @@ def mcp() -> None:
     async def _run_mcp() -> None:
         from .server import create_mcp_server
 
-        mcp_server = create_mcp_server(project_root)
+        mcp_server = create_mcp_server(project_root, cwd=str(Path.cwd().resolve()))
         asyncio.create_task(_bg_index(project_root))
         await mcp_server.run_stdio_async()
 
@@ -845,6 +896,47 @@ async def _bg_index(project_root: str) -> None:
         pass
 
 
+# --- Overlay subcommands ---
+
+
+@overlay_app.command("status")
+@_catch_daemon_start_error
+def overlay_status(
+    cwd: Path | None = _typer.Option(None, "--cwd", help="Workspace path to inspect"),
+    base_ref: str | None = _typer.Option(None, "--base", help="Git base ref"),
+) -> None:
+    """Show daemon layer metadata for the current Git repo."""
+    from . import client as _client
+
+    project_root_path = require_project_root_from(cwd.resolve() if cwd is not None else None)
+    request_cwd = str(cwd.resolve()) if cwd is not None else None
+    resp = _client.overlay_status(str(project_root_path), cwd=request_cwd, base_ref=base_ref)
+    if resp.repo_id is not None:
+        _typer.echo(f"Repo: {resp.repo_id}")
+    if not resp.layers:
+        _typer.echo("No layers.")
+        return
+    for layer in resp.layers:
+        _typer.echo(
+            f"{layer.kind:6} {layer.status:8} {layer.layer_id} "
+            f"ref={layer.ref_name or '-'} affected={layer.affected_count} "
+            f"tombstones={layer.tombstoned_count}"
+        )
+
+
+@overlay_app.command("prune")
+@_catch_daemon_start_error
+def overlay_prune() -> None:
+    """Prune expired dirty and branch overlays."""
+    from . import client as _client
+
+    resp = _client.overlay_prune()
+    if not resp.pruned_layer_ids:
+        _typer.echo("No expired layers pruned.")
+        return
+    _typer.echo(f"Pruned {len(resp.pruned_layer_ids)} layer(s).")
+
+
 # --- Daemon subcommands ---
 
 

src/cocoindex_code/client.py

@@ -40,6 +40,10 @@
     IndexRequest,
     IndexResponse,
     IndexWaitingNotice,
+    OverlayPruneRequest,
+    OverlayPruneResponse,
+    OverlayStatusRequest,
+    OverlayStatusResponse,
     ProjectStatusRequest,
     ProjectStatusResponse,
     RemoveProjectRequest,
@@ -242,14 +246,20 @@ def _send(req: Request) -> Response:
 
 def index(
     project_root: str,
+    cwd: str | None = None,
+    base_ref: str | None = None,
     on_progress: Callable[[IndexingProgress], None] | None = None,
     on_waiting: Callable[[], None] | None = None,
 ) -> IndexResponse:
     """Request indexing with streaming progress. Blocks until complete."""
     project_root = normalize_input_path(project_root)
+    if cwd is not None:
+        cwd = normalize_input_path(cwd)
     conn = _connect_and_handshake()
     try:
-        conn.send_bytes(encode_request(IndexRequest(project_root=project_root)))
+        conn.send_bytes(
+            encode_request(IndexRequest(project_root=project_root, cwd=cwd, base_ref=base_ref))
+        )
         while True:
             try:
                 data = conn.recv_bytes()
@@ -276,6 +286,8 @@ def index(
 def search(
     project_root: str,
     query: str,
+    cwd: str | None = None,
+    base_ref: str | None = None,
     languages: list[str] | None = None,
     paths: list[str] | None = None,
     limit: int = 5,
@@ -289,13 +301,17 @@ def search(
     until the final ``SearchResponse``.
     """
     project_root = normalize_input_path(project_root)
+    if cwd is not None:
+        cwd = normalize_input_path(cwd)
     conn = _connect_and_handshake()
     try:
         conn.send_bytes(
             encode_request(
                 SearchRequest(
                     project_root=project_root,
                     query=query,
+                    cwd=cwd,
+                    base_ref=base_ref,
                     languages=languages,
                     paths=paths,
                     limit=limit,
@@ -345,6 +361,21 @@ def daemon_env() -> DaemonEnvResponse:
     return _send(DaemonEnvRequest())  # type: ignore[return-value]
 
 
+def overlay_status(
+    project_root: str,
+    cwd: str | None = None,
+    base_ref: str | None = None,
+) -> OverlayStatusResponse:
+    project_root = normalize_input_path(project_root)
+    if cwd is not None:
+        cwd = normalize_input_path(cwd)
+    return _send(OverlayStatusRequest(project_root=project_root, cwd=cwd, base_ref=base_ref))  # type: ignore[return-value]
+
+
+def overlay_prune() -> OverlayPruneResponse:
+    return _send(OverlayPruneRequest())  # type: ignore[return-value]
+
+
 def doctor(
     project_root: str | None = None,
     on_result: Callable[[DoctorCheckResult], None] | None = None,