fix(config): read the KB metadata database path from config instead of hardcoding it

CHANGELOG.md

@@ -18,6 +18,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   constant score of 0.5 while the API reported `success: true`.
 - `add_documents` rejects a provider that returns fewer vectors than texts, which otherwise
   assigned each embedding to the wrong chunk. `screening.py` gained the same guard.
+- The KB-metadata SQLite path now honours `database.path` (and `PERSPICACITE_DB_PATH`) in the
+  web app, the MCP server and the CLI. It was hardcoded to a working-directory-relative
+  `./data/perspicacite.db` while the vector store already honoured `database.chroma_path`, so a
+  second instance listed the main hub's knowledge bases and wrote its vectors elsewhere. When
+  `database.path` is not set, the legacy location is kept, so existing deployments are unaffected.
+  Both resolved paths are now logged at startup.
 
 ### Added
 - `GET /api/kb/{name}/stats` reports an `embedding_health` block (`probed_chunks`,

src/perspicacite/cli.py

@@ -10,6 +10,7 @@
 
 from perspicacite import __version__
 from perspicacite.config import load_config
+from perspicacite.config.paths import resolve_session_db_path
 from perspicacite.logging import get_logger, setup_logging
 from perspicacite.pipeline.asb.run_ingest import ingest_asb_run as ingest_asb_run_pipeline
 from perspicacite.pipeline.github.bundle import ContentSpec
@@ -177,7 +178,7 @@ def serve(
     "--session-db",
     type=click.Path(path_type=Path),
     default=None,
-    help="SQLite DB for KB metadata (default: data/perspicacite.db, same as web app)",
+    help="SQLite DB for KB metadata (default: config database.path, else data/perspicacite.db)",
 )
 @click.option(
     "--chroma-dir",
@@ -259,7 +260,7 @@ async def _create_empty() -> dict[str, Any]:
         )
         return
 
-    session_db = session_db or Path("data/perspicacite.db")
+    session_db = session_db or resolve_session_db_path(config)
     chroma_dir = chroma_dir or config.database.chroma_path
 
     click.echo(f"Creating knowledge base '{name}' from {from_bibtex}...")
@@ -313,7 +314,7 @@ async def _create_empty() -> dict[str, Any]:
     "--session-db",
     type=click.Path(path_type=Path),
     default=None,
-    help="SQLite DB for KB metadata (default: data/perspicacite.db)",
+    help="SQLite DB for KB metadata (default: config database.path, else data/perspicacite.db)",
 )
 @click.option(
     "--chroma-dir",
@@ -345,7 +346,7 @@ def add_to_kb(
     if ingest_mode is not None:
         config.knowledge_base.ingest_mode = ingest_mode
 
-    session_db = session_db or Path("data/perspicacite.db")
+    session_db = session_db or resolve_session_db_path(config)
     chroma_dir = chroma_dir or config.database.chroma_path
 
     click.echo(f"Adding papers from {from_bibtex} to knowledge base '{name}'...")

src/perspicacite/config/paths.py

@@ -0,0 +1,29 @@
+"""Filesystem paths derived from configuration.
+
+The vector store has always honoured ``database.chroma_path`` while the KB
+metadata SQLite was hardcoded to a working-directory-relative path. A second
+instance therefore read the main hub's KB listing and wrote its vectors
+elsewhere, producing knowledge bases that exist in the listing but return
+nothing. These helpers give every entry point (web, MCP, CLI) one answer.
+"""
+
+from pathlib import Path
+
+from perspicacite.config.schema import Config
+
+# Where the web app, MCP server and CLI historically kept KB metadata, relative
+# to the process working directory.
+LEGACY_SESSION_DB_PATH = Path("./data/perspicacite.db")
+
+
+def resolve_session_db_path(config: Config) -> Path:
+    """Return the KB-metadata SQLite path for this configuration.
+
+    ``database.path`` is honoured only when it was actually supplied, by config
+    file or by ``PERSPICACITE_DB_PATH``. Applying the schema default instead
+    would silently relocate the metadata of every existing deployment, whose
+    knowledge bases live in the legacy working-directory path.
+    """
+    if "path" in config.database.model_fields_set:
+        return config.database.path.expanduser()
+    return LEGACY_SESSION_DB_PATH

src/perspicacite/mcp/server.py

@@ -33,6 +33,7 @@
 from pathlib import Path
 from typing import Any
 
+from perspicacite.config.paths import resolve_session_db_path
 from perspicacite.logging import get_logger
 from perspicacite.pipeline.asb.response import build_asb_response_metadata
 from perspicacite.pipeline.asb.run_ingest import ingest_asb_run as ingest_asb_run_pipeline
@@ -95,8 +96,6 @@ async def initialize(self, config: Any) -> None:
         if self.initialized:
             return
 
-        from pathlib import Path
-
         from perspicacite.llm import AsyncLLMClient
         from perspicacite.llm.embeddings import create_embedding_provider
         from perspicacite.memory.session_store import SessionStore
@@ -125,15 +124,20 @@ async def initialize(self, config: Any) -> None:
         )
 
         # Session store
-        db_path = Path("./data/perspicacite.db")
+        db_path = resolve_session_db_path(config)
         db_path.parent.mkdir(parents=True, exist_ok=True)
         self.session_store = SessionStore(db_path)
         await self.session_store.init_db()
+        logger.info(
+            "mcp_stores_initialized",
+            session_db=str(db_path),
+            chroma_path=str(config.database.chroma_path),
+        )
 
         # Provenance store (shares the same DB as the session store)
         from perspicacite.provenance.store import ProvenanceStore
 
-        sidecar_dir = Path("./data/provenance")
+        sidecar_dir = db_path.parent / "provenance"
         sidecar_dir.mkdir(parents=True, exist_ok=True)
         self.provenance_store = ProvenanceStore(db_path=db_path, sidecar_dir=sidecar_dir)
 

src/perspicacite/web/state.py

@@ -9,9 +9,9 @@
 
 import logging
 import os
-from pathlib import Path
 from typing import Any
 
+from perspicacite.config.paths import resolve_session_db_path
 from perspicacite.jobs.registry import JobRegistry
 from perspicacite.memory.session_store import SessionStore
 from perspicacite.provenance.store import ProvenanceStore
@@ -192,11 +192,17 @@ async def initialize(self):
         logger.info("Agentic orchestrator initialized")
 
         # Initialize session store FIRST so RAGEngine can receive it
-        db_path = Path("./data/perspicacite.db")
+        db_path = resolve_session_db_path(config)
         db_path.parent.mkdir(parents=True, exist_ok=True)
         self.session_store = SessionStore(db_path)
         await self.session_store.init_db()
-        logger.info("Session store initialized")
+        # Log both stores: an instance whose metadata and vectors live in
+        # different places lists knowledge bases that return nothing.
+        logger.info(
+            "Session store initialized at %s (chroma: %s)",
+            db_path,
+            config.database.chroma_path,
+        )
 
         # Initialize RAG engine for multi-mode support
         from perspicacite.rag.engine import RAGEngine

tests/unit/test_config_database_path.py

@@ -0,0 +1,70 @@
+"""Regression tests: the KB-metadata SQLite must honour `database.path`.
+
+The vector store has always read `database.chroma_path`, while the session
+store was hardcoded to a working-directory-relative `./data/perspicacite.db`.
+A second instance therefore listed the main hub's knowledge bases while writing
+its vectors elsewhere, so every KB it created "existed" but retrieved nothing.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from perspicacite.config.paths import LEGACY_SESSION_DB_PATH, resolve_session_db_path
+from perspicacite.config.schema import Config
+
+
+def test_unset_path_keeps_the_legacy_location():
+    """Existing deployments keep their metadata where it already lives."""
+    assert resolve_session_db_path(Config()) == LEGACY_SESSION_DB_PATH
+
+
+def test_setting_only_chroma_path_keeps_the_legacy_location():
+    """Touching the vector-store path must not move the metadata store."""
+    config = Config(database={"chroma_path": "/tmp/isolated-chroma"})
+    assert resolve_session_db_path(config) == LEGACY_SESSION_DB_PATH
+
+
+def test_explicit_path_is_honoured_and_home_is_expanded():
+    """`~` must be expanded; SessionStore would otherwise create a `~` directory."""
+    resolved = resolve_session_db_path(Config(database={"path": "~/isolated/memory.db"}))
+    assert resolved == Path.home() / "isolated" / "memory.db"
+    assert "~" not in str(resolved)
+
+
+def test_env_var_override_reaches_the_resolver(monkeypatch, tmp_path):
+    """PERSPICACITE_DB_PATH is documented as the isolation switch."""
+    from perspicacite.config.loader import load_config
+
+    target = tmp_path / "instance" / "memory.db"
+    monkeypatch.setenv("PERSPICACITE_DB_PATH", str(target))
+    assert resolve_session_db_path(load_config(None)) == target
+
+
+@pytest.mark.asyncio
+async def test_app_state_opens_the_configured_database(monkeypatch, tmp_path):
+    """AppState.initialize must hand the configured path to SessionStore."""
+    from perspicacite.web import state as state_module
+
+    target = tmp_path / "instance" / "memory.db"
+    monkeypatch.setenv("PERSPICACITE_DB_PATH", str(target))
+    monkeypatch.setenv("PERSPICACITE_ALLOW_MISSING_LLM_KEYS", "1")
+    monkeypatch.setenv("PERSPICACITE_DB_CHROMA_PATH", str(tmp_path / "chroma"))
+
+    opened: list[Path] = []
+
+    class _RecordingSessionStore:
+        def __init__(self, db_path):
+            opened.append(Path(db_path))
+            self.db_path = Path(db_path)
+
+        async def init_db(self):
+            return None
+
+    monkeypatch.setattr(state_module, "SessionStore", _RecordingSessionStore)
+
+    app_state = state_module.AppState()
+    await app_state.initialize()
+
+    assert opened == [target]
+    assert opened[0] != LEGACY_SESSION_DB_PATH