Chore/harden perspicacite

.claude/commands/setup.md

@@ -14,4 +14,4 @@ provider API keys (chat + embedding), Unpaywall email config, and whether a
 server is already running on :8000. Exits 0 if all critical checks pass.
 
 If something fails, the script names the exact next command to run.
-For the full step-by-step install walkthrough see [`INSTALL_AGENT.md`](../../INSTALL_AGENT.md).
+For the full step-by-step install walkthrough see [`INSTALL_AGENT.md`](../../docs/INSTALL_AGENT.md).

.github/dependabot.yml

@@ -0,0 +1,27 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    groups:
+      python-dependencies:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    groups:
+      npm-dependencies:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5

.github/workflows/ci.yml

@@ -17,19 +17,27 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
       - name: Install ruff
-        run: pip install ruff
-      - name: Ruff (report-only, --exit-zero)
-        run: ruff check src/ tests/ --exit-zero --statistics
+        run: pip install ruff==0.15.13
+      - name: Ruff (enforced — fails on any finding)
+        run: ruff check src/ tests/ --output-format=github
+      - name: Install package + dev deps (for mypy)
+        run: pip install -e ".[dev]"
+      - name: Mypy (enforced — fails on any finding)
+        run: mypy src/
 
   test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
       - name: Install package + dev deps
         run: pip install -e ".[dev]"
 
@@ -56,6 +64,7 @@ jobs:
             --deselect tests/unit/test_provenance_engine_wiring.py::test_mcp_generate_report_wires_provenance_and_message_id \
             --deselect tests/unit/test_zotero_ingest_worker.py::test_worker_dedups_by_doi_and_attaches_notes \
             --deselect tests/unit/test_zotero_ingest_worker.py::test_worker_skips_existing_doi \
+            --deselect tests/unit/test_domain_aggregator.py::test_retry_counts_as_one_circuit_failure \
             --timeout=15 --timeout-method=signal \
             --no-header --tb=line \
             --cov=src/perspicacite --cov-report=xml --cov-report=term-missing \
@@ -68,3 +77,26 @@ jobs:
           name: coverage-report
           path: coverage.xml
           retention-days: 7
+
+  frontend:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+      - name: Install dependencies
+        run: npm ci
+      - name: Lint
+        run: npm run lint
+      - name: Typecheck
+        run: npm run typecheck
+      - name: Unit tests
+        run: npm test
+      - name: Build
+        run: npm run build

.gitignore

@@ -95,12 +95,20 @@ packages_to_use/
 # Ruff cache
 .ruff_cache/
 
+# Tooling caches & coverage data
+.mypy_cache/
+.pytest_cache/
+.coverage
+.coverage.*
+
+# Local claim-graph stores (pyoxigraph/RocksDB; per-machine runtime state)
+data/claim_graphs/
+
 # Documentation files (keep only main docs)
 *.md
 !README.md
 !CHANGELOG.md
 !CONTRIBUTING.md
-!INSTALL_AGENT.md
 !.claude/commands/*.md
 !.claude/skills/**/*.md
 !LICENSE
@@ -135,7 +143,6 @@ packages_to_use/
 !docs/recipe-book-*.md
 !docs/architecture-*.md
 !docs/upstream-prompts/**/*.md
-!MANUAL_QA.md
 # GitHub community health files
 !.github/*.md
 !.github/ISSUE_TEMPLATE/*.md

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+# Pre-commit hooks for Perspicacité.
+#
+# Install once with:  uv run pre-commit install
+# Run on everything:  uv run pre-commit run --all-files
+#
+# ruff (lint) is pinned to the SAME version the CI lint gate enforces
+# (.github/workflows/ci.yml) so local and CI never disagree. ruff-format
+# runs only on the files in your commit — it does not reformat the whole
+# tree, which keeps the diff small (see the note in pyproject.toml's
+# [tool.ruff.lint] ignore block).
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.13
+    hooks:
+      - id: ruff
+        name: ruff (lint, autofix)
+        args: [--fix]
+      - id: ruff-format
+        name: ruff (format touched files)

CLAUDE.md

@@ -15,7 +15,7 @@ uv sync --dev           # install all dependencies including dev extras
 cp .env.example .env    # then add at least one LLM API key
 ```
 
-Configuration lives in `config.yml` (git-ignored; copy from `config.example.yml`). The app reads it at startup via `perspicacite.config.loader.load_config()`.
+Configuration lives in `config.yml` (git-ignored; copy from `config/example.yml`). The app reads it at startup via `perspicacite.config.loader.load_config()`.
 
 ## Commands
 
@@ -110,7 +110,7 @@ FastAPI app is defined in [src/perspicacite/web/app.py](src/perspicacite/web/app
 - `GET /api/conversations/search?q=...` — full-text search across all conversations; uses SQLite FTS5 over message content with a LIKE fallback. `SessionStore.init_db()` creates the FTS5 shadow table idempotently; `add_message()` keeps it in sync.
 - `GET /api/conversations/{id}/export?format=markdown` — downloadable Markdown rendering of a conversation (Q&A turns + cited sources/references footer).
 - New static JS files: `static/js/kb_stats.js` (KB stats tab) and `static/js/paper_detail.js` (paper-detail slide-over panel + pipeline-step badges on chat source cards).
-- `MANUAL_QA.md` (git-tracked, repo root) — human click-through checklist for Phase 5 features.
+- `docs/development/MANUAL_QA.md` (git-tracked) — human click-through checklist for Phase 5 features.
 
 ### MCP Server
 

README.md

@@ -61,7 +61,7 @@ uv sync
 ### Configure
 
 ```bash
-cp config.example.yml config.yml
+cp config/example.yml config.yml
 cp .env.example .env
 # Edit .env — add at least one LLM API key
 ```
@@ -254,7 +254,7 @@ Full flags and usage: [docs/reference/cli.md](docs/reference/cli.md).
 
 ## Configuration
 
-Copy `config.example.yml` and edit. Key sections:
+Copy `config/example.yml` and edit. Key sections:
 
 ```yaml
 llm:
@@ -274,9 +274,9 @@ mcp:
 
 Full schema: [docs/reference/config.md](docs/reference/config.md).
 
-Alternative config presets: `config.ollama.example.yml` (local Ollama),
-`config.claude_code.example.yml` (Claude Code CLI), `config.codex.example.yml`,
-`config.hermes.example.yml`, `config.openclaw.example.yml`.
+Alternative config presets: `config/providers/ollama.yml` (local Ollama),
+`config/providers/claude-code.yml` (Claude Code CLI), `config/providers/codex.yml`,
+`config/providers/hermes.yml`, `config/providers/openclaw.yml`.
 
 ---
 
@@ -305,15 +305,20 @@ Apache License 2.0 — see [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE).
 
 <p align="center">
 <b>Citation:</b> <em>An AI Pipeline for Scientific Literacy and Discovery: a Demonstration of Perspicacité-AI Integration with Knowledge Graphs</em><br>
-L. Pradi, T. Jiang, M. Feraud, M. Bekbergenova, Y. Taghzouti, L.-F. Nothias — <em>ISWC-C 2025</em>
+L. Pradi, T. Jiang, M. Feraud, M. Bekbergenova, Y. Taghzouti, L.-F. Nothias — <em>ISWC-C 2025</em><br>
+<a href="https://ceur-ws.org/Vol-4085/paper77.pdf">CEUR-WS PDF</a> · <a href="https://hal.science/hal-05233151v1">HAL archive</a>
 </p>
 
 ```bibtex
 @inproceedings{pradi2025perspicacite,
   title     = {An AI Pipeline for Scientific Literacy and Discovery: a Demonstration of Perspicacit\'{e}-AI Integration with Knowledge Graphs},
   author    = {Pradi, Lucas and Jiang, Tao and Feraud, Matthieu and Bekbergenova, Madina and Taghzouti, Yousouf and Nothias, Louis-Felix},
   booktitle = {ISWC-C 2025},
-  year      = {2025}
+  series    = {CEUR Workshop Proceedings},
+  volume    = {4085},
+  year      = {2025},
+  url       = {https://ceur-ws.org/Vol-4085/paper77.pdf},
+  note      = {HAL archive: https://hal.science/hal-05233151v1}
 }
 ```
 

config_bge_en_icl.yml → config/embeddings/bge-en-icl.yml

@@ -21,7 +21,7 @@
 # Reranker: bge-reranker-v2-m3 pairs well with all BGE embeddings.
 #
 # Launch:
-#   uv run perspicacite -c config_bge_en_icl.yml serve
+#   uv run perspicacite -c config/embeddings/bge-en-icl.yml serve
 
 version: "2.0.0"
 config_name: "bge-en-icl-port8005"

config_bge_m3.yml → config/embeddings/bge-m3.yml

@@ -20,7 +20,7 @@
 # Full benchmark: perspicacite-eval/docs/retrieval_benchmark_2026_05_26.md
 #
 # Launch:
-#   uv run perspicacite -c config_bge_m3.yml serve
+#   uv run perspicacite -c config/embeddings/bge-m3.yml serve
 
 version: "2.0.0"
 config_name: "bge-m3-port8004"
@@ -69,7 +69,7 @@ rag_modes:
   # bge-reranker-v2-m3: domain-aware pair for bge-m3 embeddings.
   # Key finding: bge-reranker HELPS weaker/domain embeddings (bge-m3, MiniLM, PubMedBERT)
   # but HURTS strong embeddings (OpenAI 3-large: −2.1 pp vs ms-marco).
-  # Do not swap to ms-marco in this config — use config_openai_large.yml for OpenAI.
+  # Do not swap to ms-marco in this config — use config/embeddings/openai-large.yml for OpenAI.
   reranker_model: "BAAI/bge-reranker-v2-m3"
 
   basic:

config_biomedbert.yml → config/embeddings/biomedbert.yml

@@ -12,7 +12,7 @@
 # No API key required (fully local via sentence-transformers).
 #
 # Launch:
-#   uv run perspicacite -c config_biomedbert.yml serve
+#   uv run perspicacite -c config/embeddings/biomedbert.yml serve
 
 version: "2.0.0"
 config_name: "biomedbert-port8006"

config_code_kb.yml → config/embeddings/code-kb.yml

@@ -22,7 +22,7 @@
 # Costs: codestral-embed billed per OpenRouter pricing (new docs only; queries cheap).
 #
 # Launch:
-#   OPENROUTER_API_KEY=$OPENROUTER_API_KEY uv run perspicacite -c config_code_kb.yml serve
+#   OPENROUTER_API_KEY=$OPENROUTER_API_KEY uv run perspicacite -c config/embeddings/code-kb.yml serve
 
 version: "2.0.0"
 config_name: "code-kb-codestral-port8003"

config_gte_qwen2_7b.yml → config/embeddings/gte-qwen2-7b.yml

@@ -18,7 +18,7 @@
 # Note: requires `st:` prefix since Alibaba-NLP/ is not in auto-detected namespaces.
 #
 # Launch:
-#   uv run perspicacite -c config_gte_qwen2_7b.yml serve
+#   uv run perspicacite -c config/embeddings/gte-qwen2-7b.yml serve
 
 version: "2.0.0"
 config_name: "gte-qwen2-7b-port8007"

config_neuml_pubmedbert.yml → config/embeddings/neuml-pubmedbert.yml

@@ -12,7 +12,7 @@
 # No API key required (fully local via sentence-transformers).
 #
 # Launch:
-#   uv run perspicacite -c config_neuml_pubmedbert.yml serve
+#   uv run perspicacite -c config/embeddings/neuml-pubmedbert.yml serve
 
 version: "2.0.0"
 config_name: "neuml-pubmedbert-port8007"

config_openai_large.yml → config/embeddings/openai-large.yml

@@ -20,7 +20,7 @@
 # Full benchmark: perspicacite-eval/docs/retrieval_benchmark_2026_05_26.md
 #
 # Launch:
-#   OPENAI_API_KEY=$OPENAI_API_KEY uv run perspicacite -c config_openai_large.yml serve
+#   OPENAI_API_KEY=$OPENAI_API_KEY uv run perspicacite -c config/embeddings/openai-large.yml serve
 
 version: "2.0.0"
 config_name: "openai-large-port8002"