Merge pull request #18 from rxf-sys/claude/setup-workflow-process-knX5t

v2 redesign: Settings tab, rich service stats, section rebuilds, drill-down drawers

Author: rxf-sys

README.md

@@ -130,6 +130,12 @@ Configuration → Access Control → **Add** user `admin-dashboard@pbs`.
 Datastore → `<your-datastore>` → Permissions → **Add** for the user with
 role `DatastoreAudit`.
 
+If you want the "Verify"-button next to each snapshot in the dashboard
+to actually work, the same user additionally needs `DatastoreReader` or
+a custom role that includes `Datastore.Verify` on the datastore.
+Read-only `DatastoreAudit` will return 403 on `POST /admin/datastore/<store>/verify`
+— the dashboard surfaces that as a toast.
+
 Copy into `PBS_TOKEN_ID` / `PBS_TOKEN_SECRET`.
 
 #### Cloudflare API
@@ -227,6 +233,14 @@ so you can iterate without Cloudflare in the loop.
   as `ext=false`. The internal LAN probes (`probe_targets`) keep TLS
   verification off because home-lab services typically use self-signed
   or private-CA certs.
+- **Probe history**: the backend persists each probe sample in a SQLite
+  file (default `/data/rxf-admin.db`, configurable via
+  `STORAGE_DB_PATH`). Retention defaults to 7 days. The service drawer
+  shows the resulting uptime % and a 60-bucket history strip. To run
+  the dashboard *without* persistent history, set `STORAGE_DB_PATH=`
+  in the env — the UI falls back to "history disabled" gracefully.
+  When deploying via docker-compose, mount a volume on `/data` so the
+  history survives container restarts.
 
 ## Tech-stack rationale
 

backend/app/clients/cloudflare.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 import httpx
 import structlog
@@ -191,6 +191,76 @@ async def fetch_certs(settings: Settings) -> list[CertInfo]:
     return sorted(dedup.values(), key=lambda c: c.days_left)
 
 
+async def fetch_access_sessions(settings: Settings, hours: int = 24, limit: int = 100) -> dict:
+    """Pull recent Cloudflare Access login events.
+
+    Uses the audit-log API at ``/accounts/{id}/access/logs/access_requests``.
+    The CF token needs the "Access: Apps and Policies: Read" permission
+    (or equivalent audit-log scope) — without it the call returns 403 and
+    we surface ``reachable=false`` with the error so the UI can explain.
+
+    Returns ``{reachable, error, last_login_iso, sessions_24h, items: [...]}``.
+    Each item has ``email``, ``app_uid``, ``allowed``, ``created_at``,
+    ``ip``, ``country``. Window-bounded server-side; we accept up to 100
+    items so a busy account doesn't blow up the cache.
+    """
+    if not (settings.cf_api_token and settings.cf_account_id):
+        return {
+            "reachable": False,
+            "error": "CF_API_TOKEN oder CF_ACCOUNT_ID nicht konfiguriert",
+            "last_login_iso": None,
+            "sessions_24h": 0,
+            "items": [],
+        }
+    since = datetime.now(timezone.utc) - timedelta(hours=hours)
+    since_iso = since.isoformat().replace("+00:00", "Z")
+    path = (
+        f"/accounts/{settings.cf_account_id}/access/logs/access_requests"
+        f"?since={since_iso}&limit={limit}"
+    )
+    async with httpx.AsyncClient(timeout=8.0) as client:
+        try:
+            body = await _get_raw(client, settings, path)
+        except httpx.HTTPError as e:
+            log.info(
+                "cloudflare.access_sessions_failed",
+                account=settings.cf_account_id,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return {
+                "reachable": False,
+                "error": f"Access-Audit-Log nicht abrufbar — Token-Scope prüfen ({type(e).__name__})",
+                "last_login_iso": None,
+                "sessions_24h": 0,
+                "items": [],
+            }
+    raw = body.get("result")
+    items: list[dict] = []
+    if isinstance(raw, list):
+        for r in raw[:limit]:
+            if not isinstance(r, dict):
+                continue
+            items.append(
+                {
+                    "email": r.get("user_email"),
+                    "app_uid": r.get("app_uid"),
+                    "allowed": bool(r.get("allowed", True)),
+                    "created_at": r.get("created_at"),
+                    "ip": r.get("ip_address"),
+                    "country": r.get("country"),
+                }
+            )
+    last = items[0]["created_at"] if items else None
+    return {
+        "reachable": True,
+        "error": None,
+        "last_login_iso": last,
+        "sessions_24h": len(items),
+        "items": items,
+    }
+
+
 async def fetch_dns_consistency(settings: Settings) -> list[DNSRecordCheck]:
     if not (settings.cf_api_token and settings.cf_zone_id and settings.cf_tunnel_id):
         return []

backend/app/clients/pbs.py

@@ -48,6 +48,49 @@ def _verify_status(verification: dict | None) -> str:
     return "—"
 
 
+async def trigger_verify(
+    settings: Settings, backup_type: str, backup_id: str, backup_time: int
+) -> str | None:
+    """Kick off a verification job for a single PBS snapshot.
+
+    Returns the resulting UPID on success, ``None`` on error. The token needs
+    the ``Datastore.Verify`` privilege on the datastore — ``DatastoreAudit``
+    alone (the default for read-only dashboards) is not sufficient.
+    """
+    if not (settings.pbs_token_id and settings.pbs_token_secret):
+        return None
+    payload = {
+        "backup-type": backup_type,
+        "backup-id": backup_id,
+        "backup-time": backup_time,
+    }
+    async with httpx.AsyncClient(verify=settings.pbs_verify_tls, timeout=10.0) as client:
+        try:
+            r = await client.post(
+                f"{_base_url(settings)}/admin/datastore/{settings.pbs_datastore}/verify",
+                headers={**_auth_header(settings), "Content-Type": "application/json"},
+                json=payload,
+            )
+            r.raise_for_status()
+        except httpx.HTTPError as e:
+            log.warning(
+                "pbs.verify_trigger_failed",
+                backup_type=backup_type,
+                backup_id=backup_id,
+                backup_time=backup_time,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return None
+        try:
+            data = r.json().get("data")
+        except ValueError:
+            data = None
+        if isinstance(data, str) and data.startswith("UPID:"):
+            return data
+        return None
+
+
 async def fetch_backup_summary(settings: Settings) -> BackupSummary:
     if not (settings.pbs_token_id and settings.pbs_token_secret):
         return BackupSummary(
@@ -96,12 +139,17 @@ async def fetch_backup_summary(settings: Settings) -> BackupSummary:
         when_iso = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
         verify = _verify_status(snap.get("verification"))
         size = int(snap.get("size", 0))
-        target = f"{snap.get('backup-type', '?')}/{snap.get('backup-id', '?')}"
+        b_type = str(snap.get("backup-type", "?"))
+        b_id = str(snap.get("backup-id", "?"))
+        target = f"{b_type}/{b_id}"
         status = "err" if verify == "failed" else ("warn" if verify == "pending" else "ok")
         jobs.append(
             BackupSnapshot(
                 id=f"{target}@{ts}",
                 target=target,
+                backup_type=b_type,
+                backup_id=b_id,
+                backup_time=ts,
                 status=status,  # type: ignore[arg-type]
                 verify=verify,  # type: ignore[arg-type]
                 size_b=size,

backend/app/clients/probes.py

@@ -5,6 +5,7 @@
 
 import httpx
 
+from .. import storage
 from ..config import Settings
 from ..models import ServiceStatus
 
@@ -85,4 +86,13 @@ async def run(svc: dict[str, str]) -> ServiceStatus:
             )
 
         results = await asyncio.gather(*(run(s) for s in SERVICES))
+
+    # Persist a sample per service for the uptime view. Best-effort; if
+    # storage is disabled or the write fails it's a no-op (see storage.py).
+    await storage.record_probes([(r.id, r.status, int(r.ms)) for r in results])
+    # Track incident transitions so we can answer "last_incident_iso" across
+    # restarts. One row per state-change; serial calls within a single
+    # incident just bump the worst_status if the new probe is more severe.
+    for r in results:
+        await storage.update_service_incident(r.id, r.status)
     return list(results)

backend/app/clients/proxmox.py

@@ -259,6 +259,43 @@ async def fetch_task_log(settings: Settings, upid: str, limit: int = 200) -> lis
     return out
 
 
+async def fetch_host_journal_for_vmid(
+    settings: Settings, vmid: int, lastentries: int = 500
+) -> list[str]:
+    """Return host-journal lines that mention this VMID.
+
+    Proxmox does not expose an API to read inside a container — the
+    Integration API only offers ``/journal`` for the host. We fetch the last
+    N entries and filter for the VMID as a whole word, which catches the
+    usual ``lxc-<vmid>``, ``pve-container@<vmid>.service``, and bare-id
+    references that systemd / pveproxy / pve-firewall emit around lifecycle
+    events. This is *not* a replacement for ``journalctl`` inside the guest,
+    but it surfaces host-side events for that container without requiring
+    SSH access.
+    """
+    import re
+
+    async with httpx.AsyncClient(verify=settings.proxmox_verify_tls, timeout=10.0) as client:
+        try:
+            data = await _get(
+                client,
+                settings,
+                f"/nodes/{settings.proxmox_node}/journal?lastentries={lastentries}",
+            )
+        except httpx.HTTPError as e:
+            log.warning("proxmox.journal_failed", vmid=vmid, error=str(e))
+            return []
+
+    if not isinstance(data, list):
+        return []
+
+    # \b<vmid>\b matches the id as a standalone token. We also accept
+    # ``lxc-<vmid>`` and ``CT <vmid>`` shapes explicitly so a numeric-prefix
+    # in a longer word (e.g. memory addresses) doesn't generate false hits.
+    pattern = re.compile(rf"(?:\blxc-{vmid}\b|\bCT\s*{vmid}\b|@{vmid}\.service|\b{vmid}\b)")
+    return [line for line in data if isinstance(line, str) and pattern.search(line)]
+
+
 async def fetch_guest_tasks(settings: Settings, vmid: int, limit: int = 10) -> list[dict]:
     """Recent Proxmox cluster tasks scoped to a single VMID."""
     async with httpx.AsyncClient(verify=settings.proxmox_verify_tls, timeout=8.0) as client:

backend/app/config.py

@@ -90,6 +90,21 @@ class Settings(BaseSettings):
     # ---- IP Geolocation (for ISP name when UniFi doesn't expose it) ----
     geoip_enabled: bool = True
 
+    # ---- Probe history (SQLite) ----
+    # Where to keep persisted service-probe history. Set to "" to disable
+    # storage entirely; the dashboard then falls back to in-memory only.
+    storage_db_path: str = "/data/rxf-admin.db"
+    # How long to keep individual probe samples (days). Older rows are dropped
+    # by a periodic cleanup task in the lifespan.
+    history_retention_days: int = 7
+    # Cleanup loop tick (seconds).
+    history_cleanup_interval_s: int = 3600
+    # Metrics sampling loop tick (seconds). Pulls guest CPU/RAM from Proxmox
+    # and WAN throughput from UniFi, writing one row per guest + one row for
+    # network. 60 s is a sensible default; lower values trade DB churn for
+    # finer-grained charts. Set to 0 to disable the loop.
+    metrics_sample_interval_s: int = 60
+
     # ---- Notifications ----
     # Discord/Slack-compatible incoming webhook URL. Empty = disabled.
     notify_webhook_url: str = ""

backend/app/main.py

@@ -9,11 +9,21 @@
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from . import storage
 from .auth import verify_cf_access
-from .clients import cloudflare, pbs, probes
+from .clients import cloudflare, pbs, probes, proxmox, unifi
 from .config import get_settings
 from .notify import NotificationCenter, run_notification_loop
-from .routers import audit, backups, certs, network, services, system, tunnel
+from .routers import (
+    audit,
+    backups,
+    certs,
+    cloudflare as cloudflare_router,
+    network,
+    services,
+    system,
+    tunnel,
+)
 
 _settings = get_settings()
 logging.basicConfig(
@@ -70,12 +80,76 @@ async def _gather_notify_snapshot() -> dict:
     }
 
 
+async def _history_cleanup_loop() -> None:
+    """Periodically drop samples older than the retention window."""
+    while True:
+        try:
+            await asyncio.sleep(_settings.history_cleanup_interval_s)
+            await storage.cleanup_old(_settings.history_retention_days)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:  # noqa: BLE001 - never let the loop die
+            structlog.get_logger().error(
+                "history.cleanup_error", error=str(e), error_type=type(e).__name__
+            )
+
+
+async def _metrics_sample_loop() -> None:
+    """Pull guest CPU/RAM and WAN throughput every tick and persist them.
+
+    Lets the dashboard draw 24h CPU/RAM-per-guest and 1h WAN-throughput
+    charts without leaning on each upstream's (missing) history endpoints.
+    Best-effort: if either upstream is unreachable on this tick we skip
+    writing for that side rather than killing the loop.
+    """
+    interval = max(15, _settings.metrics_sample_interval_s)
+    log_ = structlog.get_logger("metrics")
+    while True:
+        try:
+            await asyncio.sleep(interval)
+            guests_task = asyncio.create_task(proxmox.fetch_guests(_settings))
+            net_task = asyncio.create_task(unifi.fetch_network_snapshot(_settings))
+            guests, net = await asyncio.gather(
+                guests_task, net_task, return_exceptions=True
+            )
+            if isinstance(guests, list):
+                rows = [
+                    (g.id, float(g.cpu_pct), int(g.ram_used_b), int(g.ram_total_b))
+                    for g in guests
+                    if g.running and g.type != "HOST"
+                ]
+                await storage.record_guest_metrics(rows)
+            else:
+                log_.info("metrics.guests_skip", error=str(guests))
+            if not isinstance(net, BaseException) and net.reachable:
+                await storage.record_network_metrics(
+                    float(net.throughput_down_mbit), float(net.throughput_up_mbit)
+                )
+            elif isinstance(net, BaseException):
+                log_.info("metrics.network_skip", error=str(net))
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:  # noqa: BLE001 - never let the loop die
+            log_.error(
+                "metrics.sample_error", error=str(e), error_type=type(e).__name__
+            )
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    task: asyncio.Task | None = None
+    notify_task: asyncio.Task | None = None
+    cleanup_task: asyncio.Task | None = None
+    metrics_task: asyncio.Task | None = None
+
+    await storage.ensure_schema(_settings)
+    if storage.is_enabled():
+        cleanup_task = asyncio.create_task(_history_cleanup_loop())
+        if _settings.metrics_sample_interval_s > 0:
+            metrics_task = asyncio.create_task(_metrics_sample_loop())
+
     if _settings.notify_webhook_url:
         center = NotificationCenter(settings=_settings)
-        task = asyncio.create_task(
+        notify_task = asyncio.create_task(
             run_notification_loop(
                 center, _gather_notify_snapshot, interval_s=_settings.notify_interval_s
             )
@@ -86,7 +160,9 @@ async def lifespan(app: FastAPI):
     try:
         yield
     finally:
-        if task:
+        for task in (notify_task, cleanup_task, metrics_task):
+            if task is None:
+                continue
             task.cancel()
             try:
                 await task
@@ -135,3 +211,5 @@ async def me(claims: dict = Depends(verify_cf_access)) -> dict:
 app.include_router(network.router)
 app.include_router(certs.router)
 app.include_router(audit.router)
+app.include_router(audit.events_router)
+app.include_router(cloudflare_router.router)