vector-agent: AI code generation, RAG observability, and data downloader UI (#63)

* Update Data Downloader

* Refactor AI Code Generation to use MiniMax SDK and update dependencies

* Increase max_tokens for ChatAnthropic and enhance output file collection in sandbox server

* Succesful cases RAG

* Fix RAG

* RAG vectors

* !aistats report

* Dashboard color improvement

Author: wfr-data-acquisition

rag_vectors.png

@@ -0,0 +1,127 @@
+"""
+RAG Vector Visualizer — PCA projection of ChromaDB collections.
+Run: uv run --with scikit-learn --with matplotlib python rag_viz.py
+"""
+import json
+import subprocess
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+
+COLORS = {
+    "sensors": "#6c7ee1",
+    "runs": "#e1735e",
+    "verified_solutions": "#4ecb71",
+}
+LABEL_NAMES = {
+    "sensors": "Sensors",
+    "runs": "Runs",
+    "verified_solutions": "Verified Solutions",
+}
+
+
+def fetch_collection(name: str) -> tuple[np.ndarray, list]:
+    """Return (embeddings, labels) from a ChromaDB collection inside code-generator container."""
+    script = f"""
+import chromadb, json
+_chroma = chromadb.PersistentClient(path='/app/chroma_db')
+col = _chroma.get_collection('{name}')
+all_data = col.get(include=['documents', 'metadatas', 'embeddings'])
+embs_raw = all_data.get('embeddings', [])
+embs = [list(e) for e in embs_raw]
+ids_ = all_data.get('ids', [])
+docs = all_data.get('documents', [])
+metas = all_data.get('metadatas', [])
+labels = []
+for id_, meta, doc in zip(ids_, metas, docs):
+    if '{name}' == 'sensors':
+        labels.append(meta.get('name', id_) if meta else id_)
+    elif '{name}' == 'runs':
+        labels.append(f"{{meta.get('season','')}}:{{meta.get('key', id_[:6])}}" if meta else id_[:12])
+    else:
+        labels.append((doc or id_)[:80].replace('\\n', ' '))
+result = json.dumps({{'embs': embs, 'labels': labels}})
+print('START' + result + 'END')
+"""
+    result = subprocess.run(
+        ["docker", "exec", "code-generator", "python", "-c", script],
+        capture_output=True, text=True, check=True,
+    )
+    raw = result.stdout.strip()
+    start = raw.index("START") + 5
+    end = raw.rindex("END")
+    data = json.loads(raw[start:end])
+    return np.array(data["embs"], dtype=np.float32), data["labels"]
+
+
+def main():
+    all_X, all_labels, all_names = [], [], []
+
+    for name in ["sensors", "runs", "verified_solutions"]:
+        X, labels = fetch_collection(name)
+        n = len(X)
+        print(f"{name}: {n} vectors, dim={X.shape[1]}")
+        all_X.append(X)
+        all_labels.extend(labels)
+        all_names.extend([name] * n)
+
+    X_all = np.vstack(all_X)
+    print(f"Combined: {len(X_all)} vectors, dim={X_all.shape[1]}")
+
+    # PCA on combined space
+    pca = PCA(n_components=2)
+    X2_all = pca.fit_transform(X_all)
+
+    # Plot
+    fig, ax = plt.subplots(figsize=(14, 10))
+    cumulative = 0
+
+    for name in ["sensors", "runs", "verified_solutions"]:
+        n = sum(1 for x in all_names if x == name)
+        if n == 0:
+            continue
+        start = cumulative
+        cumulative += n
+        X2 = X2_all[start:cumulative]
+        labels_slice = all_labels[start:cumulative]
+        color = COLORS[name]
+
+        # Subsample sensors for visual clarity
+        if name == "sensors" and n > 200:
+            np.random.seed(42)
+            idx = np.random.choice(n, 200, replace=False)
+            X2 = X2[idx]
+            labels_slice = [labels_slice[i] for i in idx]
+            count_label = f"{LABEL_NAMES[name]} (200/{n})"
+        else:
+            count_label = f"{LABEL_NAMES[name]} ({n})"
+
+        ax.scatter(X2[:, 0], X2[:, 1], c=color, label=count_label,
+                    alpha=0.7, s=60,
+                    edgecolors="white" if name != "sensors" else "none",
+                    linewidths=0.5)
+
+        # Annotate runs and verified_solutions
+        if name in ("runs", "verified_solutions"):
+            for (x, y), lbl in zip(X2, labels_slice):
+                short = lbl[:32].replace("\n", " ")
+                ax.annotate(
+                    short, (x, y), fontsize=8, alpha=0.95,
+                    xytext=(6, 6), textcoords="offset points",
+                    bbox=dict(boxstyle="round,pad=0.3", facecolor="white",
+                              alpha=0.75, edgecolor="none"),
+                )
+
+    ax.set_xlabel(f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}% variance)", fontsize=12)
+    ax.set_ylabel(f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}% variance)", fontsize=12)
+    ax.set_title("RAG Vector Space — ChromaDB Collections (PCA projection)", fontsize=14, fontweight="bold")
+    ax.legend(fontsize=11, loc="upper right")
+    ax.grid(True, alpha=0.25)
+    plt.tight_layout()
+    out = "rag_vectors.png"
+    plt.savefig(out, dpi=150)
+    print(f"Saved {out}")
+
+
+if __name__ == "__main__":
+    main()

rag_viz.py

@@ -69,25 +69,19 @@ DEBUG=0
 # Data Downloader configuration
 # ------------------------------------------------------------
 POSTGRES_DSN=postgresql://wfr:wfr_password@timescaledb:5432/wfr
-# Comma-separated season list for data-downloader season switcher.
-# Format: NAME:YEAR[:COLOR_RGB], newest first.
-# Add previous seasons here to keep them accessible in the UI.
-# Example with two seasons: SEASONS=WFR25:2025,WFR26:2026
-SEASONS=WFR26:2026
+# Optional: explicitly list seasons (NAME:YEAR[:COLOR], newest first).
+# If omitted, the scanner auto-discovers all season tables from TimescaleDB.
+# Example: SEASONS=WFR25:2025,WFR26:2026
+# SEASONS=
 
 DATA_DIR=/app/data
 
-SCANNER_YEAR=2025
 SCANNER_BIN=hour
 SCANNER_INCLUDE_COUNTS=true
 SCANNER_INITIAL_CHUNK_DAYS=31
 
 SENSOR_WINDOW_DAYS=7
-# How far back to look for sensor data when scanning
 SENSOR_LOOKBACK_DAYS=365
-# If no sensor data is found in the lookback period, use this fallback range
-SENSOR_FALLBACK_START=2025-06-10T00:00:00
-SENSOR_FALLBACK_END=2025-07-10T00:00:00
 
 SCAN_INTERVAL_SECONDS=3600
 
@@ -97,13 +91,11 @@ ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173
 # End Data Downloader configuration
 
 # ------------------------------------------------------------
-# AI Code Generation (Sandbox) configuration
+# AI Code Generation — MiniMax (Anthropic-compatible SDK)
 # ------------------------------------------------------------
-# Cohere API key for AI-powered code generation
-COHERE_API_KEY=your-cohere-api-key-here
-
-# Cohere model to use (default: command-r-plus)
-COHERE_MODEL=command-r-plus
+ANTHROPIC_API_KEY=your-minimax-api-key-here
+ANTHROPIC_BASE_URL=https://api.minimaxi.com/anthropic
+ANTHROPIC_MODEL=MiniMax-M2.7
 
 # Maximum number of retries when generated code fails (default: 2)
 MAX_RETRIES=2

server/installer/.env.example

@@ -58,7 +58,12 @@ class CanFramesBatchPayload(BaseModel):
 
 @app.get("/api/health")
 def healthcheck() -> dict:
-    return {"status": "ok"}
+    ok, detail = service.check_db_connectivity()
+    return {
+        "status": "ok" if ok else "degraded",
+        "timescaledb": ok,
+        "timescaledb_detail": detail,
+    }
 
 
 def _docker_container_running(container_name: str) -> bool:
@@ -129,13 +134,19 @@ def trigger_scan(background_tasks: BackgroundTasks, season: str | None = None) -
 @app.post("/api/query")
 def query_signal(payload: DataQueryPayload, season: str | None = None) -> dict:
     limit = None if payload.no_limit else (payload.limit or 2000)
-    return service.query_signal_series(
-        payload.signal,
-        payload.start,
-        payload.end,
-        limit,
-        season=season
-    )
+    try:
+        return service.query_signal_series(
+            payload.signal,
+            payload.start,
+            payload.end,
+            limit,
+            season=season,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc))
+    except Exception as exc:
+        logger.exception("Query failed for signal %s", payload.signal)
+        raise HTTPException(status_code=503, detail=f"Database query failed: {exc}")
 
 
 # ── CAN frames batch ingest (for flight-recorder sync) ─────────────────────────
@@ -185,7 +196,7 @@ def ingest_can_frames(payload: CanFramesBatchPayload) -> dict:
     if not payload.frames:
         return {"ingested": 0}
 
-    table = f"{payload.season.lower()}_base"
+    table = payload.season.lower()
 
     # Deduplicate within batch: last frame wins for (time, message_name)
     seen: dict = {}
@@ -238,13 +249,9 @@ def ingest_can_frames(payload: CanFramesBatchPayload) -> dict:
 @app.get("/", response_class=HTMLResponse)
 def index():
     """Simple status page for debugging."""
-    try:
-        service._log_db_connectivity()
-        timescale_status = "Connected"
-        timescale_color = "green"
-    except Exception as e:
-        timescale_status = f"Error: {e}"
-        timescale_color = "red"
+    db_ok, db_detail = service.check_db_connectivity()
+    timescale_status = db_detail if db_ok else f"Error: {db_detail}"
+    timescale_color = "green" if db_ok else "red"
 
     # Default to first season for overview
     runs = service.get_runs()

server/installer/data-downloader/backend/app.py

@@ -25,9 +25,14 @@ class SeasonConfig(BaseModel):
 
 
 def _parse_seasons(raw: str | None) -> List[SeasonConfig]:
-    """Parse SEASONS env var: \"WFR25:2025:colour,WFR26:2026\"."""
+    """
+    Parse SEASONS env var: \"WFR25:2025:colour,WFR26:2026\".
+
+    Returns an empty list when SEASONS is not set — the service will then
+    auto-discover season tables directly from TimescaleDB.
+    """
     if not raw:
-        return [SeasonConfig(name="WFR25", year=2025, table="wfr25")]
+        return []
 
     seasons = []
     for part in raw.split(","):
@@ -50,9 +55,6 @@ def _parse_seasons(raw: str | None) -> List[SeasonConfig]:
         except ValueError:
             continue
 
-    if not seasons:
-        return [SeasonConfig(name="WFR25", year=2025, table="wfr25")]
-
     seasons.sort(key=lambda s: s.year, reverse=True)
     return seasons
 

server/installer/data-downloader/backend/config.py

@@ -24,9 +24,15 @@ async def run_worker():
 
     while True:
         try:
-            active_season = settings.seasons[0]  # sorted descending by year; first = active
-            logging.info(f"Running scheduled scan for active season: {active_season.name}")
-            service.run_full_scan(source="periodic", season_names=[active_season.name])
+            # scan all known seasons (auto-discovered or explicitly configured)
+            seasons = service._seasons
+            if seasons:
+                active_season = seasons[0]  # newest first
+                logging.info(f"Running scheduled scan for active season: {active_season.name}")
+                service.run_full_scan(source="periodic", season_names=[active_season.name])
+            else:
+                logging.info("No seasons available yet, running full discovery scan")
+                service.run_full_scan(source="periodic")
             logging.info("Finished scheduled scan.")
 
             if daily_time: