feat: eval real LLM, /api/audit endpoint, ag2 fallback warning, voice model hint

- eval/runner.py: add _provider_available(), run_case_real() wired to
  chat_runner.run_chat(); run_suite() auto-detects provider when stub=False
  and degrades silently to stub when no provider is reachable
- serve.py: add GET /api/audit endpoint calling run_audit() + serialising
  AuditReport as JSON (healthy/passed/failed/fixable/results[])
- cli.py: replace silent 'except ImportError: pass' in dispatch_run_cmd
  with actionable ag2 install hint for pip install ag2[ollama/anthropic]
- agent/voice.py: expand VoiceUnavailableError messages to include model
  download commands and HuggingFace URL for ggml-tiny.en.bin

REQ-321,REQ-328,REQ-331

Co-Authored-By: Oz <oz-agent@warp.dev>

Author: tbitcs

src/specsmith/agent/voice.py

@@ -102,16 +102,24 @@ def transcribe(path: Path) -> TranscribeResult:
         import whisper_cpp_python
     except Exception as exc:  # noqa: BLE001
         raise VoiceUnavailableError(
-            "whisper-cpp-python is not installed. Run "
-            "`pipx inject specsmith whisper-cpp-python` "
-            "(or `pip install specsmith[voice]`)."
+            "Voice transcription requires whisper-cpp-python.\n"
+            "Install it:\n"
+            "  pipx inject specsmith whisper-cpp-python  (recommended)\n"
+            "  pip install specsmith[voice]\n"
+            "Then download a model:\n"
+            f"  mkdir -p {default_model_dir()}\n"
+            "  curl -L https://huggingface.co/ggerganov/whisper.cpp/resolve/main/"
+            "ggml-tiny.en.bin -o ~/.specsmith/voice/ggml-tiny.en.bin"
         ) from exc
 
     model_path = _resolve_model_path()
     if model_path is None:
         raise VoiceUnavailableError(
-            "No whisper model found. Set SPECSMITH_VOICE_MODEL or place a "
-            f".bin model under {default_model_dir()}."
+            f"No whisper model found under {default_model_dir()}.\n"
+            "Download a model (replace tiny.en with base/small/medium as needed):\n"
+            "  curl -L https://huggingface.co/ggerganov/whisper.cpp/resolve/main/"
+            "ggml-tiny.en.bin -o ~/.specsmith/voice/ggml-tiny.en.bin\n"
+            "Or set SPECSMITH_VOICE_MODEL=/path/to/model.bin"
         )
 
     start = _time.perf_counter()

src/specsmith/cli.py

@@ -9002,7 +9002,13 @@ def _run_orch_json() -> None:
         return
     except ImportError:
         # AG2 not installed — fall through to manual path
-        pass
+        console.print(
+            "[yellow]\u26a0[/yellow]  ag2 not found \u2014 running single-node fallback DAG.\n"
+            "  Install full multi-agent support:\n"
+            "    [bold]pip install ag2\\[ollama][/bold]  (local Ollama)\n"
+            "    [bold]pip install ag2\\[anthropic][/bold]  (Anthropic Claude)\n"
+            "  Then re-run for parallel multi-agent dispatch."
+        )
     except Exception as exc:  # noqa: BLE001
         console.print(f"[yellow]Orchestrator unavailable ({exc}), using manual dispatch.[/yellow]")
 

src/specsmith/eval/runner.py

@@ -9,6 +9,94 @@
 from specsmith.eval import EvalCase, EvalReport, EvalResult, EvalSuite
 
 
+def _provider_available() -> str | None:
+    """Return the first available LLM provider name, or None."""
+    # Check Ollama first (local, always free)
+    try:
+        import urllib.request
+
+        req = urllib.request.Request(
+            "http://127.0.0.1:11434/api/tags", method="GET"
+        )
+        with urllib.request.urlopen(req, timeout=1):
+            pass
+        return "ollama"
+    except Exception:  # noqa: BLE001
+        pass
+    # Cloud providers via env keys
+    import os
+
+    for key, name in (
+        ("ANTHROPIC_API_KEY", "anthropic"),
+        ("OPENAI_API_KEY", "openai"),
+        ("GOOGLE_API_KEY", "gemini"),
+    ):
+        if os.environ.get(key, "").strip():
+            return name
+    return None
+
+
+def run_case_real(case: EvalCase, *, provider: str = "ollama") -> EvalResult:
+    """Run a single eval case against a live LLM provider.
+
+    Uses the specsmith provider chain (Ollama → Anthropic → OpenAI → Gemini).
+    On any provider failure, returns a failed result rather than raising so
+    the suite continues.
+    """
+    import os
+    from pathlib import Path
+
+    start = time.monotonic()
+    try:
+        from specsmith.agent.chat_runner import run_chat
+        from specsmith.agent.events import EventEmitter
+
+        project_dir = Path(os.environ.get("SPECSMITH_EVAL_PROJECT", ".")).resolve()
+        emitter = EventEmitter(stream=None)  # type: ignore[arg-type]
+        result = run_chat(
+            case.input,
+            project_dir=project_dir,
+            profile="standard",
+            session_id=f"eval-{case.id}",
+            emitter=emitter,
+            msg_block="eval-blk-001",
+        )
+        latency = (time.monotonic() - start) * 1000
+        if result is None:
+            return EvalResult(
+                case_id=case.id,
+                passed=False,
+                score=0.0,
+                latency_ms=latency,
+                model=provider,
+                provider=provider,
+                output_preview="",
+                error="provider returned None",
+            )
+        sc = score_output(result.summary, case.expected_keywords)
+        return EvalResult(
+            case_id=case.id,
+            passed=sc >= 0.5,
+            score=sc,
+            latency_ms=latency,
+            model=provider,
+            provider=provider,
+            output_preview=result.summary[:200],
+        )
+    except Exception as exc:  # noqa: BLE001
+        latency = (time.monotonic() - start) * 1000
+        return EvalResult(
+            case_id=case.id,
+            passed=False,
+            score=0.0,
+            latency_ms=latency,
+            model=provider,
+            provider=provider,
+            output_preview="",
+            error=str(exc)[:200],
+        )
+
+
 def score_output(output: str, expected_keywords: list[str]) -> float:
     """Score an output against expected keywords (0.0–1.0)."""
     if not expected_keywords:
@@ -50,15 +138,26 @@ def run_suite(
 
     Args:
         suite: The eval suite to run.
-        model: Model identifier.
-        provider: Provider name.
-        stub: If True, use stub runner (no real LLM calls).
+        model: Model identifier (ignored when stub=False; provider chosen
+               automatically via :func:`_provider_available`).
+        provider: Provider name (overridden by auto-detection when stub=False).
+        stub: If True, use stub runner. If False, auto-detect a live LLM
+              and run each case for real; silently falls back to stub when
+              no provider is reachable.
     """
+    # Auto-detect provider when real mode is requested
+    live_provider: str | None = None
+    if not stub:
+        live_provider = _provider_available()
+        if live_provider is None:
+            stub = True  # No provider found; degrade gracefully
+
     results: list[EvalResult] = []
     for case in suite.cases:
-        # Real LLM execution path will replace run_case_stub when
-        # provider integration is wired. For now, always use stub.
-        result = run_case_stub(case)
+        if stub:
+            result = run_case_stub(case)
+        else:
+            result = run_case_real(case, provider=live_provider or provider)
         results.append(result)
 
     total = len(results)

src/specsmith/serve.py

@@ -196,6 +196,8 @@ def do_GET(self) -> None:  # noqa: N802
             self._session_history()
         elif self.path.startswith("/api/session/context-seed"):
             self._session_context_seed()
+        elif self.path.startswith("/api/audit"):
+            self._audit_status()
         # ── Dispatch endpoints (REQ-332) ──────────────────────────────
         elif self.path.startswith("/api/dispatch/events"):
             self._dispatch_sse()
@@ -625,6 +627,33 @@ def _session_history(self) -> None:
         except Exception as exc:  # noqa: BLE001
             self._json_response({"ok": False, "error": str(exc), "history": []})
 
+    def _audit_status(self) -> None:
+        """GET /api/audit — return governance audit health status.
+
+        Kairos calls this to show live audit state in the governance page.
+        Returns the same data as ``specsmith audit`` in JSON form.
+        """
+        try:
+            from specsmith.auditor import run_audit
+
+            root = Path(self.agent._project_dir)  # noqa: SLF001
+            report = run_audit(root)
+            self._json_response(
+                {
+                    "ok": True,
+                    "healthy": report.healthy,
+                    "passed": report.passed,
+                    "failed": report.failed,
+                    "fixable": report.fixable,
+                    "results": [
+                        {"name": r.name, "passed": r.passed, "message": r.message}
+                        for r in report.results
+                    ],
+                }
+            )
+        except Exception as exc:  # noqa: BLE001
+            self._json_response({"ok": False, "error": str(exc)})
+
     def _session_context_seed(self) -> None:
         """GET /api/session/context-seed — return the epistemic continuity seed.