Improve agent planning and observability (#380)

* Improve agent planning and observability

* Stabilize account provisioning planner route

* Harden structured agent planner boundaries

Author: michaelmwu

.env.example

@@ -168,6 +168,9 @@ LANGFUSE_BASE_URL=
 # Docker-network Bifrost URL http://bifrost:8080/openai is allowed.
 AGENT_PLANNER_MODEL=accounts/fireworks/models/kimi-k2p6
 AGENT_FALLBACK_MODEL=gpt-4.1-mini
+# Keep this below AGENT_API_TIMEOUT_SECONDS so deterministic fallback can respond.
+AGENT_STRUCTURED_PLANNER_ENABLED=true
+AGENT_STRUCTURED_PLANNER_TIMEOUT_SECONDS=6.0
 AGENT_INTENT_NORMALIZER_ENABLED=true
 AGENT_INTENT_NORMALIZER_TIMEOUT_SECONDS=3.0
 AGENT_FAST_MODEL=

ENVIRONMENT.md

@@ -169,10 +169,12 @@ Pydantic import errors.
 - `Optional`: `AGENT_REASONING_MODEL`, `AGENT_REASONING_BASE_URL`, `AGENT_REASONING_API_KEY`
 - `Optional`: `AGENT_PLANNER_MODEL` (default: `accounts/fireworks/models/kimi-k2p6`)
 - `Optional`: `AGENT_FALLBACK_MODEL` (default: `gpt-4.1-mini`; uses `OPENAI_API_KEY` / `OPENAI_BASE_URL`)
-- `Optional`: `AGENT_INTENT_NORMALIZER_ENABLED` (default: `true`; when deterministic parsing fails, asks the fast agent model to rewrite loose phrasing into one supported command shape)
-- `Optional`: `AGENT_INTENT_NORMALIZER_TIMEOUT_SECONDS` (default: `3.0`)
+- `Optional`: `AGENT_STRUCTURED_PLANNER_ENABLED` (default: `true`; enables the production proposal-only structured planner when a model provider is configured)
+- `Optional`: `AGENT_INTENT_NORMALIZER_ENABLED` (default: `true`; enables the legacy rewrite fallback for unsupported phrasing)
+- `Optional`: `AGENT_INTENT_NORMALIZER_TIMEOUT_SECONDS` (default: `3.0`; timeout for legacy normalization calls)
+- `Optional`: `AGENT_STRUCTURED_PLANNER_TIMEOUT_SECONDS` (default: `6.0`; timeout for the production structured planner; keep it below `AGENT_API_TIMEOUT_SECONDS` so deterministic fallback can respond)
 - Note: tier-specific agent models can point at OpenAI-compatible providers such as Bifrost or Fireworks. Agent model base URLs must be HTTPS endpoints on `bifrost.508.dev`, `api.openai.com`, `api.fireworks.ai`, or `openrouter.ai`, except the internal Docker-network Bifrost URL `http://bifrost:8080/openai` is also allowed for same-host deployments. If `OPENAI_BASE_URL` points at Bifrost and tier-specific `AGENT_*` values are unset, the planner defaults to Fireworks Kimi via Bifrost as `fireworks/accounts/fireworks/models/kimi-k2p6`. Explicit Bifrost provider-prefixed planner models, such as `openrouter/openai/gpt-4.1-mini`, are passed through unchanged. If Bifrost is not configured and `FIREWORKS_API_KEY` is set, the planner falls back to direct Fireworks as `accounts/fireworks/models/kimi-k2p6`. If a configured provider is missing its usable API key, it is skipped and the fallback order is `reasoning -> strong -> fast -> AGENT_FALLBACK_MODEL -> gpt-4.1-mini`; `strong` falls back through `fast`, and `fast` falls back through the OpenAI fallback.
-- Agent tools follow the deterministic path: deterministic parsing runs first, the optional LLM intent normalizer can only rewrite unsupported phrasing into supported command shapes, policy authorizes scopes, write tools require confirmation, and the backend executes known-good tool code.
+- Agent tools follow a proposal-and-policy path: when configured, the structured planner drafts a bounded typed tool plan using the selected fast or strong tier; every drafted action is allowlisted and shape-validated before deterministic policy authorizes it. Write tools require confirmation and the backend executes only known-good tool code. Provider failures fall back to deterministic parsing, then legacy normalization for unsupported phrasing.
 - `Optional`: `GITHUB_API_TOKEN`, `GITHUB_DEFAULT_REPO`, `GITHUB_ALLOWED_REPOS` (comma-separated; GitHub Issues are the canonical code-task backend for agent-created code work, and agent tools only access the default/allowed repositories).
 - Existing integration tools also expose CRM contact search/update, DocuSeal member-agreement submission, and Migadu mailbox creation when their normal service env vars are configured.
 - Note: the current generic task tool registry is process-local and non-durable for non-code/org tasks until the task-management platform is selected.

apps/admin_dashboard/src/main.tsx

@@ -416,6 +416,9 @@ type AgentReport = {
   status_counts?: Record<string, number>
   intent_counts?: Record<string, number>
   planner_counts?: Record<string, number>
+  model_counts?: Record<string, number>
+  action_counts?: Record<string, number>
+  tool_outcome_counts?: Record<string, number>
   recent_unsupported?: Array<{
     occurred_at?: string
     actor?: string
@@ -7873,6 +7876,9 @@ function AgentView({
       ["Status", report?.status_counts || {}],
       ["Intent", report?.intent_counts || {}],
       ["Planner", report?.planner_counts || {}],
+      ["Model", report?.model_counts || {}],
+      ["Action", report?.action_counts || {}],
+      ["Tool outcome", report?.tool_outcome_counts || {}],
     ] as const
   )
     .flatMap(([label, counts]) =>

apps/api/src/five08/backend/api.py

@@ -49,6 +49,7 @@
     AgentRequest,
     AgentResponse,
     InMemoryTaskStore,
+    OpenAICompatibleAgentPlanner,
     OpenAICompatibleIntentNormalizer,
     PolicyEngine,
     ToolRegistry,
@@ -251,9 +252,7 @@ class JobsQueryFilters:
 _JOB_FUNCTIONS = JOB_FUNCTIONS
 _ONBOARDING_STATUS_FIELD = "cOnboardingState"
 _ONBOARDER_FIELD = "cOnboarder"
-_GENERIC_UNSUPPORTED_AGENT_MESSAGE = (
-    "I could not turn that into a supported task action."
-)
+_GENERIC_UNSUPPORTED_AGENT_MESSAGE = "I could not map that to a supported workflow."
 _SENSITIVE_PAYLOAD_KEY_MARKERS = (
     "api_key",
     "apikey",
@@ -352,6 +351,7 @@ def _get_agent_orchestrator() -> AgentOrchestrator:
                     ),
                 ),
                 model_config=AgentModelConfig.from_settings(settings),
+                planner=OpenAICompatibleAgentPlanner.from_settings(settings),
                 intent_normalizer=OpenAICompatibleIntentNormalizer.from_settings(
                     settings
                 ),
@@ -1820,6 +1820,20 @@ def _agent_request_audit_metadata(
         "intent": response.plan.intent if response.plan else None,
         "planner": response.plan.planner if response.plan else None,
         "operation_id": response.plan.operation_id if response.plan else None,
+        "model": response.plan.model.model if response.plan else None,
+        "model_tier": response.plan.model_tier if response.plan else None,
+        "model_source_tier": (
+            response.plan.model.source_tier if response.plan else None
+        ),
+        "action_names": (
+            [action.tool_name for action in response.plan.actions]
+            if response.plan
+            else []
+        ),
+        "tool_outcomes": [
+            {"tool_name": result.tool_name, "status": result.status}
+            for result in response.results
+        ],
         "context_sources": (
             [source.model_dump(mode="json") for source in response.plan.context_sources]
             if response.plan
@@ -3000,7 +3014,11 @@ def _shape_dashboard_agent_request_report(
     rows: list[dict[str, Any]],
 ) -> dict[str, Any]:
     summary = {
-        "total": len(rows),
+        "total": sum(
+            1
+            for row in rows
+            if str(row.get("action") or "agent.request") == "agent.request"
+        ),
         "handled": 0,
         "requires_confirmation": 0,
         "needs_clarification": 0,
@@ -3010,20 +3028,39 @@ def _shape_dashboard_agent_request_report(
     status_counts: dict[str, int] = {}
     intent_counts: dict[str, int] = {}
     planner_counts: dict[str, int] = {}
+    model_counts: dict[str, int] = {}
+    action_counts: dict[str, int] = {}
+    tool_outcome_counts: dict[str, int] = {}
     recent_unsupported: list[dict[str, Any]] = []
 
     for row in rows:
+        event_action = str(row.get("action") or "agent.request")
         metadata = row.get("metadata")
         if not isinstance(metadata, dict):
             metadata = {}
         status = str(metadata.get("status") or "unknown")
         intent = metadata.get("intent") or "unknown"
         planner = metadata.get("planner") or "unknown"
+        model = metadata.get("model") or "unknown"
+
+        if event_action == "agent.request":
+            _increment_dashboard_count(status_counts, status)
+            _increment_dashboard_count(intent_counts, intent)
+            _increment_dashboard_count(planner_counts, planner)
+            _increment_dashboard_count(model_counts, model)
+            for action_name in metadata.get("action_names") or []:
+                _increment_dashboard_count(action_counts, action_name)
+        for outcome in metadata.get("tool_outcomes") or []:
+            if not isinstance(outcome, dict):
+                continue
+            tool_name = str(outcome.get("tool_name") or "unknown")
+            status_label = str(outcome.get("status") or "unknown")
+            _increment_dashboard_count(
+                tool_outcome_counts, f"{tool_name}:{status_label}"
+            )
 
-        _increment_dashboard_count(status_counts, status)
-        _increment_dashboard_count(intent_counts, intent)
-        _increment_dashboard_count(planner_counts, planner)
-
+        if event_action != "agent.request":
+            continue
         if status in {"executed", "requires_confirmation"}:
             summary["handled"] += 1
         if status == "requires_confirmation" or metadata.get("requires_confirmation"):
@@ -3065,30 +3102,55 @@ def _shape_dashboard_agent_request_report(
         "status_counts": status_counts,
         "intent_counts": intent_counts,
         "planner_counts": planner_counts,
+        "model_counts": model_counts,
+        "action_counts": action_counts,
+        "tool_outcome_counts": tool_outcome_counts,
         "recent_unsupported": recent_unsupported,
     }
 
 
 def _dashboard_agent_request_report(limit: int) -> dict[str, Any]:
     limit = _limit_dashboard_count(limit)
     sql = """
-        SELECT
-            id::text,
-            occurred_at,
-            result,
-            actor_provider,
-            actor_subject,
-            actor_display_name,
-            correlation_id,
-            metadata
-        FROM audit_events
-        WHERE action = 'agent.request'
+        WITH recent_requests AS (
+            SELECT
+                id::text,
+                occurred_at,
+                action,
+                result,
+                actor_provider,
+                actor_subject,
+                actor_display_name,
+                correlation_id,
+                metadata
+            FROM audit_events
+            WHERE action = 'agent.request'
+            ORDER BY occurred_at DESC
+            LIMIT %s
+        ), recent_confirmations AS (
+            SELECT
+                id::text,
+                occurred_at,
+                action,
+                result,
+                actor_provider,
+                actor_subject,
+                actor_display_name,
+                correlation_id,
+                metadata
+            FROM audit_events
+            WHERE action = 'agent.confirmation'
+            ORDER BY occurred_at DESC
+            LIMIT %s
+        )
+        SELECT * FROM recent_requests
+        UNION ALL
+        SELECT * FROM recent_confirmations
         ORDER BY occurred_at DESC
-        LIMIT %s
     """
     with get_postgres_connection(settings) as conn:
         with conn.cursor(row_factory=dict_row) as cursor:
-            cursor.execute(sql, (limit,))
+            cursor.execute(sql, (limit, limit))
             rows = cursor.fetchall()
 
     return _shape_dashboard_agent_request_report(rows)
@@ -8845,7 +8907,17 @@ async def agent_confirmation_handler(
         plan=plan,
         metadata={
             "status": response.status,
+            "intent": plan.intent,
+            "planner": plan.planner,
+            "model": plan.model.model,
+            "model_tier": plan.model_tier,
+            "model_source_tier": plan.model.source_tier,
+            "action_names": [action.tool_name for action in plan.actions],
             "results": [result.model_dump(mode="json") for result in results],
+            "tool_outcomes": [
+                {"tool_name": result.tool_name, "status": result.status}
+                for result in results
+            ],
         },
     )
     return JSONResponse(

apps/api/src/five08/backend/static/dashboard/.vite/manifest.json

@@ -1,6 +1,6 @@
 {
   "index.html": {
-    "file": "assets/index-Ce7YIjfO.js",
+    "file": "assets/index-DCDV0WK0.js",
     "name": "index",
     "src": "index.html",
     "isEntry": true,

…tatic/dashboard/assets/index-Ce7YIjfO.js …tatic/dashboard/assets/index-DCDV0WK0.jsapps/api/src/five08/backend/static/dashboard/assets/index-Ce7YIjfO.js renamed to apps/api/src/five08/backend/static/dashboard/assets/index-DCDV0WK0.js apps/api/src/five08/backend/static/dashboard/assets/index-Ce7YIjfO.js renamed to apps/api/src/five08/backend/static/dashboard/assets/index-DCDV0WK0.js

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>508 Operations Dashboard</title>
-    <script type="module" crossorigin src="/dashboard/assets/index-Ce7YIjfO.js"></script>
+    <script type="module" crossorigin src="/dashboard/assets/index-DCDV0WK0.js"></script>
     <link rel="stylesheet" crossorigin href="/dashboard/assets/index-2UypYUrJ.css">
   </head>
   <body>

apps/api/src/five08/backend/static/dashboard/index.html

@@ -48,17 +48,21 @@ Agent command flow:
 /agent request or @bot mention
   -> bot resolves Discord user/guild/channel/role context
   -> POST /agent/requests on the backend API
-  -> backend parses the request into a typed plan
+  -> backend uses a structured proposal planner when configured, with deterministic fallback
+  -> backend validates the typed plan and authorizes every proposed tool
   -> deterministic backend policy authorizes each proposed tool
   -> read actions execute synchronously
   -> write actions return a frozen confirmation plan
   -> Discord confirmation button calls POST /agent/confirmations/{plan_id}
   -> backend executes the exact frozen plan inline and returns the result
 ```
 
-Current MVP scope is task-style commands only. The backend agent package keeps
-read and write tools separate, applies capability checks before every tool call,
-requires confirmation for writes, and audits request/confirmation attempts.
+The backend agent package keeps read and write tools separate, applies
+capability checks before every tool call, requires confirmation for writes, and
+audits request/confirmation attempts. The model only drafts bounded tool calls;
+it cannot authorize users or execute integrations. Supported workflows include
+tasks, GitHub issues, CRM contacts, member agreements, account provisioning,
+and private memory according to the requester's roles.
 Long-running service changes should be implemented as PR-based workflows rather
 than direct production mutations. Task reads require an explicit project filter
 to avoid guild-wide task enumeration.

apps/discord_bot/README.md

@@ -15,6 +15,7 @@
 from discord import app_commands
 from discord.ext import commands
 
+from five08.agent import AgentIdentityContext, PolicyEngine
 from five08.discord_bot.config import settings
 from five08.discord_bot.utils.audit import DiscordAuditCogMixin
 from five08.tls import default_ca_bundle_path
@@ -24,14 +25,12 @@
 _MENTION_RATE_LIMIT_MAX_REQUESTS = 5
 _PUBLIC_SAFE_CLARIFICATION_MESSAGES = frozenset(
     {
-        "I could not turn that into a supported task action.",
+        "I could not map that to a supported workflow.",
         "Which project should I search?",
         "What should the task be?",
     }
 )
-_GENERIC_UNSUPPORTED_AGENT_MESSAGE = (
-    "I could not turn that into a supported task action."
-)
+_GENERIC_UNSUPPORTED_AGENT_MESSAGE = "I could not map that to a supported workflow."
 _AGENT_RESPONSE_THREAD_NAME = "Agent response"
 _AGENT_HELP_REQUESTS = frozenset(
     {
@@ -558,25 +557,41 @@ def _agent_capabilities_message(
         roles: list[str],
         transport: Literal["slash", "mention"] = "mention",
     ) -> str:
-        normalized_roles = {role.strip().casefold() for role in roles}
-        is_admin = bool(normalized_roles & {"admin", "owner", "steering committee"})
-        is_engineer = (
-            "engineer" in normalized_roles
-            or "workflows engineer" in normalized_roles
-            or is_admin
+        scopes = PolicyEngine().scopes_for_context(
+            AgentIdentityContext(
+                discord_user_id="capability-preview",
+                organization_id="capability-preview",
+                roles=roles,
+            )
         )
         capabilities: list[str] = []
-        if is_engineer:
+        if "project:read" in scopes:
+            capabilities.append("- Tasks: search a project.")
+        task_writes: list[str] = []
+        if "task:create" in scopes:
+            task_writes.append("create tasks")
+        if "task:update_own" in scopes:
+            task_writes.append("update your own tasks")
+        if task_writes:
+            capabilities.append(f"- Task writes: {', and '.join(task_writes)}.")
+        if {"memory:read_self", "memory:write_self"} & scopes:
+            capabilities.append(
+                "- Memory: remember and review your private preferences."
+            )
+        if "github:issue:read" in scopes:
             capabilities.append(
                 "- GitHub issues: search issues or create new code-task issues."
             )
-        if is_admin:
+        if "crm:contact:read" in scopes:
             capabilities.extend(
                 [
                     "- CRM: search contacts, approve/reject onboarding, and submit member agreements.",
-                    "- Ops: create 508 accounts, Authentik SSO users, Outline invites, and mailboxes.",
                 ]
             )
+        if "user:manage" in scopes or "mailbox:create" in scopes:
+            capabilities.append(
+                "- Ops: create 508 accounts, Authentik SSO users, Outline invites, and mailboxes."
+            )
         if not capabilities:
             lines = [
                 "I do not see any agent workflows available for your current Discord roles."

apps/discord_bot/src/five08/discord_bot/cogs/agent.py

@@ -61,6 +61,8 @@ class WorkerSettings(SharedSettings):
     openrouter_api_key: str | None = None
     agent_planner_model: str = "accounts/fireworks/models/kimi-k2p6"
     agent_fallback_model: str = "gpt-4.1-mini"
+    agent_structured_planner_enabled: bool = True
+    agent_structured_planner_timeout_seconds: float = 6.0
     agent_intent_normalizer_enabled: bool = True
     agent_intent_normalizer_timeout_seconds: float = 3.0
     agent_fast_api_key: str | None = None