restructure dashboard chat around metadata artifacts

ddpui/api/dashboard_native_api.py

@@ -20,6 +20,7 @@
     DashboardAIContext,
     DashboardChatMessage,
     DashboardChatMessageRole,
+    DashboardChatMetadataArtifact,
 )
 from ddpui.models.org_preferences import OrgPreferences
 from ddpui.models.org_user import OrgUser
@@ -67,47 +68,22 @@
 dashboard_native_router = Router()
 
 
-def _get_or_create_dashboard_ai_context(dashboard: Dashboard):
-    context, _ = DashboardAIContext.objects.get_or_create(dashboard=dashboard)
-    return context
-
-
 def _serialize_dashboard_ai_context(dashboard: Dashboard, context: DashboardAIContext):
-    org_dbt = dashboard.org.dbt
+    metadata_artifact = (
+        DashboardChatMetadataArtifact.objects.filter(dashboard=dashboard)
+        .only("built_at")
+        .first()
+    )
     return DashboardAIContextResponse(
         dashboard_id=dashboard.id,
         dashboard_title=dashboard.title,
         dashboard_context_markdown=context.markdown,
         dashboard_context_updated_by=context.updated_by.user.email if context.updated_by else None,
         dashboard_context_updated_at=context.updated_at,
-        ai_context_refreshed_at=org_dbt.vector_last_ingested_at if org_dbt else None,
+        metadata_last_built_at=metadata_artifact.built_at if metadata_artifact else None,
     )
 
 
-def _serialize_dashboard_chat_bootstrap(dashboard: Dashboard) -> DashboardChatBootstrapResponse:
-    dashboard_export = DashboardService.export_dashboard_context_for_dashboard(dashboard, dashboard.org)
-
-    return DashboardChatBootstrapResponse(
-        dashboard_id=dashboard.id,
-        suggested_prompts=build_dashboard_suggested_prompts(
-            dashboard_export=dashboard_export,
-        ),
-    )
-
-
-def _ensure_dashboard_chat_feature_enabled(org) -> None:
-    """Hide dashboard chat settings endpoints unless the feature flag is enabled."""
-    if not get_all_feature_flags_for_org(org).get("AI_DASHBOARD_CHAT", False):
-        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
-
-
-def _ensure_dashboard_chat_consent_enabled(org) -> None:
-    """Require consent before writing dashboard-specific AI context."""
-    org_preferences = OrgPreferences.objects.filter(org=org).first()
-    if org_preferences is None or not org_preferences.ai_data_sharing_enabled:
-        raise HttpError(409, "Enable AI data sharing before updating dashboard AI context")
-
-
 # Endpoints
 @dashboard_native_router.get("/", response=List[DashboardResponse])
 @has_permission(["can_view_dashboards"])
@@ -164,14 +140,15 @@ def export_dashboard(request, dashboard_id: int):
 def get_dashboard_ai_context(request, dashboard_id: int):
     """Load dashboard-level AI context settings for settings management."""
     orguser: OrgUser = request.orguser
-    _ensure_dashboard_chat_feature_enabled(orguser.org)
+    if not get_all_feature_flags_for_org(orguser.org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
 
     try:
         dashboard = DashboardService.get_dashboard(dashboard_id, orguser.org)
     except DashboardNotFoundError as err:
         raise HttpError(404, "Dashboard not found") from err
 
-    context = _get_or_create_dashboard_ai_context(dashboard)
+    context, _ = DashboardAIContext.objects.get_or_create(dashboard=dashboard)
 
     return _serialize_dashboard_ai_context(dashboard, context)
 
@@ -184,14 +161,21 @@ def get_dashboard_ai_context(request, dashboard_id: int):
 def get_dashboard_chat_bootstrap(request, dashboard_id: int):
     """Return deterministic UI bootstrap data for dashboard chat."""
     orguser: OrgUser = request.orguser
-    _ensure_dashboard_chat_feature_enabled(orguser.org)
+    if not get_all_feature_flags_for_org(orguser.org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
 
     try:
         dashboard = DashboardService.get_dashboard(dashboard_id, orguser.org)
     except DashboardNotFoundError as err:
         raise HttpError(404, "Dashboard not found") from err
 
-    return _serialize_dashboard_chat_bootstrap(dashboard)
+    dashboard_export = DashboardService.export_dashboard_context_for_dashboard(dashboard, dashboard.org)
+    return DashboardChatBootstrapResponse(
+        dashboard_id=dashboard.id,
+        suggested_prompts=build_dashboard_suggested_prompts(
+            dashboard_export=dashboard_export,
+        ),
+    )
 
 
 @dashboard_native_router.post(
@@ -208,7 +192,8 @@ def set_dashboard_chat_message_feedback(
 ):
     """Persist one locked thumbs-up/thumbs-down selection for an assistant answer."""
     orguser: OrgUser = request.orguser
-    _ensure_dashboard_chat_feature_enabled(orguser.org)
+    if not get_all_feature_flags_for_org(orguser.org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
 
     message = (
         DashboardChatMessage.objects.select_related("session")
@@ -254,15 +239,18 @@ def update_dashboard_ai_context(
 ):
     """Update dashboard-level AI context markdown for settings management."""
     orguser: OrgUser = request.orguser
-    _ensure_dashboard_chat_feature_enabled(orguser.org)
-    _ensure_dashboard_chat_consent_enabled(orguser.org)
+    if not get_all_feature_flags_for_org(orguser.org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
+    org_preferences = OrgPreferences.objects.filter(org=orguser.org).first()
+    if org_preferences is None or not org_preferences.ai_data_sharing_enabled:
+        raise HttpError(409, "Enable AI data sharing before updating dashboard AI context")
 
     try:
         dashboard = DashboardService.get_dashboard(dashboard_id, orguser.org)
     except DashboardNotFoundError as err:
         raise HttpError(404, "Dashboard not found") from err
 
-    context = _get_or_create_dashboard_ai_context(dashboard)
+    context, _ = DashboardAIContext.objects.get_or_create(dashboard=dashboard)
     context.markdown = payload.dashboard_context_markdown
     context.updated_by = orguser
     context.updated_at = timezone.now()

ddpui/api/org_preferences_api.py

@@ -8,14 +8,22 @@
 from ddpui.models.org_supersets import OrgSupersets
 from ddpui.models.org_plans import OrgPlans
 from ddpui.models.userpreferences import UserPreferences
+from ddpui.models.dashboard import Dashboard
 from ddpui.models.dashboard_chat import OrgAIContext
+from ddpui.core.dashboard_chat.metadata.build_service import (
+    DashboardChatMetadataBuildService,
+    summarize_dashboard_metadata_status,
+)
+from ddpui.celeryworkers.tasks import build_dashboard_chat_metadata_artifacts
 from ddpui.schemas.org_preferences_schema import (
     CreateOrgPreferencesSchema,
     UpdateLLMOptinSchema,
     UpdateDiscordNotificationsSchema,
     OrgAIDashboardChatSettingsResponse,
     UpdateOrgAIDashboardChatSchema,
     OrgAIDashboardChatStatusResponse,
+    OrgAIDashboardChatMetadataStatusResponse,
+    TriggerOrgAIDashboardChatMetadataBuildSchema,
 )
 from ddpui.core.notifications.notifications_functions import create_notification
 from ddpui.schemas.notifications_api_schemas import NotificationDataSchema
@@ -33,8 +41,11 @@
 
 orgpreference_router = Router()
 
+
 def _serialize_ai_dashboard_chat_settings(org, org_preferences, org_context):
     org_dbt = org.dbt
+    native_dashboards = list(Dashboard.objects.filter(org=org, dashboard_type="native").order_by("id"))
+    metadata_summary = summarize_dashboard_metadata_status(native_dashboards)
     return OrgAIDashboardChatSettingsResponse(
         feature_flag_enabled=get_all_feature_flags_for_org(org).get(
             "AI_DASHBOARD_CHAT",
@@ -53,19 +64,23 @@ def _serialize_ai_dashboard_chat_settings(org, org_preferences, org_context):
         else None,
         org_context_updated_at=org_context.updated_at,
         dbt_configured=org_dbt is not None,
-        ai_context_refreshed_at=org_dbt.vector_last_ingested_at if org_dbt else None,
+        metadata_last_built_at=metadata_summary["last_built_at"],
+        metadata_ready_dashboard_count=metadata_summary["ready_dashboard_count"],
+        metadata_total_dashboard_count=metadata_summary["total_dashboard_count"],
     )
 
 
 def _serialize_ai_dashboard_chat_status(org, org_preferences):
     org_dbt = org.dbt
+    native_dashboards = list(Dashboard.objects.filter(org=org, dashboard_type="native").order_by("id"))
+    metadata_summary = summarize_dashboard_metadata_status(native_dashboards)
     feature_flag_enabled = get_all_feature_flags_for_org(org).get(
         "AI_DASHBOARD_CHAT",
         False,
     )
     ai_data_sharing_enabled = bool(org_preferences.ai_data_sharing_enabled)
     dbt_configured = org_dbt is not None
-    vector_last_ingested_at = org_dbt.vector_last_ingested_at if org_dbt else None
+    last_built_at = metadata_summary["last_built_at"]
 
     return OrgAIDashboardChatStatusResponse(
         feature_flag_enabled=feature_flag_enabled,
@@ -74,13 +89,13 @@ def _serialize_ai_dashboard_chat_status(org, org_preferences):
             feature_flag_enabled
             and ai_data_sharing_enabled
             and dbt_configured
-            and vector_last_ingested_at is not None
+            and metadata_summary["ready_dashboard_count"] > 0
         ),
         dbt_configured=dbt_configured,
-        ai_context_refreshed_at=vector_last_ingested_at,
+        metadata_last_built_at=last_built_at,
+        metadata_ready_dashboard_count=metadata_summary["ready_dashboard_count"],
+        metadata_total_dashboard_count=metadata_summary["total_dashboard_count"],
     )
-
-
 @orgpreference_router.post("/")
 def create_org_preferences(request, payload: CreateOrgPreferencesSchema):
     """Creates preferences for an organization"""
@@ -268,6 +283,67 @@ def get_ai_dashboard_chat_status(request):
     }
 
 
+@orgpreference_router.get(
+    "/ai-dashboard-chat/metadata/status",
+    response=OrgAIDashboardChatMetadataStatusResponse,
+)
+@has_permission(["can_manage_org_settings"])
+def get_ai_dashboard_chat_metadata_status(request):
+    """Return per-dashboard metadata build status for the current org."""
+    orguser: OrgUser = request.orguser
+    org = orguser.org
+    if not get_all_feature_flags_for_org(org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
+    return OrgAIDashboardChatMetadataStatusResponse(
+        **summarize_dashboard_metadata_status(
+            list(Dashboard.objects.filter(org=org, dashboard_type="native").order_by("id"))
+        )
+    )
+
+
+@orgpreference_router.post(
+    "/ai-dashboard-chat/metadata/build",
+    response=OrgAIDashboardChatMetadataStatusResponse,
+)
+@has_permission(["can_manage_org_settings"])
+def build_ai_dashboard_chat_metadata(
+    request,
+    payload: TriggerOrgAIDashboardChatMetadataBuildSchema,
+):
+    """Manually build dashboard metadata artifacts for the current org."""
+    orguser: OrgUser = request.orguser
+    org = orguser.org
+
+    if not get_all_feature_flags_for_org(org).get("AI_DASHBOARD_CHAT", False):
+        raise HttpError(404, "Chat with dashboards is not enabled for this organization")
+    if org.dbt is None:
+        raise HttpError(409, "Configure dbt before building dashboard chat metadata")
+
+    dashboards = list(Dashboard.objects.filter(org=org, dashboard_type="native").order_by("id"))
+    if payload.dashboard_id is not None:
+        dashboards = [dashboard for dashboard in dashboards if dashboard.id == payload.dashboard_id]
+        if not dashboards:
+            raise HttpError(404, "Dashboard not found")
+
+    builder_model = str(payload.builder_model or "o4-mini")
+    build_service = DashboardChatMetadataBuildService()
+    build_service.mark_dashboards_building(
+        dashboards=dashboards,
+        builder_model=builder_model,
+    )
+    build_dashboard_chat_metadata_artifacts.delay(
+        org.id,
+        [dashboard.id for dashboard in dashboards],
+        builder_model,
+        orguser.id,
+    )
+    return OrgAIDashboardChatMetadataStatusResponse(
+        **summarize_dashboard_metadata_status(
+            list(Dashboard.objects.filter(org=org, dashboard_type="native").order_by("id"))
+        )
+    )
+
+
 @orgpreference_router.get("/toolinfo")
 def get_tools_versions(request):
     """get versions of the tools used in the system"""