Merge pull request #1331 from Kiln-AI/leonard/kil-601-posthog-analytics-for-chat

chore: add analytics to chat

Author: leonardmq

app/web_ui/src/lib/chat/chat_session_store.ts

@@ -1,4 +1,5 @@
 import { writable, get, type Readable } from "svelte/store"
+import posthog from "posthog-js"
 import {
   streamChat,
   chatGenerateId,
@@ -134,7 +135,7 @@ export function createChatSessionStore(
     }))
   }
 
-  function beginStreaming(text: string) {
+  function beginStreaming(text: string, isRetry = false) {
     removeErrors()
     const currentMessages = get(persisted).messages
     const traceId =
@@ -165,6 +166,16 @@ export function createChatSessionStore(
       lastSentAppState: currentAppState,
     }))
 
+    posthog.capture("chat_message_sent", {
+      is_new_conversation: !traceId && !isRetry,
+      message_length: text.length,
+      has_app_context_header: !!header,
+      message_count: currentMessages.length,
+    })
+    if (!traceId && !isRetry) {
+      posthog.capture("chat_conversation_started")
+    }
+
     combined.update((s) => ({
       ...s,
       toolExecuting: false,
@@ -284,6 +295,7 @@ export function createChatSessionStore(
 
   function stop(): void {
     if (abortController) {
+      posthog.capture("chat_stopped")
       abortController.abort()
     }
   }
@@ -299,12 +311,13 @@ export function createChatSessionStore(
       }
     }
     if (lastUserIdx === -1) return
+    posthog.capture("chat_retry")
     const userText = msgs[lastUserIdx].content ?? ""
     persisted.update((p) => ({
       ...p,
       messages: p.messages.slice(0, lastUserIdx),
     }))
-    beginStreaming(userText)
+    beginStreaming(userText, true)
   }
 
   function reset(): void {
@@ -390,8 +403,17 @@ export function createChatSessionStore(
     resolver(decisions)
   }
 
+  function toolNameForCallId(toolCallId: string): string | undefined {
+    return get(combined).toolApprovalWaiter?.payload.items.find(
+      (i) => i.toolCallId === toolCallId,
+    )?.toolName
+  }
+
   function applyToolApprovalRun(toolCallId: string): void {
     if (!get(combined).toolApprovalWaiter) return
+    posthog.capture("chat_tool_approval_run", {
+      tool_name: toolNameForCallId(toolCallId),
+    })
     combined.update((s) => ({
       ...s,
       toolApprovalPicks: { ...s.toolApprovalPicks, [toolCallId]: true },
@@ -401,6 +423,9 @@ export function createChatSessionStore(
 
   function applyToolApprovalSkip(toolCallId: string): void {
     if (!get(combined).toolApprovalWaiter) return
+    posthog.capture("chat_tool_approval_skip", {
+      tool_name: toolNameForCallId(toolCallId),
+    })
     combined.update((s) => ({
       ...s,
       toolApprovalPicks: { ...s.toolApprovalPicks, [toolCallId]: false },

app/web_ui/src/routes/(app)/chat/chat.svelte

@@ -1,6 +1,8 @@
 <script lang="ts">
   import { onMount, onDestroy, tick } from "svelte"
+  import { get } from "svelte/store"
   import { fly } from "svelte/transition"
+  import posthog from "posthog-js"
   import ChatCostDisclaimer from "./chat_cost_disclaimer.svelte"
   import type { ChatMessage, ChatMessagePart } from "$lib/chat/streaming_chat"
   import { CHAT_CLIENT_VERSION_TOO_OLD } from "$lib/error_codes"
@@ -430,6 +432,9 @@
   }
 
   export function newChat() {
+    posthog.capture("chat_new_chat_clicked", {
+      had_messages: get(store).messages.length > 0,
+    })
     store.reset()
   }
 

app/web_ui/src/routes/(app)/chat/chat_cost_disclaimer.svelte

@@ -1,4 +1,5 @@
 <script lang="ts">
+  import posthog from "posthog-js"
   import { chat_cost_disclaimer_acknowledged } from "$lib/stores"
   import Dialog from "$lib/ui/dialog.svelte"
 
@@ -9,6 +10,7 @@
 
   export function prompt(): Promise<boolean> {
     if (pendingPromise) return pendingPromise
+    posthog.capture("chat_cost_disclaimer_shown")
     pendingPromise = new Promise<boolean>((resolve) => {
       pendingResolve = resolve
       dialog.show()
@@ -17,6 +19,7 @@
   }
 
   function approve(): boolean {
+    posthog.capture("chat_cost_disclaimer_accepted")
     chat_cost_disclaimer_acknowledged.set(true)
     const resolve = pendingResolve
     pendingResolve = null
@@ -26,6 +29,8 @@
   }
 
   function dismiss() {
+    if (pendingResolve === null) return
+    posthog.capture("chat_cost_disclaimer_declined")
     const resolve = pendingResolve
     pendingResolve = null
     pendingPromise = null

app/web_ui/src/routes/(app)/chat/chat_history.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
   import { createEventDispatcher } from "svelte"
+  import posthog from "posthog-js"
   import { client } from "$lib/api_client"
   import { hydrateSessionFromSnapshot } from "$lib/chat/session_messages"
   import type { LoadedChatSessionDetail } from "$lib/chat/chat_history_apply"
@@ -48,7 +49,9 @@
   }
 
   export function open() {
-    historyDialog?.show()
+    if (!historyDialog) return
+    historyDialog.show()
+    posthog.capture("chat_history_opened")
     void loadSessionList()
   }
 
@@ -77,6 +80,9 @@
       const { messages, continuationTraceId } =
         hydrateSessionFromSnapshot(snapshot)
       dispatch("apply", { messages, continuationTraceId })
+      posthog.capture("chat_history_session_loaded", {
+        message_count: messages.length,
+      })
       close()
     } catch (e) {
       sessionsError = createKilnError(e)
@@ -97,6 +103,7 @@
         return
       }
       sessionRows = sessionRows.filter((r) => r.id !== sessionId)
+      posthog.capture("chat_history_session_deleted")
     } catch (e) {
       sessionsError = createKilnError(e)
     } finally {