[pull] main from tldraw:main

apps/dotcom/client/src/fairy/FairyApp.tsx

@@ -1,5 +1,6 @@
 import { MAX_FAIRY_COUNT } from '@tldraw/dotcom-shared'
 import {
+	ChatHistoryItem,
 	FAIRY_VARIANTS,
 	FairyConfig,
 	FairyVariantType,
@@ -20,6 +21,14 @@ import { FairyThrowTool } from './FairyThrowTool'
 import { getRandomFairyName } from './getRandomFairyName'
 import { getRandomFairyPersonality } from './getRandomFairyPersonality'
 
+function stripDiffFromChatItem(item: ChatHistoryItem): ChatHistoryItem {
+	if (item.type === 'action') {
+		const { diff: _diff, ...rest } = item
+		return rest as ChatHistoryItem
+	}
+	return item
+}
+
 export function FairyApp({
 	setAgents,
 	fileId,
@@ -209,7 +218,12 @@ export function FairyApp({
 			const fairyState: PersistedFairyState = {
 				agents: agentsRef.current.reduce(
 					(acc, agent) => {
-						acc[agent.id] = agent.serializeState()
+						const agentState = agent.serializeState()
+						// Strip diff field from chat history before sending
+						if (agentState.chatHistory) {
+							agentState.chatHistory = agentState.chatHistory.map(stripDiffFromChatItem)
+						}
+						acc[agent.id] = agentState
 						return acc
 					},
 					{} as Record<string, PersistedFairyAgentState>
@@ -247,6 +261,77 @@ export function FairyApp({
 		}
 	}, [app, fairyConfigs, fileId])
 
+	// Append chat messages to database
+	useEffect(() => {
+		if (!app || agentsRef.current.length === 0 || !fileId) return
+
+		const sentMessageIds = new Map<string, Set<string>>() // agentId -> Set of sent IDs
+
+		// Initialize sent message IDs for all agents
+		agentsRef.current.forEach((agent) => {
+			const chatHistory = agent.$chatHistory.get()
+			const sent = new Set<string>()
+
+			chatHistory.forEach((item) => {
+				// Skip legacy messages without IDs
+				if (!item.id) return
+
+				// Skip incomplete actions (mirror the sending logic)
+				if (item.type === 'action' && !item.action.complete) return
+
+				// Mark complete messages as sent
+				sent.add(item.id)
+			})
+
+			sentMessageIds.set(agent.id, sent)
+		})
+
+		const appendMessages = throttle(() => {
+			// Don't append if we're currently loading state
+			if (isLoadingStateRef.current) return
+
+			const allMessagesToAppend: ChatHistoryItem[] = []
+
+			agentsRef.current.forEach((agent) => {
+				const chatHistory = agent.$chatHistory.get()
+				const sent = sentMessageIds.get(agent.id) || new Set()
+
+				chatHistory.forEach((item) => {
+					// Skip legacy messages without IDs
+					if (!item.id) return
+
+					// Skip incomplete actions
+					if (item.type === 'action' && !item.action.complete) return
+
+					// Skip if already sent
+					if (sent.has(item.id)) return
+
+					// Strip diff field from action items before sending
+					allMessagesToAppend.push(stripDiffFromChatItem(item))
+					sent.add(item.id)
+				})
+			})
+
+			if (allMessagesToAppend.length > 0) {
+				app.appendFairyChatMessages(fileId, allMessagesToAppend)
+			}
+		}, 2000) // Append maximum every 2 seconds
+
+		const fairyCleanupFns: (() => void)[] = []
+		agentsRef.current.forEach((agent) => {
+			const cleanup = react(`${agent.id} chat history`, () => {
+				agent.$chatHistory.get()
+				appendMessages()
+			})
+			fairyCleanupFns.push(cleanup)
+		})
+
+		return () => {
+			appendMessages.flush()
+			fairyCleanupFns.forEach((cleanup) => cleanup())
+		}
+	}, [app, fairyConfigs, fileId])
+
 	return null
 }
 

apps/dotcom/client/src/fairy/actions/EndCurrentProjectActionUtil.ts

@@ -1,4 +1,5 @@
 import { EndCurrentProjectAction, Streaming } from '@tldraw/fairy-shared'
+import { uniqueId } from 'tldraw'
 import { deleteProjectAndAssociatedTasks } from '../FairyProjects'
 import { getFairyTasksByProjectId } from '../FairyTaskList'
 import { AgentHelpers } from '../fairy-agent/agent/AgentHelpers'
@@ -42,6 +43,7 @@ export class EndCurrentProjectActionUtil extends AgentActionUtil<EndCurrentProje
 				memberAgent.$chatHistory.update((prev) => [
 					...prev,
 					{
+						id: uniqueId(),
 						type: 'memory-transition',
 						memoryLevel: 'fairy',
 						agentFacingMessage: `I led and completed the "${project.title}" project with ${otherMemberIds.length} other fairy(s): ${otherMemberIds.join(', ')}`,
@@ -59,6 +61,7 @@ export class EndCurrentProjectActionUtil extends AgentActionUtil<EndCurrentProje
 				memberAgent.$chatHistory.update((prev) => [
 					...prev,
 					{
+						id: uniqueId(),
 						type: 'memory-transition',
 						memoryLevel: 'fairy',
 						agentFacingMessage: `I completed ${count} ${taskWord} as part of the "${project.title}" project, with ${otherMemberIds.length} other fairy(s): ${otherMemberIds.join(', ')}`,

apps/dotcom/client/src/fairy/actions/EndDuoProjectActionUtil.ts

@@ -1,4 +1,5 @@
 import { EndDuoProjectAction, Streaming } from '@tldraw/fairy-shared'
+import { uniqueId } from 'tldraw'
 import { deleteProjectAndAssociatedTasks } from '../FairyProjects'
 import { getFairyTasksByProjectId } from '../FairyTaskList'
 import { AgentHelpers } from '../fairy-agent/agent/AgentHelpers'
@@ -40,6 +41,7 @@ export class EndDuoProjectActionUtil extends AgentActionUtil<EndDuoProjectAction
 				memberAgent.$chatHistory.update((prev) => [
 					...prev,
 					{
+						id: uniqueId(),
 						type: 'memory-transition',
 						memoryLevel: 'fairy',
 						agentFacingMessage: `I completed ${count} ${taskWord} as part of the "${project.title}" project with my partner.`,
@@ -57,6 +59,7 @@ export class EndDuoProjectActionUtil extends AgentActionUtil<EndDuoProjectAction
 					memberAgent.$chatHistory.update((prev) => [
 						...prev,
 						{
+							id: uniqueId(),
 							type: 'memory-transition',
 							memoryLevel: 'fairy',
 							agentFacingMessage: `I completed ${count} ${taskWord} as part of the "${project.title}" project with my partner.`,

apps/dotcom/client/src/fairy/actions/MarkDroneTaskDoneActionUtil.ts

@@ -1,4 +1,5 @@
 import { MarkDroneTaskDoneAction, Streaming } from '@tldraw/fairy-shared'
+import { uniqueId } from 'tldraw'
 import { AgentHelpers } from '../fairy-agent/agent/AgentHelpers'
 import { setFairyTaskStatusAndNotifyCompletion } from '../FairyTaskList'
 import { AgentActionUtil } from './AgentActionUtil'
@@ -27,6 +28,7 @@ export class MarkDroneTaskDoneActionUtil extends AgentActionUtil<MarkDroneTaskDo
 		this.agent.$chatHistory.update((prev) => [
 			...prev,
 			{
+				id: uniqueId(),
 				type: 'memory-transition',
 				memoryLevel: 'project',
 				agentFacingMessage: `I just finished the task.\nID: "${currentTaskId}"\nTitle: "${currentTask.title}"\nDescription: "${currentTask.text}".`,

apps/dotcom/client/src/fairy/actions/MarkDuoTaskDoneActionUtil.ts

@@ -1,4 +1,5 @@
 import { MarkDuoTaskDoneAction, Streaming } from '@tldraw/fairy-shared'
+import { uniqueId } from 'tldraw'
 import { AgentHelpers } from '../fairy-agent/agent/AgentHelpers'
 import { setFairyTaskStatusAndNotifyCompletion } from '../FairyTaskList'
 import { AgentActionUtil } from './AgentActionUtil'
@@ -27,6 +28,7 @@ export class MarkDuoTaskDoneActionUtil extends AgentActionUtil<MarkDuoTaskDoneAc
 		this.agent.$chatHistory.update((prev) => [
 			...prev,
 			{
+				id: uniqueId(),
 				type: 'memory-transition',
 				memoryLevel: 'project',
 				agentFacingMessage: `I just finished the task.\nID: "${currentTaskId}"\nTitle: "${currentTask.title}"\nDescription: "${currentTask.text}".`,

apps/dotcom/client/src/fairy/actions/MarkSoloTaskDoneActionUtil.ts

@@ -1,4 +1,5 @@
 import { MarkSoloTaskDoneAction, Streaming } from '@tldraw/fairy-shared'
+import { uniqueId } from 'tldraw'
 import { AgentHelpers } from '../fairy-agent/agent/AgentHelpers'
 import { setFairyTaskStatusAndNotifyCompletion } from '../FairyTaskList'
 import { AgentActionUtil } from './AgentActionUtil'
@@ -27,6 +28,7 @@ export class MarkSoloTaskDoneActionUtil extends AgentActionUtil<MarkSoloTaskDone
 		this.agent.$chatHistory.update((prev) => [
 			...prev,
 			{
+				id: uniqueId(),
 				type: 'memory-transition',
 				memoryLevel: 'fairy',
 				agentFacingMessage: `I just finished the task.\nID: "${currentTaskId}"\nTitle: "${currentTask.title}"\nDescription: "${currentTask.text}".`,

apps/dotcom/client/src/fairy/fairy-agent/agent/FairyAgent.ts

@@ -680,6 +680,7 @@ export class FairyAgent {
 		this.$chatHistory.update((prev) => [
 			...prev,
 			{
+				id: uniqueId(),
 				type: 'continuation',
 				data: resolvedData,
 				memoryLevel: eventualModeDefinition.memoryLevel,
@@ -1023,6 +1024,7 @@ export class FairyAgent {
 		// Add the action to chat history
 		if (util.savesToHistory()) {
 			const historyItem: ChatHistoryItem = {
+				id: uniqueId(),
 				type: 'action',
 				action,
 				diff,
@@ -1558,6 +1560,7 @@ function requestAgentActions({ agent, request }: { agent: FairyAgent; request: A
 	const userFacingMessage = request.userMessages.join('\n')
 
 	const promptHistoryItem: ChatHistoryPromptItem = {
+		id: uniqueId(),
 		type: 'prompt',
 		promptSource: request.source,
 		agentFacingMessage,

apps/dotcom/client/src/tla/app/TldrawApp.ts

@@ -28,6 +28,7 @@ import {
 	parseFlags,
 	schema as zeroSchema,
 } from '@tldraw/dotcom-shared'
+import { ChatHistoryItem } from '@tldraw/fairy-shared'
 import {
 	Result,
 	assert,
@@ -822,6 +823,13 @@ export class TldrawApp {
 		})
 	}
 
+	appendFairyChatMessages(fileId: string, messages: ChatHistoryItem[]) {
+		this.z.mutate.file_state.appendFairyChatMessage({
+			fileId,
+			messages,
+		})
+	}
+
 	onFileEdit(fileId: string) {
 		this.updateFileState(fileId, { lastEditAt: Date.now() })
 	}

apps/dotcom/client/src/tla/components/TlaEditor/TlaEditor.tsx

@@ -11,6 +11,7 @@ import {
 	TLSessionStateSnapshot,
 	TLUiDialogsContextType,
 	Tldraw,
+	TldrawOverlays,
 	TldrawUiMenuItem,
 	createSessionStateSnapshotSignal,
 	getDefaultUserPresence,
@@ -305,6 +306,7 @@ function TlaEditorInner({ fileSlug, deepLinks }: TlaEditorProps) {
 			...components,
 			Overlays: () => (
 				<>
+					<TldrawOverlays />
 					{canShowFairies ? (
 						<Suspense fallback={<div />}>
 							<FairyVision agents={agents} />

apps/dotcom/client/src/utils/csp.ts

@@ -36,6 +36,8 @@ export const cspDirectives: { [key: string]: string[] } = {
 		'https://fonts.googleapis.com',
 		// paddle
 		'https://*.paddle.com',
+		// profitwell (loaded by paddle)
+		'https://public.profitwell.com',
 	],
 	'font-src': [`'self'`, `https://fonts.googleapis.com`, `https://fonts.gstatic.com`, 'data:'],
 	'frame-src': [
@@ -62,6 +64,8 @@ export const cspDirectives: { [key: string]: string[] } = {
 		'https://static.reo.dev',
 		// paddle
 		'https://*.paddle.com',
+		// profitwell (loaded by paddle)
+		'https://public.profitwell.com',
 	],
 	'worker-src': [`'self'`, `blob:`],
 	'style-src': [`'self'`, `'unsafe-inline'`, `https://fonts.googleapis.com`],

apps/dotcom/zero-cache/migrations/027_add_fairy_messages.sql

@@ -0,0 +1,13 @@
+
+CREATE TABLE file_fairy_messages (
+  "id" VARCHAR PRIMARY KEY,
+  "fileId" VARCHAR NOT NULL,
+  "userId" VARCHAR NOT NULL,
+  "message" VARCHAR NOT NULL,
+  "createdAt" BIGINT NOT NULL,
+  "updatedAt" BIGINT NOT NULL,
+  CONSTRAINT file_fairy_messages_file_id_fkey FOREIGN KEY ("fileId") REFERENCES public."file"("id") ON DELETE CASCADE,
+  CONSTRAINT file_fairy_messages_user_id_fkey FOREIGN KEY ("userId") REFERENCES public."user"("id") ON DELETE CASCADE
+);
+
+CREATE INDEX file_fairy_messages_file_user_idx ON file_fairy_messages("fileId", "userId");

packages/dotcom-shared/src/mutators.ts

@@ -384,7 +384,80 @@ export function createMutators(userId: string) {
 				await tx.mutate.file_state.upsert(fileState)
 			},
 			updateFairies: async (tx, { fileId, fairyState }: { fileId: string; fairyState: string }) => {
-				await tx.mutate.file_fairies.upsert({ fileId, userId, fairyState })
+				if (tx.location !== 'server') {
+					await tx.mutate.file_fairies.upsert({ fileId, userId, fairyState })
+					return
+				}
+
+				const MAX_SIZE_PER_AGENT = 300 * 1024 // 300kb
+				const TRUNCATE_THRESHOLD = 350 * 1024 // 350kb
+				let truncatedState = fairyState
+
+				try {
+					const state = JSON.parse(fairyState)
+					if (state.agents && typeof state.agents === 'object') {
+						for (const aid in state.agents) {
+							const agent = state.agents[aid]
+							if (agent.chatHistory && Array.isArray(agent.chatHistory)) {
+								const agentHistoryStr = JSON.stringify(agent.chatHistory)
+								const originalSize = agentHistoryStr.length
+								if (originalSize > TRUNCATE_THRESHOLD) {
+									const originalCount = agent.chatHistory.length
+									// Estimate how many messages to keep based on average size
+									const avgSize = originalSize / originalCount
+									const estimatedKeep = Math.max(1, Math.floor(MAX_SIZE_PER_AGENT / avgSize))
+
+									// Keep estimated number (at least 1 message)
+									agent.chatHistory = agent.chatHistory.slice(-estimatedKeep)
+								}
+							}
+						}
+						truncatedState = JSON.stringify(state)
+					}
+				} finally {
+					await tx.mutate.file_fairies.upsert({ fileId, userId, fairyState: truncatedState })
+				}
+			},
+			appendFairyChatMessage: async (
+				tx,
+				{ fileId, messages }: { fileId: string; messages: any[] }
+			) => {
+				if (messages.length === 0) {
+					return
+				}
+				// Only insert on the backend
+				if (tx.location !== 'server') return
+
+				try {
+					let now = Date.now()
+
+					// Build batch upsert
+					const values: any[] = []
+					const placeholders: string[] = []
+					let paramIndex = 1
+
+					messages.forEach((item) => {
+						const message = JSON.stringify(item)
+						const id = item.id
+
+						placeholders.push(
+							`($${paramIndex}, $${paramIndex + 1}, $${paramIndex + 2}, $${paramIndex + 3}, $${paramIndex + 4}, $${paramIndex + 5})`
+						)
+						values.push(id, fileId, userId, message, now, now)
+						paramIndex += 6
+						now++ // Increment to preserve order
+					})
+
+					await tx.dbTransaction.query(
+						`INSERT INTO file_fairy_messages ("id", "fileId", "userId", "message", "createdAt", "updatedAt")
+						VALUES ${placeholders.join(', ')}
+						ON CONFLICT ("id")
+						DO UPDATE SET "message" = EXCLUDED."message", "updatedAt" = EXCLUDED."updatedAt"`,
+						values
+					)
+				} catch (e) {
+					console.error('Failed to append fairy chat messages:', e)
+				}
 			},
 		},