feat(vfs): send typed workspace snapshot for append-only deltas

Build the workspace inventory from the primary db (fixes replica-lag staleness)
and emit it as a typed VfsSnapshotV1 `vfs` payload alongside the markdown, so the
mothership can diff it into append-only baseline/delta messages. Generate the TS
contract mirror from the Go-owned JSON schema (sync-vfs-snapshot-contract) and
sort connector types so diffs stay byte-stable.

Author: Sg312

apps/sim/lib/copilot/chat/payload.ts

@@ -6,6 +6,7 @@ import { isPaid } from '@/lib/billing/plan-helpers'
 import { getExposedIntegrationTools } from '@/lib/copilot/integration-tools'
 import { getToolEntry } from '@/lib/copilot/tool-executor/router'
 import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
+import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1'
 import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
 import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags'
 import { buildUserSkillTool } from '@/lib/mothership/skills'
@@ -33,6 +34,7 @@ interface BuildPayloadParams {
   prefetch?: boolean
   implicitFeedback?: string
   workspaceContext?: string
+  vfs?: VfsSnapshotV1
   userPermission?: string
   userTimezone?: string
   userMetadata?: {
@@ -366,6 +368,7 @@ export async function buildCopilotRequestPayload(
     ...(mothershipTools.length > 0 ? { mothershipTools } : {}),
     ...(commands && commands.length > 0 ? { commands } : {}),
     ...(params.workspaceContext ? { workspaceContext: params.workspaceContext } : {}),
+    ...(params.vfs ? { vfs: params.vfs } : {}),
     ...(params.userPermission ? { userPermission: params.userPermission } : {}),
     ...(params.userTimezone ? { userTimezone: params.userTimezone } : {}),
     ...(params.userMetadata &&

apps/sim/lib/copilot/chat/post.test.ts

@@ -17,7 +17,7 @@ const getUserEntityPermissions = permissionsMockFns.mockGetUserEntityPermissions
 
 const {
   getEffectiveDecryptedEnv,
-  generateWorkspaceContext,
+  generateWorkspaceSnapshot,
   processContextsServer,
   resolveActiveResourceContext,
   buildCopilotRequestPayload,
@@ -31,7 +31,7 @@ const {
   mockPublishStatusChanged,
 } = vi.hoisted(() => ({
   getEffectiveDecryptedEnv: vi.fn(),
-  generateWorkspaceContext: vi.fn(),
+  generateWorkspaceSnapshot: vi.fn(),
   processContextsServer: vi.fn(),
   resolveActiveResourceContext: vi.fn(),
   buildCopilotRequestPayload: vi.fn(),
@@ -56,7 +56,7 @@ vi.mock('@/lib/environment/utils', () => ({
 }))
 
 vi.mock('@/lib/copilot/chat/workspace-context', () => ({
-  generateWorkspaceContext,
+  generateWorkspaceSnapshot,
 }))
 
 vi.mock('@/lib/copilot/chat/process-contents', () => ({
@@ -142,7 +142,7 @@ describe('handleUnifiedChatPost', () => {
     })
     getUserEntityPermissions.mockResolvedValue('write')
     getEffectiveDecryptedEnv.mockResolvedValue({ API_KEY: 'secret' })
-    generateWorkspaceContext.mockResolvedValue('workspace context')
+    generateWorkspaceSnapshot.mockResolvedValue({ markdown: 'workspace context', snapshot: undefined })
     processContextsServer.mockResolvedValue([])
     resolveActiveResourceContext.mockResolvedValue(null)
     buildCopilotRequestPayload.mockImplementation(async (params: Record<string, unknown>) => params)
@@ -178,7 +178,7 @@ describe('handleUnifiedChatPost', () => {
     )
 
     expect(response.status).toBe(200)
-    expect(generateWorkspaceContext).toHaveBeenCalledWith('ws-1', 'user-1')
+    expect(generateWorkspaceSnapshot).toHaveBeenCalledWith('ws-1', 'user-1')
     expect(buildCopilotRequestPayload).toHaveBeenCalledWith(
       expect.objectContaining({
         model: 'claude-opus-4-8',

apps/sim/lib/copilot/chat/post.ts

@@ -22,7 +22,8 @@ import {
   resolveActiveResourceContext,
 } from '@/lib/copilot/chat/process-contents'
 import { finalizeAssistantTurn } from '@/lib/copilot/chat/terminal-state'
-import { generateWorkspaceContext } from '@/lib/copilot/chat/workspace-context'
+import { generateWorkspaceSnapshot } from '@/lib/copilot/chat/workspace-context'
+import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1'
 import { chatPubSub } from '@/lib/copilot/chat-status'
 import { COPILOT_REQUEST_MODES } from '@/lib/copilot/constants'
 import {
@@ -183,6 +184,7 @@ type UnifiedChatBranch =
         prefetch?: boolean
         implicitFeedback?: string
         workspaceContext?: string
+        vfs?: VfsSnapshotV1
       }) => Promise<Record<string, unknown>>
       buildExecutionContext: (params: {
         userId: string
@@ -210,6 +212,7 @@ type UnifiedChatBranch =
         userTimezone?: string
         userMetadata?: { name?: string; email?: string; timezone?: string }
         workspaceContext?: string
+        vfs?: VfsSnapshotV1
       }) => Promise<Record<string, unknown>>
       buildExecutionContext: (params: {
         userId: string
@@ -898,7 +901,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
         ? withCopilotSpan(
             TraceSpan.CopilotChatBuildWorkspaceContext,
             { [TraceAttr.WorkspaceId]: workspaceId },
-            () => generateWorkspaceContext(workspaceId, authenticatedUserId),
+            () => generateWorkspaceSnapshot(workspaceId, authenticatedUserId),
             activeOtelRoot.context
           )
         : Promise.resolve(undefined)
@@ -943,14 +946,19 @@ export async function handleUnifiedChatPost(req: NextRequest) {
         activeOtelRoot.context
       )
 
-      const [agentContexts, userPermission, workspaceContext, , executionContext] =
+      const [agentContexts, userPermission, workspaceSnapshot, , executionContext] =
         await Promise.all([
           agentContextsPromise,
           userPermissionPromise,
           workspaceContextPromise,
           persistUserMessagePromise,
           executionContextPromise,
         ])
+      // Both halves come from one primary-db fetch (workspace-context.ts):
+      // `workspaceContext` is the markdown transition fallback, `vfs` is the
+      // typed snapshot Go diffs into baseline+delta messages.
+      const workspaceContext = workspaceSnapshot?.markdown
+      const vfs = workspaceSnapshot?.snapshot
 
       executionContext.userPermission = userPermission ?? undefined
 
@@ -987,6 +995,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
                 prefetch: body.prefetch,
                 implicitFeedback: body.implicitFeedback,
                 workspaceContext,
+                vfs,
               })
             : branch.buildPayload({
                 message: body.message,
@@ -999,6 +1008,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
                 userTimezone: body.userTimezone,
                 userMetadata,
                 workspaceContext,
+                vfs,
               }),
         activeOtelRoot.context
       )

apps/sim/lib/copilot/chat/workspace-context.ts

@@ -1,4 +1,9 @@
-import { dbReplica } from '@sim/db'
+import { db } from '@sim/db'
+import type {
+  VfsSnapshotV1,
+  VfsSnapshotV1Job,
+  VfsSnapshotV1Workflow,
+} from '@/lib/copilot/generated/vfs-snapshot-v1'
 import {
   knowledgeBase,
   knowledgeConnector,
@@ -311,15 +316,20 @@ export function buildWorkspaceContextMd(data: WorkspaceMdData): string {
  * discovery rules; the LLM reads dynamic workspace state from VFS files.
  * The LLM never writes this file directly.
  */
-export async function generateWorkspaceContext(
+// Fetch + assemble the workspace inventory data once, from the PRIMARY db
+// (read-your-writes: a just-edited workflow is visible immediately, so the
+// injected snapshot can't lag behind a `glob`). Both the markdown inventory and
+// the typed VFS snapshot are built from this single fetch. Returns null when the
+// workspace is unavailable or a fetch fails.
+async function buildWorkspaceMdData(
   workspaceId: string,
   userId: string
-): Promise<string> {
+): Promise<WorkspaceMdData | null> {
   try {
     await assertActiveWorkspaceAccess(workspaceId, userId)
     const wsRow = await getWorkspaceWithOwner(workspaceId)
     if (!wsRow) {
-      return '## Workspace\n(unavailable)'
+      return null
     }
 
     const [
@@ -337,7 +347,7 @@ export async function generateWorkspaceContext(
     ] = await Promise.all([
       getUsersWithPermissions(workspaceId),
 
-      dbReplica
+      db
         .select({
           id: workflow.id,
           name: workflow.name,
@@ -349,7 +359,7 @@ export async function generateWorkspaceContext(
         .from(workflow)
         .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt))),
 
-      dbReplica
+      db
         .select({
           id: workflowFolder.id,
           name: workflowFolder.name,
@@ -358,7 +368,7 @@ export async function generateWorkspaceContext(
         .from(workflowFolder)
         .where(and(eq(workflowFolder.workspaceId, workspaceId), isNull(workflowFolder.archivedAt))),
 
-      dbReplica
+      db
         .select({
           id: knowledgeBase.id,
           name: knowledgeBase.name,
@@ -367,7 +377,7 @@ export async function generateWorkspaceContext(
         .from(knowledgeBase)
         .where(and(eq(knowledgeBase.workspaceId, workspaceId), isNull(knowledgeBase.deletedAt))),
 
-      dbReplica
+      db
         .select({
           id: userTableDefinitions.id,
           name: userTableDefinitions.name,
@@ -387,7 +397,7 @@ export async function generateWorkspaceContext(
 
       listCustomTools({ userId, workspaceId }),
 
-      dbReplica
+      db
         .select({
           id: mcpServers.id,
           name: mcpServers.name,
@@ -399,7 +409,7 @@ export async function generateWorkspaceContext(
 
       listSkills({ workspaceId, includeBuiltins: false }),
 
-      dbReplica
+      db
         .select({
           id: workflowSchedule.id,
           jobTitle: workflowSchedule.jobTitle,
@@ -422,7 +432,7 @@ export async function generateWorkspaceContext(
     const kbIds = kbs.map((kb) => kb.id)
     const connectorRows =
       kbIds.length > 0
-        ? await dbReplica
+        ? await db
             .select({
               knowledgeBaseId: knowledgeConnector.knowledgeBaseId,
               connectorType: knowledgeConnector.connectorType,
@@ -459,7 +469,7 @@ export async function generateWorkspaceContext(
       return path
     }
 
-    return buildWorkspaceMd({
+    return {
       workspace: wsRow,
       members,
       workflows: workflows.map((wf) => ({
@@ -468,7 +478,11 @@ export async function generateWorkspaceContext(
       })),
       knowledgeBases: kbs.map((kb) => ({
         ...kb,
-        connectorTypes: connectorTypesByKb.get(kb.id),
+        // Sort connector types so the snapshot is order-stable: the DB query has
+        // no ORDER BY, and the Go delta engine compares item JSON byte-wise, so
+        // an unsorted (but unchanged) list would emit a spurious "modified"
+        // delta and needlessly bust the prompt cache.
+        connectorTypes: connectorTypesByKb.get(kb.id)?.sort(stableCompare),
       })),
       tables: tables.map((t) => ({ id: t.id, name: t.name, description: t.description })),
       files: files.map((f) => ({
@@ -499,13 +513,128 @@ export async function generateWorkspaceContext(
           lifecycle: j.lifecycle,
           sourceTaskName: j.sourceTaskName,
         })),
-    })
+    }
   } catch (err) {
-    logger.error('Failed to generate workspace context', {
+    logger.error('Failed to build workspace data', {
       workspaceId,
       error: toError(err).message,
     })
-    return '## Workspace\n(unavailable)\n\n## Workflows\n(unavailable)\n\n## Knowledge Bases\n(unavailable)\n\n## Tables\n(unavailable)\n\n## Files\n(unavailable)\n\n## Connected Integrations\n(unavailable)'
+    return null
+  }
+}
+
+const WORKSPACE_CONTEXT_UNAVAILABLE_MD =
+  '## Workspace\n(unavailable)\n\n## Workflows\n(unavailable)\n\n## Knowledge Bases\n(unavailable)\n\n## Tables\n(unavailable)\n\n## Files\n(unavailable)\n\n## Connected Integrations\n(unavailable)'
+
+/**
+ * Generate WORKSPACE.md markdown from current DB state (primary db). The LLM
+ * reads dynamic workspace state from VFS files; it never writes this file.
+ */
+export async function generateWorkspaceContext(
+  workspaceId: string,
+  userId: string
+): Promise<string> {
+  const data = await buildWorkspaceMdData(workspaceId, userId)
+  return data ? buildWorkspaceMd(data) : WORKSPACE_CONTEXT_UNAVAILABLE_MD
+}
+
+/**
+ * Build BOTH the markdown inventory and the typed VFS snapshot from a single
+ * primary-db fetch. The snapshot is the structured form Go diffs into
+ * baseline+delta messages; the markdown is the transition fallback. Returns null
+ * when the workspace is unavailable.
+ */
+export async function generateWorkspaceSnapshot(
+  workspaceId: string,
+  userId: string
+): Promise<{ markdown: string; snapshot: VfsSnapshotV1 } | null> {
+  const data = await buildWorkspaceMdData(workspaceId, userId)
+  if (!data) return null
+  return { markdown: buildWorkspaceMd(data), snapshot: buildVfsSnapshot(data) }
+}
+
+/**
+ * Map the workspace inventory data to the typed VFS snapshot contract. Pure;
+ * mirrors buildWorkspaceMd's field selection. Resource order is irrelevant — Go
+ * diffs by stable id, not position.
+ */
+export function buildVfsSnapshot(data: WorkspaceMdData): VfsSnapshotV1 {
+  const workflows: VfsSnapshotV1Workflow[] = data.workflows.map((wf) => ({
+    id: wf.id,
+    name: wf.name,
+    path: canonicalWorkflowVfsDir({ name: wf.name, folderPath: wf.folderPath }),
+    ...(wf.description ? { description: wf.description } : {}),
+    ...(wf.isDeployed ? { isDeployed: true } : {}),
+    ...(wf.folderPath ? { folderPath: wf.folderPath } : {}),
+  }))
+  const jobs: VfsSnapshotV1Job[] = (data.jobs ?? [])
+    .filter((j) => j.status !== 'completed')
+    .map((j) => ({
+      id: j.id,
+      ...(j.title ? { title: j.title } : {}),
+      ...(j.prompt ? { prompt: j.prompt } : {}),
+      ...(j.cronExpression ? { cronExpression: j.cronExpression } : {}),
+      ...(j.status ? { status: j.status } : {}),
+      ...(j.lifecycle ? { lifecycle: j.lifecycle } : {}),
+      ...(j.sourceTaskName ? { sourceTaskName: j.sourceTaskName } : {}),
+    }))
+  return {
+    ...(data.workspace
+      ? {
+          workspace: {
+            id: data.workspace.id,
+            name: data.workspace.name,
+            ...(data.workspace.ownerId ? { ownerId: data.workspace.ownerId } : {}),
+          },
+        }
+      : {}),
+    members: data.members.map((m) => ({
+      ...(m.name ? { name: m.name } : {}),
+      email: m.email,
+      ...(m.permissionType ? { permissionType: m.permissionType } : {}),
+    })),
+    workflows,
+    knowledgeBases: data.knowledgeBases.map((kb) => ({
+      id: kb.id,
+      name: kb.name,
+      ...(kb.description ? { description: kb.description } : {}),
+      ...(kb.connectorTypes && kb.connectorTypes.length > 0
+        ? { connectorTypes: kb.connectorTypes }
+        : {}),
+    })),
+    tables: data.tables.map((t) => ({
+      id: t.id,
+      name: t.name,
+      ...(t.description ? { description: t.description } : {}),
+    })),
+    files: data.files.map((f) => ({
+      id: f.id,
+      name: f.name,
+      path: canonicalWorkspaceFilePath({ folderPath: f.folderPath, name: f.name }),
+      ...(f.type ? { type: f.type } : {}),
+      ...(f.size ? { size: f.size } : {}),
+      ...(f.folderPath ? { folderPath: f.folderPath } : {}),
+    })),
+    integrations: data.oauthIntegrations.map((c) => ({
+      id: c.id,
+      providerId: c.providerId,
+      ...(c.displayName ? { displayName: c.displayName } : {}),
+      ...(c.role ? { role: c.role } : {}),
+    })),
+    envVars: data.envVariables,
+    customTools: (data.customTools ?? []).map((t) => ({ id: t.id, name: t.name })),
+    mcpServers: (data.mcpServers ?? []).map((s) => ({
+      id: s.id,
+      name: s.name,
+      ...(s.url ? { url: s.url } : {}),
+      ...(s.enabled ? { enabled: true } : {}),
+    })),
+    skills: (data.skills ?? []).map((s) => ({
+      id: s.id,
+      name: s.name,
+      ...(s.description ? { description: s.description } : {}),
+    })),
+    jobs,
   }
 }