improvement(tables): versioned CSV snapshot cache for table mounts + parallel multipart uploader

apps/sim/background/table-export.ts

@@ -3,9 +3,9 @@ import { runTableExport, type TableExportPayload } from '@/lib/table/export-runn
 
 /**
  * Trigger.dev wrapper around `runTableExport`. Retry-safe: a retried attempt regenerates the file
- * from scratch (failures clean up their partial upload), and the `table_jobs` ownership gate
- * stops a run that lost the job. `medium-1x` — the serialized file is buffered in memory before
- * the single-shot storage upload (~hundreds of MB worst case for enterprise 1M-row tables).
+ * from scratch (failures abort/clean up their partial upload), and the `table_jobs` ownership gate
+ * stops a run that lost the job. The file streams to storage in bounded multipart chunks (no longer
+ * buffered whole), so `medium-1x` is now headroom rather than a hard requirement.
  */
 export const tableExportTask = task({
   id: 'table-export',

apps/sim/lib/copilot/tools/handlers/function-execute.test.ts

@@ -0,0 +1,153 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockIsFeatureEnabled,
+  mockGetTableById,
+  mockListTables,
+  mockQueryRows,
+  mockGetOrCreateTableSnapshot,
+  mockDownloadFile,
+  mockExecuteTool,
+} = vi.hoisted(() => ({
+  mockIsFeatureEnabled: vi.fn(),
+  mockGetTableById: vi.fn(),
+  mockListTables: vi.fn(),
+  mockQueryRows: vi.fn(),
+  mockGetOrCreateTableSnapshot: vi.fn(),
+  mockDownloadFile: vi.fn(),
+  mockExecuteTool: vi.fn(),
+}))
+
+vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mockIsFeatureEnabled }))
+vi.mock('@/lib/table/service', () => ({
+  getTableById: mockGetTableById,
+  listTables: mockListTables,
+}))
+vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows }))
+vi.mock('@/lib/table/snapshot-cache', () => ({
+  getOrCreateTableSnapshot: mockGetOrCreateTableSnapshot,
+}))
+vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile }))
+vi.mock('@/tools', () => ({ executeTool: mockExecuteTool }))
+// Workspace-file + VFS surfaces are unused on the tables-only path; stub to avoid heavy loads.
+vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
+  fetchWorkspaceFileBuffer: vi.fn(),
+  findWorkspaceFileRecord: vi.fn(),
+  getSandboxWorkspaceFilePath: vi.fn(),
+  listWorkspaceFiles: vi.fn(),
+}))
+vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => ({
+  listWorkspaceFileFolders: vi.fn(),
+}))
+vi.mock('@/lib/copilot/vfs/path-utils', () => ({
+  decodeVfsPathSegments: (p: string) => p.split('/'),
+  encodeVfsPathSegments: (s: string[]) => s.join('/'),
+}))
+vi.mock('@/lib/copilot/vfs/workflow-alias-resolver', () => ({
+  resolveWorkflowAliasForWorkspace: vi.fn().mockResolvedValue(null),
+}))
+vi.mock('@/lib/copilot/vfs/workflow-aliases', () => ({
+  isPlanAliasPath: () => false,
+  workflowAliasSandboxPath: (p: string) => p,
+}))
+
+import { executeFunctionExecute } from '@/lib/copilot/tools/handlers/function-execute'
+
+const table = {
+  id: 'tbl_1',
+  workspaceId: 'ws_1',
+  rowCount: 1000,
+  schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] },
+}
+
+const context = { workspaceId: 'ws_1', userId: 'u1' }
+
+function mountedFiles() {
+  const params = mockExecuteTool.mock.calls[0][1] as {
+    _sandboxFiles?: Array<{ path: string; content: string }>
+  }
+  return params._sandboxFiles ?? []
+}
+
+describe('executeFunctionExecute table mounts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockExecuteTool.mockResolvedValue({ success: true })
+    mockGetTableById.mockResolvedValue(table)
+    mockIsFeatureEnabled.mockResolvedValue(false)
+    mockQueryRows.mockResolvedValue({ rows: [{ data: { name: 'Ada' } }] })
+  })
+
+  it('flag OFF: drains the table inline via queryRows (existing path)', async () => {
+    await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+
+    expect(mockQueryRows).toHaveBeenCalledTimes(1)
+    expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled()
+    const files = mountedFiles()
+    expect(files[0].path).toBe('/home/user/tables/tbl_1.csv')
+    expect(files[0].content).toBe('name\nAda')
+  })
+
+  it('flag ON + large table: mounts by reference from the snapshot, no row drain', async () => {
+    mockIsFeatureEnabled.mockImplementation((flag: string) =>
+      Promise.resolve(flag === 'table-snapshot-cache')
+    )
+    mockGetOrCreateTableSnapshot.mockResolvedValue({
+      key: 'table-snapshots/ws_1/tbl_1/v5.csv',
+      size: 9,
+      version: 5,
+    })
+    mockDownloadFile.mockResolvedValue(Buffer.from('name\nAda\n'))
+
+    await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+
+    expect(mockGetOrCreateTableSnapshot).toHaveBeenCalledTimes(1)
+    expect(mockQueryRows).not.toHaveBeenCalled()
+    expect(mockDownloadFile).toHaveBeenCalledWith(
+      expect.objectContaining({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', context: 'execution' })
+    )
+    const files = mountedFiles()
+    expect(files[0].path).toBe('/home/user/tables/tbl_1.csv')
+    expect(files[0].content).toBe('name\nAda\n')
+  })
+
+  it('flag ON but small table stays on the inline path', async () => {
+    mockIsFeatureEnabled.mockImplementation((flag: string) =>
+      Promise.resolve(flag === 'table-snapshot-cache')
+    )
+    mockGetTableById.mockResolvedValue({ ...table, rowCount: 10 })
+
+    await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+
+    expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled()
+    expect(mockQueryRows).toHaveBeenCalledTimes(1)
+  })
+
+  it('flag ON: throws when the snapshot exceeds the per-file mount limit', async () => {
+    mockIsFeatureEnabled.mockImplementation((flag: string) =>
+      Promise.resolve(flag === 'table-snapshot-cache')
+    )
+    mockGetOrCreateTableSnapshot.mockResolvedValue({
+      key: 'table-snapshots/ws_1/tbl_1/v5.csv',
+      size: 20 * 1024 * 1024,
+      version: 5,
+    })
+
+    await expect(
+      executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+    ).rejects.toThrow(/per-file mount limit/)
+    expect(mockDownloadFile).not.toHaveBeenCalled()
+  })
+
+  it('rejects a table that belongs to another workspace (tenant isolation)', async () => {
+    mockGetTableById.mockResolvedValue({ ...table, workspaceId: 'ws_2' })
+
+    await expect(
+      executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+    ).rejects.toThrow(/Input table not found/)
+    expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/copilot/tools/handlers/function-execute.ts

@@ -5,13 +5,15 @@ import { isPlanAliasPath, workflowAliasSandboxPath } from '@/lib/copilot/vfs/wor
 import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
 import { queryRows } from '@/lib/table/rows/service'
 import { getTableById, listTables } from '@/lib/table/service'
+import { getOrCreateTableSnapshot } from '@/lib/table/snapshot-cache'
 import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
 import {
   fetchWorkspaceFileBuffer,
   findWorkspaceFileRecord,
   getSandboxWorkspaceFilePath,
   listWorkspaceFiles,
 } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { downloadFile } from '@/lib/uploads/core/storage-service'
 import { executeTool as executeAppTool } from '@/tools'
 import type { ToolExecutionContext, ToolExecutionResult } from '../../tool-executor/types'
 
@@ -21,6 +23,13 @@ const MAX_FILE_SIZE = 10 * 1024 * 1024
 const MAX_TOTAL_SIZE = 50 * 1024 * 1024
 const MAX_MOUNTED_FILES = 500
 
+/**
+ * Below this row count a table mounts via the direct inline CSV path — the version-keyed snapshot
+ * cache (storage round-trip) only pays off for larger/hot tables. Behind the feature flag either
+ * way; this just keeps tiny one-shot tables on the cheaper path.
+ */
+const SNAPSHOT_MIN_ROWS = 500
+
 interface SandboxFile {
   path: string
   content: string
@@ -249,6 +258,7 @@ async function resolveInputFiles(
     const tablePathLookup = hasTablePathRefs
       ? new Map((await listTables(workspaceId)).map((table) => [table.name, table]))
       : undefined
+    const snapshotCacheEnabled = await isFeatureEnabled('table-snapshot-cache')
     for (const tableRef of inputTables) {
       const tableId =
         typeof tableRef === 'string'
@@ -263,6 +273,36 @@ async function resolveInputFiles(
           `Input table not found: "${tableId}". Pass the table id (tbl_...) from tables/{name}/meta.json, or a tables/{name}/meta.json path.`
         )
       }
+      const sandboxPath =
+        typeof tableRef === 'object' && tableRef !== null
+          ? (tableRef as CanonicalTableInput).sandboxPath
+          : undefined
+      const mountPath = sandboxPath || `/home/user/tables/${table.id}.csv`
+
+      // Large/hot tables mount by reference from a version-keyed CSV snapshot in object storage —
+      // size is known before bytes are pulled (like workspace files), bounding web-process memory.
+      if (snapshotCacheEnabled && table.rowCount >= SNAPSHOT_MIN_ROWS) {
+        const snapshot = await getOrCreateTableSnapshot(table, 'copilot-fn-exec')
+        if (snapshot.size > MAX_FILE_SIZE) {
+          throw new Error(
+            `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.`
+          )
+        }
+        if (totalSize + snapshot.size > MAX_TOTAL_SIZE) {
+          throw new Error(
+            `Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.`
+          )
+        }
+        const buffer = await downloadFile({
+          key: snapshot.key,
+          context: 'execution',
+          maxBytes: MAX_FILE_SIZE,
+        })
+        totalSize += buffer.length
+        sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') })
+        continue
+      }
+
       const rows = await queryRows(table, {}, 'copilot-fn-exec')
 
       const allKeys = new Set(table.schema.columns.map((column) => column.name))
@@ -290,14 +330,7 @@ async function resolveInputFiles(
         )
       }
       const csvContent = csvLines.join('\n')
-      const sandboxPath =
-        typeof tableRef === 'object' && tableRef !== null
-          ? (tableRef as CanonicalTableInput).sandboxPath
-          : undefined
-      sandboxFiles.push({
-        path: sandboxPath || `/home/user/tables/${table.id}.csv`,
-        content: csvContent,
-      })
+      sandboxFiles.push({ path: mountPath, content: csvContent })
     }
   }
 

apps/sim/lib/core/config/env.ts

@@ -73,6 +73,7 @@ export const env = createEnv({
     BILLING_ENABLED:                       z.boolean().optional(),                 // Enable billing enforcement and usage tracking
     FREE_API_DEPLOYMENT_GATE_ENABLED:      z.boolean().optional(),                 // Block free-plan accounts from programmatic execution (API/MCP/A2A/generic webhooks/chat embeds). Requires BILLING_ENABLED. Off by default for dark rollout
     TABLES_FRACTIONAL_ORDERING:            z.boolean().optional(),                 // Order table rows by fractional order_key (O(1) insert/delete) instead of integer position
+    TABLE_SNAPSHOT_CACHE:                  z.boolean().optional(),                 // Mount tables into sandboxes by reference via a version-keyed CSV snapshot in object storage instead of draining the whole table into web-process heap
 
     // Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans.
     FREE_TABLES_LIMIT:                     z.number().optional(),                  // Max user tables per workspace on free tier (default: 3)

apps/sim/lib/core/config/feature-flags.ts

@@ -74,6 +74,14 @@ const FEATURE_FLAGS = {
       'user context — use enabled:true for global rollout rather than per-user targeting.',
     fallback: 'MOTHERSHIP_BETA_FEATURES',
   },
+  'table-snapshot-cache': {
+    description:
+      'Mount Sim tables into code sandboxes by reference via a version-keyed CSV snapshot in ' +
+      'object storage (reused across runs until the table mutates) instead of draining the whole ' +
+      'table into web-process heap. resolveInputFiles evaluates without user context — use ' +
+      'enabled:true for global rollout rather than per-user targeting.',
+    fallback: 'TABLE_SNAPSHOT_CACHE',
+  },
 } satisfies Record<string, FeatureFlagDefinition>
 
 /**