improvement(tables): mount snapshots by presigned URL so the sandbox fetches directly (raise cap to 500MB)

Author: TheodoreSpeaks

apps/sim/lib/copilot/tools/handlers/function-execute.test.ts

@@ -10,6 +10,8 @@ const {
   mockQueryRows,
   mockGetOrCreateTableSnapshot,
   mockDownloadFile,
+  mockGeneratePresignedDownloadUrl,
+  mockHasCloudStorage,
   mockExecuteTool,
 } = vi.hoisted(() => ({
   mockIsFeatureEnabled: vi.fn(),
@@ -18,6 +20,8 @@ const {
   mockQueryRows: vi.fn(),
   mockGetOrCreateTableSnapshot: vi.fn(),
   mockDownloadFile: vi.fn(),
+  mockGeneratePresignedDownloadUrl: vi.fn(),
+  mockHasCloudStorage: vi.fn(),
   mockExecuteTool: vi.fn(),
 }))
 
@@ -29,8 +33,13 @@ vi.mock('@/lib/table/service', () => ({
 vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows }))
 vi.mock('@/lib/table/snapshot-cache', () => ({
   getOrCreateTableSnapshot: mockGetOrCreateTableSnapshot,
+  SNAPSHOT_MAX_BYTES: 500 * 1024 * 1024,
+}))
+vi.mock('@/lib/uploads/core/storage-service', () => ({
+  downloadFile: mockDownloadFile,
+  generatePresignedDownloadUrl: mockGeneratePresignedDownloadUrl,
+  hasCloudStorage: mockHasCloudStorage,
 }))
-vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile }))
 vi.mock('@/tools', () => ({ executeTool: mockExecuteTool }))
 // Workspace-file + VFS surfaces are unused on the tables-only path; stub to avoid heavy loads.
 vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
@@ -67,18 +76,22 @@ const context = { workspaceId: 'ws_1', userId: 'u1' }
 
 function mountedFiles() {
   const params = mockExecuteTool.mock.calls[0][1] as {
-    _sandboxFiles?: Array<{ path: string; content: string }>
+    _sandboxFiles?: Array<{ path: string; type?: string; content?: string; url?: string }>
   }
   return params._sandboxFiles ?? []
 }
 
+const snapshotCacheOn = (flag: string) => Promise.resolve(flag === 'table-snapshot-cache')
+
 describe('executeFunctionExecute table mounts', () => {
   beforeEach(() => {
     vi.clearAllMocks()
     mockExecuteTool.mockResolvedValue({ success: true })
     mockGetTableById.mockResolvedValue(table)
     mockIsFeatureEnabled.mockResolvedValue(false)
     mockQueryRows.mockResolvedValue({ rows: [{ data: { name: 'Ada' } }] })
+    mockHasCloudStorage.mockReturnValue(true)
+    mockGeneratePresignedDownloadUrl.mockResolvedValue('https://s3.example/presigned?sig=abc')
   })
 
   it('flag OFF: drains the table inline via queryRows (existing path)', async () => {
@@ -91,33 +104,55 @@ describe('executeFunctionExecute table mounts', () => {
     expect(files[0].content).toBe('name\nAda')
   })
 
-  it('flag ON + large table: mounts by reference from the snapshot, no row drain', async () => {
-    mockIsFeatureEnabled.mockImplementation((flag: string) =>
-      Promise.resolve(flag === 'table-snapshot-cache')
-    )
+  it('flag ON + cloud storage: mounts by presigned URL, no bytes through web', async () => {
+    mockIsFeatureEnabled.mockImplementation(snapshotCacheOn)
     mockGetOrCreateTableSnapshot.mockResolvedValue({
       key: 'table-snapshots/ws_1/tbl_1/v5.csv',
       size: 9,
       version: 5,
     })
-    mockDownloadFile.mockResolvedValue(Buffer.from('name\nAda\n'))
 
     await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
 
     expect(mockGetOrCreateTableSnapshot).toHaveBeenCalledTimes(1)
     expect(mockQueryRows).not.toHaveBeenCalled()
+    expect(mockDownloadFile).not.toHaveBeenCalled()
+    expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith(
+      'table-snapshots/ws_1/tbl_1/v5.csv',
+      'execution',
+      expect.any(Number)
+    )
+    expect(mountedFiles()[0]).toEqual({
+      type: 'url',
+      path: '/home/user/tables/tbl_1.csv',
+      url: 'https://s3.example/presigned?sig=abc',
+    })
+  })
+
+  it('flag ON + local storage: falls back to a buffered content mount', async () => {
+    mockIsFeatureEnabled.mockImplementation(snapshotCacheOn)
+    mockHasCloudStorage.mockReturnValue(false)
+    mockGetOrCreateTableSnapshot.mockResolvedValue({
+      key: 'table-snapshots/ws_1/tbl_1/v5.csv',
+      size: 9,
+      version: 5,
+    })
+    mockDownloadFile.mockResolvedValue(Buffer.from('name\nAda\n'))
+
+    await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+
+    expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled()
     expect(mockDownloadFile).toHaveBeenCalledWith(
       expect.objectContaining({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', context: 'execution' })
     )
-    const files = mountedFiles()
-    expect(files[0].path).toBe('/home/user/tables/tbl_1.csv')
-    expect(files[0].content).toBe('name\nAda\n')
+    const file = mountedFiles()[0]
+    expect(file.path).toBe('/home/user/tables/tbl_1.csv')
+    expect(file.content).toBe('name\nAda\n')
+    expect(file.type).toBeUndefined()
   })
 
   it('flag ON but small table stays on the inline path', async () => {
-    mockIsFeatureEnabled.mockImplementation((flag: string) =>
-      Promise.resolve(flag === 'table-snapshot-cache')
-    )
+    mockIsFeatureEnabled.mockImplementation(snapshotCacheOn)
     mockGetTableById.mockResolvedValue({ ...table, rowCount: 10 })
 
     await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
@@ -126,10 +161,23 @@ describe('executeFunctionExecute table mounts', () => {
     expect(mockQueryRows).toHaveBeenCalledTimes(1)
   })
 
-  it('flag ON: throws when the snapshot exceeds the per-file mount limit', async () => {
-    mockIsFeatureEnabled.mockImplementation((flag: string) =>
-      Promise.resolve(flag === 'table-snapshot-cache')
-    )
+  it('flag ON + cloud: throws when the snapshot exceeds the table mount limit', async () => {
+    mockIsFeatureEnabled.mockImplementation(snapshotCacheOn)
+    mockGetOrCreateTableSnapshot.mockResolvedValue({
+      key: 'table-snapshots/ws_1/tbl_1/v5.csv',
+      size: 600 * 1024 * 1024,
+      version: 5,
+    })
+
+    await expect(
+      executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never)
+    ).rejects.toThrow(/table mount limit/)
+    expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled()
+  })
+
+  it('flag ON + local: throws when the snapshot exceeds the per-file mount limit', async () => {
+    mockIsFeatureEnabled.mockImplementation(snapshotCacheOn)
+    mockHasCloudStorage.mockReturnValue(false)
     mockGetOrCreateTableSnapshot.mockResolvedValue({
       key: 'table-snapshots/ws_1/tbl_1/v5.csv',
       size: 20 * 1024 * 1024,

apps/sim/lib/copilot/tools/handlers/function-execute.ts

@@ -5,15 +5,19 @@ import { isPlanAliasPath, workflowAliasSandboxPath } from '@/lib/copilot/vfs/wor
 import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
 import { queryRows } from '@/lib/table/rows/service'
 import { getTableById, listTables } from '@/lib/table/service'
-import { getOrCreateTableSnapshot } from '@/lib/table/snapshot-cache'
+import { getOrCreateTableSnapshot, SNAPSHOT_MAX_BYTES } from '@/lib/table/snapshot-cache'
 import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
 import {
   fetchWorkspaceFileBuffer,
   findWorkspaceFileRecord,
   getSandboxWorkspaceFilePath,
   listWorkspaceFiles,
 } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
-import { downloadFile } from '@/lib/uploads/core/storage-service'
+import {
+  downloadFile,
+  generatePresignedDownloadUrl,
+  hasCloudStorage,
+} from '@/lib/uploads/core/storage-service'
 import { executeTool as executeAppTool } from '@/tools'
 import type { ToolExecutionContext, ToolExecutionResult } from '../../tool-executor/types'
 
@@ -30,11 +34,15 @@ const MAX_MOUNTED_FILES = 500
  */
 const SNAPSHOT_MIN_ROWS = 500
 
-interface SandboxFile {
-  path: string
-  content: string
-  encoding?: 'base64'
-}
+/**
+ * Lifetime of the presigned URL handed to the sandbox to fetch a snapshot. Long enough to download
+ * a large file at sandbox startup; the URL grants read to only that one version-pinned object.
+ */
+const SNAPSHOT_URL_TTL_SECONDS = 600
+
+type SandboxFile =
+  | { type?: 'content'; path: string; content: string; encoding?: 'base64' }
+  | { type: 'url'; path: string; url: string }
 
 interface CanonicalFileInput {
   path: string
@@ -279,10 +287,30 @@ async function resolveInputFiles(
           : undefined
       const mountPath = sandboxPath || `/home/user/tables/${table.id}.csv`
 
-      // Large/hot tables mount by reference from a version-keyed CSV snapshot in object storage —
-      // size is known before bytes are pulled (like workspace files), bounding web-process memory.
+      // Large/hot tables mount by reference from a version-keyed CSV snapshot in object storage.
       if (snapshotCacheEnabled && table.rowCount >= SNAPSHOT_MIN_ROWS) {
         const snapshot = await getOrCreateTableSnapshot(table, 'copilot-fn-exec')
+
+        if (hasCloudStorage()) {
+          // Mount by reference: the sandbox fetches the snapshot straight from storage via a
+          // presigned URL, so the bytes never pass through the web process — the only ceiling is
+          // sandbox disk (enforced at materialization by SNAPSHOT_MAX_BYTES).
+          if (snapshot.size > SNAPSHOT_MAX_BYTES) {
+            throw new Error(
+              `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${SNAPSHOT_MAX_BYTES / 1024 / 1024}MB table mount limit.`
+            )
+          }
+          const url = await generatePresignedDownloadUrl(
+            snapshot.key,
+            'execution',
+            SNAPSHOT_URL_TTL_SECONDS
+          )
+          sandboxFiles.push({ type: 'url', path: mountPath, url })
+          continue
+        }
+
+        // Local storage: a presigned URL is an app-internal serve path a remote sandbox can't
+        // reach, so fall back to buffering the bytes through the web process (file-mount guards).
         if (snapshot.size > MAX_FILE_SIZE) {
           throw new Error(
             `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.`

apps/sim/lib/execution/e2b.test.ts

@@ -0,0 +1,108 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { CodeLanguage } from '@/lib/execution/languages'
+
+const { mockCreate, mockRunCode, mockCommandsRun, mockFilesWrite, mockKill } = vi.hoisted(() => ({
+  mockCreate: vi.fn(),
+  mockRunCode: vi.fn(),
+  mockCommandsRun: vi.fn(),
+  mockFilesWrite: vi.fn(),
+  mockKill: vi.fn(),
+}))
+
+vi.mock('@e2b/code-interpreter', () => ({ Sandbox: { create: mockCreate } }))
+vi.mock('@/lib/core/config/env', () => ({ env: { E2B_API_KEY: 'test-key' } }))
+
+import { executeInE2B, executeShellInE2B } from '@/lib/execution/e2b'
+
+describe('e2b sandbox inputs', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockCreate.mockResolvedValue({
+      sandboxId: 'sb_1',
+      files: { write: mockFilesWrite },
+      commands: { run: mockCommandsRun },
+      runCode: mockRunCode,
+      kill: mockKill,
+    })
+    mockRunCode.mockResolvedValue({
+      error: null,
+      text: '',
+      logs: { stdout: [], stderr: [] },
+      results: [],
+    })
+    // Default: shell code run + any fetch succeed.
+    mockCommandsRun.mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 })
+  })
+
+  it('fetches a url entry via curl with URL/DST/DIR passed as envs (no inline write)', async () => {
+    await executeInE2B({
+      code: 'x',
+      language: CodeLanguage.JavaScript,
+      timeoutMs: 1000,
+      sandboxFiles: [
+        { type: 'url', path: '/home/user/tables/t.csv', url: 'https://s3.example/p?a=1&b=2' },
+      ],
+    })
+
+    expect(mockCommandsRun).toHaveBeenCalledTimes(1)
+    const [cmd, opts] = mockCommandsRun.mock.calls[0]
+    expect(cmd).toContain('curl')
+    expect(cmd).toContain('mkdir -p')
+    // URL/path go through envs, never interpolated into the command string.
+    expect(cmd).not.toContain('https://s3.example')
+    expect(opts.envs).toEqual({
+      URL: 'https://s3.example/p?a=1&b=2',
+      DST: '/home/user/tables/t.csv',
+      DIR: '/home/user/tables',
+    })
+    expect(opts.user).toBeUndefined() // code sandbox runs as default user
+    expect(mockFilesWrite).not.toHaveBeenCalled()
+  })
+
+  it('writes a content entry inline (no fetch)', async () => {
+    await executeInE2B({
+      code: 'x',
+      language: CodeLanguage.JavaScript,
+      timeoutMs: 1000,
+      sandboxFiles: [{ path: '/home/user/f.txt', content: 'hi' }],
+    })
+
+    expect(mockFilesWrite).toHaveBeenCalledWith('/home/user/f.txt', 'hi')
+    expect(mockCommandsRun).not.toHaveBeenCalled()
+  })
+
+  it('fetches as root in the shell sandbox', async () => {
+    await executeShellInE2B({
+      code: 'echo hi',
+      envs: {},
+      timeoutMs: 1000,
+      sandboxFiles: [{ type: 'url', path: '/home/user/tables/t.csv', url: 'https://s3.example/p' }],
+    })
+
+    const fetchCall = mockCommandsRun.mock.calls.find((c) => c[1]?.envs?.URL)
+    expect(fetchCall).toBeDefined()
+    expect(fetchCall?.[0]).toContain('curl')
+    expect(fetchCall?.[1].user).toBe('root')
+  })
+
+  it('throws a clear error and kills the sandbox when the fetch fails', async () => {
+    mockCommandsRun.mockRejectedValueOnce(new Error('curl: (22) 403'))
+
+    await expect(
+      executeInE2B({
+        code: 'x',
+        language: CodeLanguage.JavaScript,
+        timeoutMs: 1000,
+        sandboxFiles: [
+          { type: 'url', path: '/home/user/tables/t.csv', url: 'https://s3.example/p' },
+        ],
+      })
+    ).rejects.toThrow(/Failed to fetch mounted file into sandbox/)
+
+    expect(mockKill).toHaveBeenCalled()
+    expect(mockRunCode).not.toHaveBeenCalled()
+  })
+})