feat: add Content-Encoding option for file operations

.changeset/ripe-ties-hang.md

@@ -0,0 +1,6 @@
+---
+'@e2b/python-sdk': patch
+'e2b': patch
+---
+
+added gzip support for sandbox file/upload download

packages/js-sdk/src/index.ts

@@ -22,7 +22,13 @@ export type { Logger } from './logs'
 export { getSignature } from './sandbox/signature'
 
 export { FileType } from './sandbox/filesystem'
-export type { WriteInfo, EntryInfo, Filesystem } from './sandbox/filesystem'
+export type {
+  WriteInfo,
+  EntryInfo,
+  Filesystem,
+  FilesystemEncodingOpts,
+  FilesystemFormatOpts,
+} from './sandbox/filesystem'
 export { FilesystemEventType } from './sandbox/filesystem/watchHandle'
 export type {
   FilesystemEvent,

packages/js-sdk/src/sandbox/filesystem/index.ts

@@ -32,7 +32,7 @@
   ENVD_VERSION_RECURSIVE_WATCH,
 } from '../../envd/versions'
 import { InvalidArgumentError, TemplateError } from '../../errors'
-import { toBlob } from '../../utils'
+import { toUploadBody } from '../../utils'
 
 /**
  * Sandbox filesystem object information.
@@ -138,6 +138,29 @@
   user?: Username
 }
 
+/**
+ * Options for gzip compression of the request/response body.
+ */
+export interface FilesystemEncodingOpts {
+  /**
+   * When true, uploads will be gzip-compressed and downloads
+   * will request gzip-encoded responses.
+   */
+  gzip?: boolean
+}
+
+/**
+ * Options for the format of the file content returned by read operations.
+ */
+export interface FilesystemFormatOpts {
+  /**
+   * Format of the file content.
+   *
+   * @default 'text'
+   */
+  format?: 'text' | 'bytes' | 'blob' | 'stream'
+}
+
 export interface FilesystemListOpts extends FilesystemRequestOpts {
   /**
    * Depth of the directory to list.
@@ -196,7 +219,7 @@
    */
   async read(
     path: string,
-    opts?: FilesystemRequestOpts & { format?: 'text' }
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts & { format?: 'text' }
   ): Promise<string>
   /**
    * Read file content as a `Uint8Array`.
@@ -211,7 +234,7 @@
    */
   async read(
     path: string,
-    opts?: FilesystemRequestOpts & { format: 'bytes' }
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts & { format: 'bytes' }
   ): Promise<Uint8Array>
   /**
    * Read file content as a `Blob`.
@@ -226,7 +249,7 @@
    */
   async read(
     path: string,
-    opts?: FilesystemRequestOpts & { format: 'blob' }
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts & { format: 'blob' }
   ): Promise<Blob>
   /**
    * Read file content as a `ReadableStream`.
@@ -241,13 +264,11 @@
    */
   async read(
     path: string,
-    opts?: FilesystemRequestOpts & { format: 'stream' }
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts & { format: 'stream' }
   ): Promise<ReadableStream<Uint8Array>>
   async read(
     path: string,
-    opts?: FilesystemRequestOpts & {
-      format?: 'text' | 'stream' | 'bytes' | 'blob'
-    }
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts & FilesystemFormatOpts
   ): Promise<unknown> {
     const format = opts?.format ?? 'text'
 
@@ -259,6 +280,11 @@
       user = defaultUsername
     }
 
+    const headers: Record<string, string> = {}
+    if (opts?.gzip) {
+      headers['Accept-Encoding'] = 'gzip'
+    }
+
     const res = await this.envdApi.api.GET('/files', {
       params: {
         query: {
@@ -268,6 +294,7 @@
       },
       parseAs: format === 'bytes' ? 'arrayBuffer' : format,
       signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs),
+      headers,
     })
 
     const err = await handleEnvdApiError(res)
@@ -306,11 +333,11 @@
   async write(
     path: string,
     data: string | ArrayBuffer | Blob | ReadableStream,
-    opts?: FilesystemRequestOpts
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts
   ): Promise<WriteInfo>
   async write(
     files: WriteEntry[],
-    opts?: FilesystemRequestOpts
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts
   ): Promise<WriteInfo[]>
   async write(
     pathOrFiles: string | WriteEntry[],
@@ -319,8 +346,8 @@
       | ArrayBuffer
       | Blob
       | ReadableStream
-      | FilesystemRequestOpts,
-    opts?: FilesystemRequestOpts
+      | (FilesystemRequestOpts & FilesystemEncodingOpts),
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts
   ): Promise<WriteInfo | WriteInfo[]> {
     if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) {
       throw new Error('Path or files are required')
@@ -336,7 +363,7 @@
       typeof pathOrFiles === 'string'
         ? {
             path: pathOrFiles,
-            writeOpts: opts as FilesystemRequestOpts,
+            writeOpts: opts as FilesystemRequestOpts & FilesystemEncodingOpts,
             writeFiles: [
               {
                 data: dataOrOpts as
@@ -349,17 +376,14 @@
           }
         : {
             path: undefined,
-            writeOpts: dataOrOpts as FilesystemRequestOpts,
+            writeOpts: dataOrOpts as FilesystemRequestOpts &
+              FilesystemEncodingOpts,
             writeFiles: pathOrFiles as WriteEntry[],
           }
 
     if (writeFiles.length === 0) return [] as WriteInfo[]
 
-    const formData = new FormData()
-    for (let i = 0; i < writeFiles.length; i++) {
-      const file = writeFiles[i]
-      formData.append('file', await toBlob(file.data), writeFiles[i].path)
-    }
+    const useGzip = writeOpts?.gzip === true
 
     let user = writeOpts?.user
     if (
@@ -369,29 +393,43 @@
       user = defaultUsername
     }
 
-    const res = await this.envdApi.api.POST('/files', {
-      params: {
-        query: {
-          path,
-          username: user,
-        },
-      },
-      bodySerializer: () => formData,
-      signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs),
-      body: {},
-    })
+    const results: WriteInfo[] = []
+    for (const file of writeFiles) {
+      const filePath = (file as WriteEntry).path ?? path
 
-    const err = await handleEnvdApiError(res)
-    if (err) {
-      throw err
-    }
+      const headers: Record<string, string> = {
+        'Content-Type': 'application/octet-stream',
+      }
 
-    const files = res.data as WriteInfo[]
-    if (!files) {
-      throw new Error('Expected to receive information about written file')
+      if (useGzip) {
+        headers['Content-Encoding'] = 'gzip'
+      }
+      const body = toUploadBody(file.data, useGzip)
+
+      const res = await this.envdApi.api.POST('/files', {
+        params: {
+          query: {
+            path: filePath,
+            username: user,
+          },
+        },
+        bodySerializer: () => body,
+        signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs),
+        body: {},
+        headers,
+      })
+
+      const err = await handleEnvdApiError(res)
+      if (err) throw err
+
+      const fileInfos = res.data as WriteInfo[]
+      if (!fileInfos) {
+        throw new Error('Expected to receive information about written file')
+      }
+      results.push(...fileInfos)
     }
 
-    return files.length === 1 && path ? files[0] : files
+    return results.length === 1 && path ? results[0] : results
   }
 
   /**
@@ -411,7 +449,7 @@
    */
   async writeFiles(
     files: WriteEntry[],
-    opts?: FilesystemRequestOpts
+    opts?: FilesystemRequestOpts & FilesystemEncodingOpts
   ): Promise<WriteInfo[]> {
     return this.write(files, opts) as Promise<WriteInfo[]>
   }

packages/js-sdk/src/utils.ts

@@ -108,19 +108,25 @@ export async function wait(ms: number) {
 }
 
 /**
- * Convert data to a Blob, avoiding unnecessary conversions when possible.
+ * Prepare data for upload as a BodyInit, optionally gzip-compressed.
+ * Streams data directly without buffering into memory.
  */
-export function toBlob(
-  data: string | ArrayBuffer | Blob | ReadableStream
-): Blob | Promise<Blob> {
-  // Already a Blob - use directly
-  if (data instanceof Blob) {
-    return data
+export function toUploadBody(
+  data: string | ArrayBuffer | Blob | ReadableStream,
+  gzip?: boolean
+): BodyInit {
+  if (gzip) {
+    const stream =
+      data instanceof ReadableStream
+        ? data
+        : data instanceof Blob
+          ? data.stream()
+          : new Blob([data]).stream()
+    return stream.pipeThrough(new CompressionStream('gzip'))
   }
-  // String or ArrayBuffer - create Blob
-  if (typeof data === 'string' || data instanceof ArrayBuffer) {
-    return new Blob([data])
+
+  if (data instanceof ReadableStream || data instanceof Blob) {
+    return data
   }
-  // ReadableStream - must consume to get Blob
-  return new Response(data).blob()
+  return new Blob([data])
 }

packages/js-sdk/tests/sandbox/files/contentEncoding.test.ts

@@ -0,0 +1,94 @@
+import { assert } from 'vitest'
+
+import { WriteEntry } from '../../../src/sandbox/filesystem'
+import { isDebug, sandboxTest } from '../../setup.js'
+
+sandboxTest(
+  'write and read file with gzip content encoding',
+  async ({ sandbox }) => {
+    const filename = 'test_gzip_write.txt'
+    const content = 'This is a test file with gzip encoding.'
+
+    const info = await sandbox.files.write(filename, content, {
+      gzip: true,
+    })
+    assert.equal(info.name, filename)
+    assert.equal(info.type, 'file')
+    assert.equal(info.path, `/home/user/${filename}`)
+
+    const readContent = await sandbox.files.read(filename, {
+      gzip: true,
+    })
+    assert.equal(readContent, content)
+
+    if (isDebug) {
+      await sandbox.files.remove(filename)
+    }
+  }
+)
+
+sandboxTest(
+  'write with gzip and read without encoding',
+  async ({ sandbox }) => {
+    const filename = 'test_gzip_write_plain_read.txt'
+    const content = 'Written with gzip, read without.'
+
+    await sandbox.files.write(filename, content, {
+      gzip: true,
+    })
+
+    const readContent = await sandbox.files.read(filename)
+    assert.equal(readContent, content)
+
+    if (isDebug) {
+      await sandbox.files.remove(filename)
+    }
+  }
+)
+
+sandboxTest('writeFiles with gzip content encoding', async ({ sandbox }) => {
+  const files: WriteEntry[] = [
+    { path: 'gzip_multi_1.txt', data: 'File 1 content' },
+    { path: 'gzip_multi_2.txt', data: 'File 2 content' },
+    { path: 'gzip_multi_3.txt', data: 'File 3 content' },
+  ]
+
+  const infos = await sandbox.files.writeFiles(files, {
+    gzip: true,
+  })
+
+  assert.equal(infos.length, files.length)
+
+  for (let i = 0; i < files.length; i++) {
+    const readContent = await sandbox.files.read(files[i].path)
+    assert.equal(readContent, files[i].data)
+  }
+
+  if (isDebug) {
+    for (const file of files) {
+      await sandbox.files.remove(file.path)
+    }
+  }
+})
+
+sandboxTest(
+  'read file as bytes with gzip content encoding',
+  async ({ sandbox }) => {
+    const filename = 'test_gzip_bytes.txt'
+    const content = 'Binary content with gzip.'
+
+    await sandbox.files.write(filename, content)
+
+    const readBytes = await sandbox.files.read(filename, {
+      format: 'bytes',
+      gzip: true,
+    })
+    assert.instanceOf(readBytes, Uint8Array)
+    const decoded = new TextDecoder().decode(readBytes)
+    assert.equal(decoded, content)
+
+    if (isDebug) {
+      await sandbox.files.remove(filename)
+    }
+  }
+)