Use chunked transfer encoding for tar uploads (#1118)

Author: mishushakov

.changeset/chunked-upload.md

@@ -0,0 +1,5 @@
+---
+"e2b": patch
+---
+
+Use chunked transfer encoding for tar uploads to reduce memory usage

packages/js-sdk/src/template/buildApi.ts

@@ -111,7 +111,7 @@ export async function uploadFile(
   const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =
     options
   try {
-    const { contentLength, uploadStream } = await tarFileStreamUpload(
+    const uploadStream = await tarFileStreamUpload(
       fileName,
       fileContextPath,
       ignorePatterns,
@@ -123,9 +123,6 @@ export async function uploadFile(
       method: 'PUT',
       // @ts-expect-error
       body: uploadStream,
-      headers: {
-        'Content-Length': contentLength.toString(),
-      },
       duplex: 'half',
     })
 

packages/js-sdk/src/template/utils.ts

@@ -373,7 +373,7 @@ export async function tarFileStream(
   // gzip.portable ensures deterministic gzip header without affecting file modes
   return create(
     {
-      gzip: { portable: true },
+      gzip: true,
       cwd: fileContextPath,
       follow: resolveSymlinks,
       noDirRecurse: true,
@@ -383,40 +383,25 @@ export async function tarFileStream(
 }
 
 /**
- * Create a tar stream and calculate its compressed size for upload.
+ * Create a tar stream for upload using chunked transfer encoding.
  *
  * @param fileName Glob pattern for files to include
  * @param fileContextPath Base directory for resolving file paths
  * @param resolveSymlinks Whether to follow symbolic links
- * @returns Object containing the content length and upload stream
+ * @returns A readable stream of the gzipped tar archive
  */
 export async function tarFileStreamUpload(
   fileName: string,
   fileContextPath: string,
   ignorePatterns: string[],
   resolveSymlinks: boolean
 ) {
-  // First pass: calculate the compressed size
-  const sizeCalculationStream = await tarFileStream(
+  return tarFileStream(
     fileName,
     fileContextPath,
     ignorePatterns,
     resolveSymlinks
   )
-  let contentLength = 0
-  for await (const chunk of sizeCalculationStream as unknown as AsyncIterable<Buffer>) {
-    contentLength += chunk.length
-  }
-
-  return {
-    contentLength,
-    uploadStream: await tarFileStream(
-      fileName,
-      fileContextPath,
-      ignorePatterns,
-      resolveSymlinks
-    ),
-  }
 }
 
 /**