diff --git a/.changeset/chunked-upload.md b/.changeset/chunked-upload.md new file mode 100644 index 0000000000..15f54a2309 --- /dev/null +++ b/.changeset/chunked-upload.md @@ -0,0 +1,5 @@ +--- +"e2b": patch +--- + +Use chunked transfer encoding for tar uploads to reduce memory usage diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts index ffffe1e509..d56d953c67 100644 --- a/packages/js-sdk/src/template/buildApi.ts +++ b/packages/js-sdk/src/template/buildApi.ts @@ -111,7 +111,7 @@ export async function uploadFile( const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } = options try { - const { contentLength, uploadStream } = await tarFileStreamUpload( + const uploadStream = await tarFileStreamUpload( fileName, fileContextPath, ignorePatterns, @@ -123,9 +123,6 @@ export async function uploadFile( method: 'PUT', // @ts-expect-error body: uploadStream, - headers: { - 'Content-Length': contentLength.toString(), - }, duplex: 'half', }) diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.ts index ad907ea834..4a03fc5ad6 100644 --- a/packages/js-sdk/src/template/utils.ts +++ b/packages/js-sdk/src/template/utils.ts @@ -373,7 +373,7 @@ export async function tarFileStream( // gzip.portable ensures deterministic gzip header without affecting file modes return create( { - gzip: { portable: true }, + gzip: true, cwd: fileContextPath, follow: resolveSymlinks, noDirRecurse: true, @@ -383,12 +383,12 @@ export async function tarFileStream( } /** - * Create a tar stream and calculate its compressed size for upload. + * Create a tar stream for upload using chunked transfer encoding. * * @param fileName Glob pattern for files to include * @param fileContextPath Base directory for resolving file paths * @param resolveSymlinks Whether to follow symbolic links - * @returns Object containing the content length and upload stream + * @returns A readable stream of the gzipped tar archive */ export async function tarFileStreamUpload( fileName: string, @@ -396,27 +396,12 @@ export async function tarFileStreamUpload( ignorePatterns: string[], resolveSymlinks: boolean ) { - // First pass: calculate the compressed size - const sizeCalculationStream = await tarFileStream( + return tarFileStream( fileName, fileContextPath, ignorePatterns, resolveSymlinks ) - let contentLength = 0 - for await (const chunk of sizeCalculationStream as unknown as AsyncIterable) { - contentLength += chunk.length - } - - return { - contentLength, - uploadStream: await tarFileStream( - fileName, - fileContextPath, - ignorePatterns, - resolveSymlinks - ), - } } /**