diff --git a/.changeset/fix-js-sdk-upload-content-length.md b/.changeset/fix-js-sdk-upload-content-length.md new file mode 100644 index 0000000000..7d4b690059 --- /dev/null +++ b/.changeset/fix-js-sdk-upload-content-length.md @@ -0,0 +1,5 @@ +--- +"e2b": patch +--- + +fix(js-sdk): buffer template tar archive before upload so `fetch` sets `Content-Length` instead of falling back to `Transfer-Encoding: chunked`. S3 presigned PUT URLs reject chunked requests with `501 NotImplemented`, breaking template uploads in self-hosted deployments backed by S3-compatible storage. Aligns the JS SDK with the Python SDK, which already buffers via `io.BytesIO`. diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts index d699efe03c..b070d2bbb1 100644 --- a/packages/js-sdk/src/template/buildApi.ts +++ b/packages/js-sdk/src/template/buildApi.ts @@ -1,5 +1,5 @@ import { ApiClient, handleApiError, paths, components } from '../api' -import { stripAnsi } from '../utils' +import { dynamicImport, stripAnsi } from '../utils' import { BuildError, FileUploadError, TemplateError } from '../errors' import { LogEntry } from './logger' import { getBuildStepIndex, tarFileStreamUpload } from './utils' @@ -119,12 +119,25 @@ export async function uploadFile( resolveSymlinks ) - // The compiler assumes this is Web fetch API, but it's actually Node.js fetch API + // Buffer the archive before uploading so fetch sets Content-Length. + // S3 presigned PUT URLs reject Transfer-Encoding: chunked with 501 + // NotImplemented, which is what Node's fetch falls back to when the + // body is a Readable without a known length. See e2b-dev/e2b#1243. + // The Python SDK takes the same approach (build_api.py:upload_file). + // Dynamically import so the browser bundle doesn't pull in node:stream. + // tar's Pack extends Minipass and is iterable as AsyncIterable at + // runtime, but the cli's tsconfig (preserveSymlinks) doesn't surface that + // through the type chain — cast via unknown. + const { buffer } = await dynamicImport< + typeof import('node:stream/consumers') + >('node:stream/consumers') + const uploadBody = await buffer( + uploadStream as unknown as AsyncIterable + ) + const res = await fetch(url, { method: 'PUT', - // @ts-expect-error - body: uploadStream, - duplex: 'half', + body: uploadBody, }) if (!res.ok) { diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.ts new file mode 100644 index 0000000000..3d0cd25b05 --- /dev/null +++ b/packages/js-sdk/tests/template/uploadFile.test.ts @@ -0,0 +1,68 @@ +import { describe, test, expect, beforeAll, afterAll } from 'vitest' +import { writeFile, mkdtemp, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' +import { createServer, type IncomingMessage, type Server } from 'http' +import { AddressInfo } from 'net' +import { uploadFile } from '../../src/template/buildApi' + +// Regression test for e2b-dev/e2b#1243 — uploadFile used to pass a Node +// Readable directly to fetch, which made undici fall back to +// Transfer-Encoding: chunked. S3 presigned PUT URLs reject that with 501 +// NotImplemented. The fix buffers the archive first so Content-Length is set. +describe('uploadFile transfer encoding', () => { + let testDir: string + let server: Server + let baseUrl: string + let capturedHeaders: IncomingMessage['headers'] = {} + let capturedBodyLength = 0 + + beforeAll(async () => { + testDir = await mkdtemp(join(tmpdir(), 'uploadFile-test-')) + await writeFile(join(testDir, 'hello.txt'), 'hello world') + + server = createServer((req, res) => { + capturedHeaders = req.headers + let bytes = 0 + req.on('data', (chunk: Buffer) => { + bytes += chunk.length + }) + req.on('end', () => { + capturedBodyLength = bytes + res.writeHead(200) + res.end() + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + baseUrl = `http://127.0.0.1:${port}/upload` + }) + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())) + await rm(testDir, { recursive: true, force: true }) + }) + + test('sets Content-Length and does not use chunked transfer encoding', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + ignorePatterns: [], + resolveSymlinks: false, + }, + undefined + ) + + expect(capturedHeaders['content-length']).toBeDefined() + const contentLength = Number(capturedHeaders['content-length']) + expect(contentLength).toBeGreaterThan(0) + expect(contentLength).toBe(capturedBodyLength) + + const transferEncoding = capturedHeaders['transfer-encoding'] + if (transferEncoding !== undefined) { + expect(transferEncoding.toLowerCase()).not.toContain('chunked') + } + }) +}) diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py new file mode 100644 index 0000000000..b50000744b --- /dev/null +++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py @@ -0,0 +1,71 @@ +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from e2b.api.client.client import AuthenticatedClient +from e2b.template_async.build_api import upload_file + + +# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length +# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs +# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length +# automatically when we pass bytes (tar_buffer.getvalue()); this test guards +# against someone swapping the bytes for a generator/stream later. +# +# The mock server runs in a daemon thread and doesn't need to be async — the +# httpx.AsyncClient connects to it via asyncio sockets without blocking the +# event loop. + + +def _make_server(): + state = {"headers": None, "body_length": 0} + + class Handler(BaseHTTPRequestHandler): + def do_PUT(self): + state["headers"] = dict(self.headers) + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) if length else b"" + state["body_length"] = len(body) + self.send_response(200) + self.end_headers() + + def log_message(self, *args, **kwargs): + return + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, state + + +async def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"] is not None + content_length = state["headers"].get("Content-Length") + assert content_length is not None + assert int(content_length) > 0 + assert int(content_length) == state["body_length"] + + transfer_encoding = state["headers"].get("Transfer-Encoding") + if transfer_encoding is not None: + assert "chunked" not in transfer_encoding.lower() diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py new file mode 100644 index 0000000000..76b4216d58 --- /dev/null +++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py @@ -0,0 +1,67 @@ +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from e2b.api.client.client import AuthenticatedClient +from e2b.template_sync.build_api import upload_file + + +# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length +# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs +# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length +# automatically when we pass bytes (tar_buffer.getvalue()); this test guards +# against someone swapping the bytes for a generator/stream later. + + +def _make_server(): + state = {"headers": None, "body_length": 0} + + class Handler(BaseHTTPRequestHandler): + def do_PUT(self): + state["headers"] = dict(self.headers) + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) if length else b"" + state["body_length"] = len(body) + self.send_response(200) + self.end_headers() + + def log_message(self, *args, **kwargs): + return + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, state + + +def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"] is not None + content_length = state["headers"].get("Content-Length") + assert content_length is not None + assert int(content_length) > 0 + assert int(content_length) == state["body_length"] + + transfer_encoding = state["headers"].get("Transfer-Encoding") + if transfer_encoding is not None: + assert "chunked" not in transfer_encoding.lower()