diff --git a/.changeset/python-pyqwest-template-uploads.md b/.changeset/python-pyqwest-template-uploads.md new file mode 100644 index 0000000000..660881adb3 --- /dev/null +++ b/.changeset/python-pyqwest-template-uploads.md @@ -0,0 +1,11 @@ +--- +"@e2b/python-sdk": minor +--- + +Move template build-context uploads (to S3 presigned URLs) onto +[`pyqwest`](https://pypi.org/project/pyqwest/) via its httpx-compatible +transport adapter. Content-Length framing for the streamed archive body is +preserved (S3 rejects chunked transfer encoding). The 1-hour upload timeout +now bounds the entire upload rather than each socket operation, and +`verify_ssl=False` on the client is no longer honored for uploads (pyqwest +has no insecure-TLS option). diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py index c9209fa5df..7acfc97c7b 100644 --- a/packages/python-sdk/e2b/template_async/build_api.py +++ b/packages/python-sdk/e2b/template_async/build_api.py @@ -4,8 +4,10 @@ from typing import Callable, Optional, List, Union import httpx +from pyqwest import HTTPTransport -from e2b.api import handle_api_exception +from e2b.api import handle_api_exception, proxy_to_config +from e2b.api.client_async import AsyncApiPyqwestTransport from e2b.io_utils import aiter_io_chunks from e2b.api.client.api.templates import ( post_v3_templates, @@ -118,6 +120,7 @@ async def upload_file( upload_timeout = ( request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS ) + upload_proxy = proxy_to_config(getattr(api_client, "_proxy", None)) try: tar_file = tar_file_stream( file_name, context_path, ignore_patterns, resolve_symlinks, gzip @@ -125,16 +128,27 @@ async def upload_file( try: size = os.fstat(tar_file.fileno()).st_size + # Through the pyqwest adapter the upload timeout is a + # whole-request deadline for the entire transfer, not a per-write + # bound as with the httpx transport this replaced. async with httpx.AsyncClient( timeout=httpx.Timeout(upload_timeout), - verify=api_client._verify_ssl, follow_redirects=api_client._follow_redirects, - proxy=getattr(api_client, "_proxy", None), - http2=False, + transport=AsyncApiPyqwestTransport( + HTTPTransport( + tls_include_system_certs=True, + proxy=( + upload_proxy.to_pyqwest() + if upload_proxy is not None + else None + ), + ) + ), ) as client: # Stream the archive from disk via an async iterator. The # explicit Content-Length suppresses chunked transfer - # encoding, which S3 presigned URLs reject. + # encoding, which S3 presigned URLs reject; reqwest keeps the + # Content-Length framing for the streamed body. response = await client.put( url, content=aiter_io_chunks(tar_file), diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py index e838d87887..dfe79e9477 100644 --- a/packages/python-sdk/e2b/template_sync/build_api.py +++ b/packages/python-sdk/e2b/template_sync/build_api.py @@ -3,8 +3,10 @@ from typing import Callable, Optional, List, Union import httpx +from pyqwest import SyncHTTPTransport -from e2b.api import handle_api_exception +from e2b.api import handle_api_exception, proxy_to_config +from e2b.api.client_sync import ApiPyqwestTransport from e2b.api.client.api.templates import ( post_v3_templates, get_templates_template_id_files_hash, @@ -116,21 +118,33 @@ def upload_file( upload_timeout = ( request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS ) + upload_proxy = proxy_to_config(getattr(api_client, "_proxy", None)) try: tar_file = tar_file_stream( file_name, context_path, ignore_patterns, resolve_symlinks, gzip ) try: + # Through the pyqwest adapter the upload timeout is a + # whole-request deadline for the entire transfer, not a per-write + # bound as with the httpx transport this replaced. with httpx.Client( timeout=httpx.Timeout(upload_timeout), - verify=api_client._verify_ssl, follow_redirects=api_client._follow_redirects, - proxy=getattr(api_client, "_proxy", None), - http2=False, + transport=ApiPyqwestTransport( + SyncHTTPTransport( + tls_include_system_certs=True, + proxy=( + upload_proxy.to_pyqwest() + if upload_proxy is not None + else None + ), + ) + ), ) as client: # httpx streams the archive from disk in chunks and sets # Content-Length from the file size—S3 presigned URLs reject - # chunked transfer encoding. + # chunked transfer encoding, and reqwest keeps the + # Content-Length framing for the streamed body. response = client.put(url, content=tar_file) response.raise_for_status() finally: diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py index 4b61e7c57e..c7136a7516 100644 --- a/packages/python-sdk/tests/async/template_async/test_upload_file.py +++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py @@ -26,7 +26,9 @@ def _make_server(): class Handler(BaseHTTPRequestHandler): def do_PUT(self): - state["headers"] = dict(self.headers) + # hyper (pyqwest) sends lowercase header names where httpcore + # title-cased them; compare case-insensitively. + state["headers"] = {k.lower(): v for k, v in self.headers.items()} length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) if length else b"" state["body_length"] = len(body) @@ -72,15 +74,15 @@ def get_async_httpx_client(self): thread.join(timeout=5) assert state["headers"] is not None - content_length = state["headers"].get("Content-Length") + content_length = state["headers"].get("content-length") assert content_length is not None assert int(content_length) > 0 assert int(content_length) == state["body_length"] - transfer_encoding = state["headers"].get("Transfer-Encoding") + transfer_encoding = state["headers"].get("transfer-encoding") if transfer_encoding is not None: assert "chunked" not in transfer_encoding.lower() - assert "Authorization" not in state["headers"] + assert "authorization" not in state["headers"] async def _capture_upload_timeout(tmp_path, request_timeout=None): diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py index 74b65fa8c5..0c14be6397 100644 --- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py +++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py @@ -22,7 +22,9 @@ def _make_server(): class Handler(BaseHTTPRequestHandler): def do_PUT(self): - state["headers"] = dict(self.headers) + # hyper (pyqwest) sends lowercase header names where httpcore + # title-cased them; compare case-insensitively. + state["headers"] = {k.lower(): v for k, v in self.headers.items()} length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) if length else b"" state["body_length"] = len(body) @@ -68,15 +70,15 @@ def get_httpx_client(self): thread.join(timeout=5) assert state["headers"] is not None - content_length = state["headers"].get("Content-Length") + content_length = state["headers"].get("content-length") assert content_length is not None assert int(content_length) > 0 assert int(content_length) == state["body_length"] - transfer_encoding = state["headers"].get("Transfer-Encoding") + transfer_encoding = state["headers"].get("transfer-encoding") if transfer_encoding is not None: assert "chunked" not in transfer_encoding.lower() - assert "Authorization" not in state["headers"] + assert "authorization" not in state["headers"] def _capture_upload_timeout(tmp_path, request_timeout=None):