Merge pull request #7 from bluedynamics/fix/s3-max-pool-connections

fix: set explicit max_pool_connections on boto3 S3 client (closes #6)

Author: jensens

CHANGES.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.4.3 (unreleased)
+
+- Fix: boto3 S3 client is now created with an explicit
+  `max_pool_connections` via `botocore.Config` (default 50, overridable
+  via `PGTHUMBOR_S3_MAX_POOL_CONNECTIONS`).  The boto3 default of 10
+  caused urllib3 pool-full warnings and connection churn under normal
+  Thumbor load (30 thumbnails per listing page times active visitors),
+  which in turn correlated with intermittent Thumbor 400s on aaf-6
+  prod.  50 covers `asyncio.to_thread`'s default executor
+  (`min(32, cpu+4)`) plus headroom.
+  Fixes [#6](https://github.com/bluedynamics/zodb-pgjsonb-thumborblobloader/issues/6).
+
 ## 0.4.2 (2026-04-02)
 
 - Fix: S3 loader now reads `PGTHUMBOR_S3_ACCESS_KEY` and `PGTHUMBOR_S3_SECRET_KEY`

src/zodb_pgjsonb_thumborblobloader/s3.py

@@ -6,6 +6,7 @@
 
 from __future__ import annotations
 
+from botocore.config import Config
 from botocore.exceptions import ClientError
 
 import asyncio
@@ -15,6 +16,13 @@
 
 logger = logging.getLogger(__name__)
 
+# boto3's default urllib3 max_pool_connections is 10 — too low for
+# concurrent Thumbor image loads (30 thumbnails per listing page times
+# active visitors easily exceeds it, causing connection discard/reopen
+# churn and handshake-failure-induced 400s).  50 covers
+# asyncio.to_thread's default executor (min(32, cpu+4)) plus headroom.
+DEFAULT_MAX_POOL_CONNECTIONS = 50
+
 _s3_client = None
 _s3_config: tuple[str, str, str] | None = None
 
@@ -28,7 +36,15 @@ def _get_s3_client(bucket: str, region: str, endpoint: str = ""):
 
     import boto3
 
-    kwargs: dict = {"region_name": region}
+    max_pool = int(
+        os.environ.get(
+            "PGTHUMBOR_S3_MAX_POOL_CONNECTIONS", str(DEFAULT_MAX_POOL_CONNECTIONS)
+        )
+    )
+    kwargs: dict = {
+        "region_name": region,
+        "config": Config(max_pool_connections=max_pool),
+    }
     if endpoint:
         kwargs["endpoint_url"] = endpoint
     access_key = os.environ.get("PGTHUMBOR_S3_ACCESS_KEY", "")

tests/test_loader_s3.py

@@ -30,6 +30,34 @@ def _reset_s3_client():
     s3_mod._s3_config = None
 
 
+class TestS3ClientConfig:
+    """Verify the boto3 client is created with an explicit max_pool_connections."""
+
+    def test_default_max_pool_connections(self, monkeypatch):
+        from unittest.mock import patch
+        from zodb_pgjsonb_thumborblobloader import s3 as s3_mod
+
+        monkeypatch.delenv("PGTHUMBOR_S3_MAX_POOL_CONNECTIONS", raising=False)
+
+        with patch("boto3.client") as mock_client:
+            s3_mod._get_s3_client("bucket", "us-east-1")
+
+        kwargs = mock_client.call_args.kwargs
+        assert kwargs["config"].max_pool_connections == 50
+
+    def test_env_override_max_pool_connections(self, monkeypatch):
+        from unittest.mock import patch
+        from zodb_pgjsonb_thumborblobloader import s3 as s3_mod
+
+        monkeypatch.setenv("PGTHUMBOR_S3_MAX_POOL_CONNECTIONS", "128")
+
+        with patch("boto3.client") as mock_client:
+            s3_mod._get_s3_client("bucket", "us-east-1")
+
+        kwargs = mock_client.call_args.kwargs
+        assert kwargs["config"].max_pool_connections == 128
+
+
 class TestLoadS3Blob:
     """Test loading blobs from S3 via s3_key."""