fix: respect HTTP(S)_PROXY env vars for httpx client (#2632)

Author: haakonvt

cognite/client/_http_client.py

@@ -53,30 +53,17 @@ def get_global_async_httpx_client() -> httpx.AsyncClient:
         return _global_async_httpx_clients[loop]
     except KeyError:
         pass
-    async_transport = httpx.AsyncHTTPTransport(
+
+    client = _global_async_httpx_clients[loop] = httpx.AsyncClient(
         proxy=global_config.proxy,
-        retries=0,  # 'retries': The maximum number of retries when trying to establish a connection.
         verify=not global_config.disable_ssl,
         limits=httpx.Limits(
-            # max_connections: The maximum number of concurrent HTTP connections that
-            #     the pool should allow. Any attempt to send a request on a pool that
-            #     would exceed this amount will block until a connection is available.
-            # max_keepalive_connections: The maximum number of idle HTTP connections
-            #     that will be maintained in the pool.
-            # keepalive_expiry: The duration in seconds that an idle HTTP connection
-            #     may be maintained for before being expired from the pool.
             max_connections=global_config.max_connection_pool_size,
-            max_keepalive_connections=None,  # defaults to match max_connections
-            keepalive_expiry=5,  # copy httpx default
+            max_keepalive_connections=None,
+            keepalive_expiry=5,
         ),
-    )
-    client = _global_async_httpx_clients[loop] = httpx.AsyncClient(
-        transport=async_transport,
         follow_redirects=global_config.follow_redirects,
         cookies=NoCookiesPlease(),
-        # Below should not be needed when we pass transport, but... :)
-        proxy=global_config.proxy,
-        verify=not global_config.disable_ssl,
     )
     return client
 

tests/tests_unit/test_http_client.py

@@ -1,9 +1,19 @@
 from __future__ import annotations
 
+import ssl
+from collections.abc import Iterator
+
 import httpx
 import pytest
 
-from cognite.client._http_client import AsyncHTTPClientWithRetryConfig, RetryTracker
+from cognite.client._http_client import (
+    AsyncHTTPClientWithRetryConfig,
+    NoCookiesPlease,
+    RetryTracker,
+    _global_async_httpx_clients,
+    get_global_async_httpx_client,
+)
+from cognite.client.config import global_config
 
 
 @pytest.fixture
@@ -107,3 +117,57 @@ def test_is_auto_retryable(self, default_config: AsyncHTTPClientWithRetryConfig)
         # 409 is not in the list of status codes to retry, but we set is_auto_retryable=True, which should override it
         assert rt.should_retry_status_code(make_http_status_error(409), is_auto_retryable=True) is True
         assert rt.should_retry_status_code(make_http_status_error(409), is_auto_retryable=False) is False
+
+
+@pytest.fixture
+def _clean_global_httpx_clients() -> Iterator[None]:
+    _global_async_httpx_clients.clear()
+    yield
+    _global_async_httpx_clients.clear()
+
+
+@pytest.mark.usefixtures("_clean_global_httpx_clients")
+class TestGetGlobalAsyncHttpxClient:
+    async def test_no_proxy_when_env_unset_and_config_unset(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        # Ensure test env does not randomly have these set:
+        monkeypatch.delenv("HTTPS_PROXY", raising=False)
+        monkeypatch.delenv("HTTP_PROXY", raising=False)
+
+        client = get_global_async_httpx_client()
+
+        assert not client._mounts
+
+    async def test_env_proxy_is_respected(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("HTTPS_PROXY", "http://magicenvproxy:666")
+
+        client = get_global_async_httpx_client()
+
+        assert len(client._mounts) == 1
+
+        # If the below asserts fail due to httpx/httpcore private API changes, just keep the assert above.
+        (transport,) = client._mounts.values()
+        assert transport._pool._proxy_url.host == b"magicenvproxy"  # type: ignore[union-attr]
+        assert transport._pool._proxy_url.port == 666  # type: ignore[union-attr]
+
+    async def test_explicit_proxy_config_still_works(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(global_config, "proxy", "http://explicit:1234")
+
+        client = get_global_async_httpx_client()
+
+        assert client._mounts
+
+    async def test_client_settings_match_global_config(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        """Verify pool limits, SSL, redirect, and cookie settings are forwarded correctly."""
+        monkeypatch.setattr(global_config, "max_connection_pool_size", 42)
+        monkeypatch.setattr(global_config, "disable_ssl", True)
+        monkeypatch.setattr(global_config, "follow_redirects", True)
+        client = get_global_async_httpx_client()
+
+        assert client.follow_redirects is True
+        assert isinstance(client._cookies.jar, NoCookiesPlease)
+
+        pool = client._transport._pool  # type: ignore[attr-defined]
+        assert pool._max_connections == 42
+        assert pool._keepalive_expiry == 5
+        assert pool._ssl_context.verify_mode == ssl.CERT_NONE  # disable_ssl should cause this
+        assert pool._ssl_context.check_hostname is False