From 1f3396a0cd660604d2076cfc55a7020a112b34f3 Mon Sep 17 00:00:00 2001 From: Zhihan Jiang Date: Sat, 11 Jul 2026 18:17:29 -0700 Subject: [PATCH] fix: reserve ephemeral-port headroom for the auto max-connections default max_connections=-1 resolved to the entire ephemeral-port range (system_maximum_ports x distinct_endpoints). Establishing connections in a burst (e.g. max_throughput at start-up) then tries to bind every ephemeral port at once; with recently-closed ports held in TIME_WAIT, the tail of bind() calls intermittently fails with EADDRNOTAVAIL. Resolve the auto limit to a fraction of the budget (AUTO_MAX_CONNECTIONS_BUDGET_FRACTION = 0.5) so the zero-config default reserves headroom; an explicit max_connections may still use the full budget. Remove the now-dead get_ephemeral_port_limit / get_used_port_count. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../endpoint_client/config.py | 16 +++++- .../endpoint_client/utils.py | 51 ------------------- .../test_http_client_config.py | 36 ++++++++----- 3 files changed, 38 insertions(+), 65 deletions(-) diff --git a/src/inference_endpoint/endpoint_client/config.py b/src/inference_endpoint/endpoint_client/config.py index a0198cb5c..0e3c35705 100644 --- a/src/inference_endpoint/endpoint_client/config.py +++ b/src/inference_endpoint/endpoint_client/config.py @@ -76,6 +76,16 @@ def _endpoint_destination(url: str) -> tuple[str | None, int]: } +# The auto (``max_connections=-1``) pool claims this fraction of the ephemeral-port +# budget rather than the whole range. Binding the entire range leaves the OS no +# headroom: under bursty connection establishment the transient TIME_WAIT backlog +# occupies much of the range, so binding new sockets intermittently fails with +# EADDRNOTAVAIL. The reserved margin keeps the zero-config default safe; the +# remainder is still a large pool and scales with distinct endpoints. An explicit +# ``max_connections`` may use the full budget. +AUTO_MAX_CONNECTIONS_BUDGET_FRACTION = 0.5 + + class HTTPClientConfig(WithUpdatesMixin, BaseModel): """HTTP endpoint client configuration. @@ -280,7 +290,11 @@ def _resolve_defaults(self) -> HTTPClientConfig: port_budget = system_maximum_ports * max(1, distinct_endpoints) if self.max_connections == -1: - object.__setattr__(self, "max_connections", port_budget) + object.__setattr__( + self, + "max_connections", + max(1, int(port_budget * AUTO_MAX_CONNECTIONS_BUDGET_FRACTION)), + ) elif self.max_connections > 0: if self.max_connections > port_budget: raise RuntimeError( diff --git a/src/inference_endpoint/endpoint_client/utils.py b/src/inference_endpoint/endpoint_client/utils.py index 50fb056c7..c1ebf6b0c 100644 --- a/src/inference_endpoint/endpoint_client/utils.py +++ b/src/inference_endpoint/endpoint_client/utils.py @@ -57,54 +57,3 @@ def get_ephemeral_port_range() -> tuple[int, int]: return 49152, 65535 raise OSError(f"Ephemeral port range detection is not supported on {sys.platform}.") - - -def get_used_port_count() -> int: - """Count TCP sockets using ephemeral ports. - On Linux, reads from /proc/net/tcp and /proc/net/tcp6. - - Returns: - Number of TCP sockets using ephemeral ports. - """ - if _IS_DARWIN: - return 0 - if not _IS_LINUX: - raise OSError(f"TCP socket counting is not supported on {sys.platform}.") - - low, high = get_ephemeral_port_range() - count = 0 - - for proc_file in ("/proc/net/tcp", "/proc/net/tcp6"): - try: - with open(proc_file) as f: - next(f) # Skip header - for line in f: - # local_address is second field: "IP:PORT" in hex - parts = line.split() - if len(parts) < 2: - continue - local_addr = parts[1] # e.g., "0100007F:1F90" - port_hex = local_addr.split(":")[1] - port = int(port_hex, 16) - if low <= port <= high: - count += 1 - except (OSError, ValueError, IndexError): - pass - - return count - - -def get_ephemeral_port_limit() -> int: - """Get the number of available ephemeral ports. - - Reads the configured port range and subtracts currently used ports - to determine how many new connections can be established. - - Returns: - Number of available ephemeral ports. - """ - low, high = get_ephemeral_port_range() - total_range = high - low + 1 - used = get_used_port_count() - available = max(0, total_range - used) - return available diff --git a/tests/unit/endpoint_client/test_http_client_config.py b/tests/unit/endpoint_client/test_http_client_config.py index 220e4b367..8ba4e826e 100644 --- a/tests/unit/endpoint_client/test_http_client_config.py +++ b/tests/unit/endpoint_client/test_http_client_config.py @@ -46,6 +46,7 @@ def test_http_client_config_constructs_when_numa_unsupported(self): assert c.num_workers == 10 +@pytest.mark.unit class TestEndpointBudgetScaling: """max_connections budget scales with the number of distinct endpoints. @@ -64,29 +65,23 @@ def test_auto_budget_scales_with_distinct_endpoints(self): ], num_workers=10, ) - assert c.max_connections == 30000 # 10000 ports x 3 distinct endpoints + assert c.max_connections == 15000 # 3 x 10000 budget, auto claims half - def test_single_endpoint_budget_unchanged(self): + def test_single_endpoint_auto_reserves_headroom(self): with patch.object(cfg, "get_ephemeral_port_range", return_value=(1, 10000)): c = cfg.HTTPClientConfig( endpoint_urls=["http://10.0.0.1:8000"], num_workers=10 ) - assert c.max_connections == 10000 # single endpoint -> unchanged + assert c.max_connections == 5000 # 10000 budget, auto claims half - def test_live_socket_usage_does_not_reject_overall_connection_limit(self): - with ( - patch.object(cfg, "get_ephemeral_port_range", return_value=(1, 10000)), - patch.object( - cfg, "get_ephemeral_port_limit", return_value=0, create=True - ) as live_port_limit, - ): + def test_explicit_max_connections_below_budget_unchanged(self): + with patch.object(cfg, "get_ephemeral_port_range", return_value=(1, 10000)): c = cfg.HTTPClientConfig( endpoint_urls=["http://10.0.0.1:8000"], num_workers=10, max_connections=10, ) assert c.max_connections == 10 - live_port_limit.assert_not_called() def test_duplicate_endpoints_do_not_inflate_budget(self): # Same (host, port) repeated (even with different paths) is one @@ -100,7 +95,7 @@ def test_duplicate_endpoints_do_not_inflate_budget(self): ], num_workers=10, ) - assert c.max_connections == 10000 # 1 distinct (host, port) + assert c.max_connections == 5000 # 1 distinct (host, port), auto claims half def test_explicit_max_connections_within_scaled_budget_ok(self): # 25000 exceeds one endpoint's budget (10000) but fits 3 (30000). @@ -125,6 +120,21 @@ def test_explicit_max_connections_exceeding_scaled_budget_raises(self): max_connections=40000, # > 2 x 10000 ) + def test_auto_reserves_headroom_but_explicit_may_use_full_budget(self): + # Auto (-1) leaves the OS headroom; an explicit value may still claim + # the full per-endpoint budget. + with patch.object(cfg, "get_ephemeral_port_range", return_value=(1, 10000)): + auto = cfg.HTTPClientConfig( + endpoint_urls=["http://10.0.0.1:8000"], num_workers=10 + ) + full = cfg.HTTPClientConfig( + endpoint_urls=["http://10.0.0.1:8000"], + num_workers=10, + max_connections=10000, + ) + assert auto.max_connections < 10000 + assert full.max_connections == 10000 + @pytest.mark.unit class TestEndpointDestination: @@ -161,4 +171,4 @@ def test_schemeless_budget_scales_with_distinct_hosts(self): endpoint_urls=["10.0.0.1:8000", "10.0.0.2:8000"], num_workers=10, ) - assert c.max_connections == 20000 # 10000 ports x 2 distinct hosts + assert c.max_connections == 10000 # 2 x 10000 budget, auto claims half