From 5408af0bf77226550c3f9777b4933684866ee223 Mon Sep 17 00:00:00 2001 From: Sion Kang Date: Tue, 21 Jul 2026 14:17:27 +0900 Subject: [PATCH 1/2] fix: force aiohttp's ThreadedResolver to stop the c-ares channel leak Since aiodns 3.2, aiohttp selects the aiodns/pycares AsyncResolver as its default resolver whenever the package is importable. Every ephemeral ClientSession/TCPConnector then creates a c-ares channel that is never destroyed, which on long-running services accumulates native heap memory (~0.5 MB/h observed on an idle production agent via eBPF memleak) and one leaked /dev/urandom fd per channel (~85,600 open fds on 16.5-day-old agent workers, converging on fd exhaustion). Same root-cause family as aio-libs/aiodns#191. aiohttp offers no configuration knob for the resolver choice and passing resolver= at every connector creation site is easy to miss, so add force_threaded_dns_resolver() to ai.backend.common.networking and call it once at each service entrypoint before any connector is created (the override survives the aiotools worker fork). An A/B test on a production agent confirmed this eliminates both symptoms completely. This intentionally leaves the aiodns/pycares packages installed so the change is backportable without touching release-branch lockfiles; the dependency removal itself lands separately on main (#12979). Co-Authored-By: Claude Opus 4.8 --- src/ai/backend/account_manager/server.py | 2 + src/ai/backend/agent/server.py | 2 + src/ai/backend/appproxy/coordinator/server.py | 2 + src/ai/backend/appproxy/worker/server.py | 2 + src/ai/backend/common/networking.py | 27 +++++++++++++ src/ai/backend/manager/server.py | 2 + src/ai/backend/storage/server.py | 2 + src/ai/backend/web/server.py | 2 + tests/unit/common/test_networking.py | 40 +++++++++++++++++++ 9 files changed, 81 insertions(+) create mode 100644 tests/unit/common/test_networking.py diff --git a/src/ai/backend/account_manager/server.py b/src/ai/backend/account_manager/server.py index 03a36ebeb35..4ec454efb67 100644 --- a/src/ai/backend/account_manager/server.py +++ b/src/ai/backend/account_manager/server.py @@ -39,6 +39,7 @@ from ai.backend.common.metrics.multiprocess_setup import cleanup_prometheus_multiprocess_dir from ai.backend.common.metrics.profiler import Profiler, PyroscopeArgs from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.types import HostPortPair from ai.backend.common.utils import env_info from ai.backend.logging import BraceStyleAdapter, Logger, LogLevel @@ -431,6 +432,7 @@ def main( """ Start the account-manager service as a foreground process. """ + force_threaded_dns_resolver() log_level = LogLevel.DEBUG if debug else log_level server_config = load_config(config_path, log_level) diff --git a/src/ai/backend/agent/server.py b/src/ai/backend/agent/server.py index 314f3e77e04..f34047f1f1b 100644 --- a/src/ai/backend/agent/server.py +++ b/src/ai/backend/agent/server.py @@ -103,6 +103,7 @@ ) from ai.backend.common.metrics.metric import CommonMetricRegistry from ai.backend.common.metrics.profiler import Profiler, PyroscopeArgs +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.service_discovery.etcd_discovery.service_discovery import ( ETCDServiceDiscovery, ETCDServiceDiscoveryArgs, @@ -1717,6 +1718,7 @@ def main( log_level: LogLevel, ) -> int: """Start the agent service as a foreground process.""" + force_threaded_dns_resolver() if debug: log_level = LogLevel.DEBUG diff --git a/src/ai/backend/appproxy/coordinator/server.py b/src/ai/backend/appproxy/coordinator/server.py index 9152801c67c..0c6093428a9 100644 --- a/src/ai/backend/appproxy/coordinator/server.py +++ b/src/ai/backend/appproxy/coordinator/server.py @@ -99,6 +99,7 @@ from ai.backend.common.metrics.http import build_api_metric_middleware from ai.backend.common.metrics.multiprocess_setup import cleanup_prometheus_multiprocess_dir from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.service_discovery.event_publisher import ServiceDiscoveryEventPublisher from ai.backend.common.service_discovery.redis_discovery.service_discovery import ( RedisServiceDiscovery, @@ -1173,6 +1174,7 @@ def main(ctx: click.Context, config_path: Path | None, debug: bool, log_level: L """ Start the proxy-coordinator service as a foreground process. """ + force_threaded_dns_resolver() log_level = LogLevel.DEBUG if debug else log_level server_config = load_config(config_path, log_level) diff --git a/src/ai/backend/appproxy/worker/server.py b/src/ai/backend/appproxy/worker/server.py index 109b4e17de1..a13b026b725 100644 --- a/src/ai/backend/appproxy/worker/server.py +++ b/src/ai/backend/appproxy/worker/server.py @@ -100,6 +100,7 @@ from ai.backend.common.metrics.http import build_api_metric_middleware from ai.backend.common.metrics.multiprocess_setup import cleanup_prometheus_multiprocess_dir from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.service_discovery.event_publisher import ServiceDiscoveryEventPublisher from ai.backend.common.service_discovery.redis_discovery.service_discovery import ( RedisServiceDiscovery, @@ -982,6 +983,7 @@ def main(ctx: click.Context, config_path: Path, debug: bool, log_level: LogLevel """ Start the proxy-worker service as a foreground process. """ + force_threaded_dns_resolver() log_level = LogLevel.DEBUG if debug else log_level server_config = load_config(config_path, log_level) diff --git a/src/ai/backend/common/networking.py b/src/ai/backend/common/networking.py index 1138fba5dfe..5a79778550c 100644 --- a/src/ai/backend/common/networking.py +++ b/src/ai/backend/common/networking.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, TypeVar, overload import aiohttp +import aiohttp.connector +import aiohttp.resolver if TYPE_CHECKING: import yarl @@ -13,11 +15,36 @@ __all__ = ( "curl", "find_free_port", + "force_threaded_dns_resolver", ) T = TypeVar("T") +def force_threaded_dns_resolver() -> None: + """Make aiohttp use its ThreadedResolver regardless of aiodns presence. + + Since aiodns 3.2, aiohttp selects the aiodns/pycares ``AsyncResolver`` as + its default resolver whenever the package is importable. Every ephemeral + ``ClientSession``/``TCPConnector`` then creates a c-ares channel that is + never destroyed, leaking native heap memory and one ``/dev/urandom`` file + descriptor per channel on long-running services (see also + https://github.com/aio-libs/aiodns/issues/191 for the aiodns >= 3.3 + variant that additionally exhausts inotify watches). + + aiohttp offers no configuration knob for this choice and injecting + ``resolver=`` at every connector creation site is easy to miss, so we + override the module-level defaults once at service startup. Our outbound + HTTP traffic targets a small set of fixed hosts, making threaded + ``getaddrinfo`` resolution entirely sufficient. + """ + aiohttp.resolver.DefaultResolver = aiohttp.resolver.ThreadedResolver + # aiohttp.connector re-imports DefaultResolver at import time, so its copy + # must be overridden as well; setattr avoids mypy's attr-defined complaint + # about the non-re-exported name. + setattr(aiohttp.connector, "DefaultResolver", aiohttp.resolver.ThreadedResolver) + + @overload async def curl( url: str | yarl.URL, diff --git a/src/ai/backend/manager/server.py b/src/ai/backend/manager/server.py index a552ede36de..9539dbc610b 100644 --- a/src/ai/backend/manager/server.py +++ b/src/ai/backend/manager/server.py @@ -46,6 +46,7 @@ from ai.backend.common.metrics.metric import CommonMetricRegistry from ai.backend.common.metrics.multiprocess_setup import cleanup_prometheus_multiprocess_dir from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.utils import env_info from ai.backend.logging import BraceStyleAdapter, Logger, LogLevel from ai.backend.logging.otel import ( @@ -499,6 +500,7 @@ def main( """ Start the manager service as a foreground process. """ + force_threaded_dns_resolver() log_level = LogLevel.DEBUG if debug else log_level if config_path is None: diff --git a/src/ai/backend/storage/server.py b/src/ai/backend/storage/server.py index fd119fc3e43..a866485e06f 100644 --- a/src/ai/backend/storage/server.py +++ b/src/ai/backend/storage/server.py @@ -61,6 +61,7 @@ from ai.backend.common.metrics.multiprocess_setup import cleanup_prometheus_multiprocess_dir from ai.backend.common.metrics.profiler import Profiler, PyroscopeArgs from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.plugin import AbstractPlugin, BasePluginContext from ai.backend.common.runner.types import Runner from ai.backend.common.service_discovery.etcd_discovery.service_discovery import ( @@ -820,6 +821,7 @@ def main( debug: bool = False, ) -> int: """Start the storage-proxy service as a foreground process.""" + force_threaded_dns_resolver() log_level = LogLevel.DEBUG if debug else log_level try: local_config = load_local_config(config_path, log_level=log_level) diff --git a/src/ai/backend/web/server.py b/src/ai/backend/web/server.py index 07ac2f2bcfc..dd388737a52 100644 --- a/src/ai/backend/web/server.py +++ b/src/ai/backend/web/server.py @@ -66,6 +66,7 @@ from ai.backend.common.health_checker.types import ComponentId from ai.backend.common.middlewares.exception import general_exception_middleware from ai.backend.common.msgpack import DEFAULT_PACK_OPTS, DEFAULT_UNPACK_OPTS +from ai.backend.common.networking import force_threaded_dns_resolver from ai.backend.common.web.session import ( Session, extra_config_headers, @@ -1289,6 +1290,7 @@ def main( debug: bool, ) -> None: """Start the webui host service as a foreground process.""" + force_threaded_dns_resolver() # Delete this part when you remove --debug option raw_cfg = tomli.loads(Path(config_path).read_text(encoding="utf-8")) diff --git a/tests/unit/common/test_networking.py b/tests/unit/common/test_networking.py new file mode 100644 index 00000000000..a585952a669 --- /dev/null +++ b/tests/unit/common/test_networking.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import aiohttp.connector +import aiohttp.resolver +import pytest + +from ai.backend.common.networking import force_threaded_dns_resolver + + +@pytest.fixture +def restore_default_resolver() -> Iterator[None]: + orig_resolver = aiohttp.resolver.DefaultResolver + orig_connector: Any = vars(aiohttp.connector)["DefaultResolver"] + yield + aiohttp.resolver.DefaultResolver = orig_resolver + setattr(aiohttp.connector, "DefaultResolver", orig_connector) + + +def test_force_threaded_dns_resolver_overrides_defaults( + restore_default_resolver: None, +) -> None: + force_threaded_dns_resolver() + assert aiohttp.resolver.DefaultResolver is aiohttp.resolver.ThreadedResolver + assert vars(aiohttp.connector)["DefaultResolver"] is aiohttp.resolver.ThreadedResolver + + +async def test_new_connector_uses_threaded_resolver( + restore_default_resolver: None, +) -> None: + force_threaded_dns_resolver() + conn = aiohttp.TCPConnector() + try: + # The connector must not fall back to the aiodns/pycares AsyncResolver, + # whose per-instance c-ares channels leak on long-running services. + assert isinstance(conn._resolver, aiohttp.resolver.ThreadedResolver) + finally: + await conn.close() From 62c351f7695f5a43da8b3e961f6f564a380f9f4d Mon Sep 17 00:00:00 2001 From: Sion Kang Date: Tue, 21 Jul 2026 14:18:05 +0900 Subject: [PATCH 2/2] docs: add changelog for #12985 Co-Authored-By: Claude Opus 4.8 --- changes/12985.fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/12985.fix.md diff --git a/changes/12985.fix.md b/changes/12985.fix.md new file mode 100644 index 00000000000..e27ec2f5bd6 --- /dev/null +++ b/changes/12985.fix.md @@ -0,0 +1 @@ +Fix a native memory and file-descriptor leak in every long-running service making aiohttp client requests: since aiodns 3.2, aiohttp implicitly defaulted to the aiodns/pycares resolver, leaking one c-ares channel (native heap plus a `/dev/urandom` fd) per ephemeral client session. All service entrypoints now force aiohttp's threaded resolver at startup.