diff --git a/changes/12979.fix.md b/changes/12979.fix.md new file mode 100644 index 00000000000..05b4b7a927c --- /dev/null +++ b/changes/12979.fix.md @@ -0,0 +1 @@ +Fix a native memory and file-descriptor leak in all long-running components making aiohttp client requests: since aiodns 3.2, aiohttp defaulted to the aiodns/pycares resolver, and every ephemeral client session leaked a c-ares channel (native heap growth plus one `/dev/urandom` fd per channel, ~85k fds on 16-day-old agents). aiodns/pycares are removed so aiohttp uses its threaded resolver, and the boot-time self-IP lookup now uses the stdlib `loop.getaddrinfo()`. diff --git a/python.lock b/python.lock index 0f7646a0628..9285d34e060 100644 --- a/python.lock +++ b/python.lock @@ -59,18 +59,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/29/80a0a91bd35b46bf31dc53a620780ee7c5b48b4dab7e60fdc4979641846a/aiodataloader-0.4.3-py3-none-any.whl", hash = "sha256:f2d57675e4c7a5cf7efc4c42697d307b951e1a9f40c22df3531a4b9cb7758229", size = 11029, upload-time = "2025-11-29T10:14:09.417Z" }, ] -[[package]] -name = "aiodns" -version = "3.2.0" -source = { registry = "https://pypi.org/simple/" } -dependencies = [ - { name = "pycares" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/84/41a6a2765abc124563f5380e76b9b24118977729e25a84112f8dfb2b33dc/aiodns-3.2.0.tar.gz", hash = "sha256:62869b23409349c21b072883ec8998316b234c9a9e36675756e8e317e8768f72", size = 7823, upload-time = "2024-03-31T11:27:30.639Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/14/13c65b1bd59f7e707e0cc0964fbab45c003f90292ed267d159eeeeaa2224/aiodns-3.2.0-py3-none-any.whl", hash = "sha256:e443c0c27b07da3174a109fd9e736d69058d808f144d3c9d56dbd1776964c5f5", size = 5735, upload-time = "2024-03-31T11:27:28.615Z" }, -] - [[package]] name = "aiodocker" version = "0.27.0" @@ -1833,7 +1821,6 @@ source = { virtual = "." } dependencies = [ { name = "aioboto3" }, { name = "aiodataloader" }, - { name = "aiodns" }, { name = "aiodocker" }, { name = "aiofiles" }, { name = "aiohttp" }, @@ -1903,7 +1890,6 @@ dependencies = [ { name = "pexpect" }, { name = "prometheus-client" }, { name = "psutil" }, - { name = "pycares" }, { name = "pycryptodome" }, { name = "pydantic", extra = ["email"] }, { name = "pyhumps" }, @@ -1965,7 +1951,6 @@ dependencies = [ requires-dist = [ { name = "aioboto3", specifier = "~=15.0.0" }, { name = "aiodataloader", specifier = "~=0.4.2" }, - { name = "aiodns", specifier = "==3.2" }, { name = "aiodocker", specifier = "==0.27.0" }, { name = "aiofiles", specifier = "~=24.1.0" }, { name = "aiohttp", specifier = "~=3.13.3" }, @@ -2035,7 +2020,6 @@ requires-dist = [ { name = "pexpect", specifier = "~=4.8" }, { name = "prometheus-client", specifier = "~=0.21.1" }, { name = "psutil", specifier = "~=7.0" }, - { name = "pycares", specifier = "~=4.11.0" }, { name = "pycryptodome", specifier = ">=3.20.0" }, { name = "pydantic", extras = ["email"], specifier = "~=2.11.3" }, { name = "pyhumps", specifier = "~=3.8.0" }, @@ -2270,30 +2254,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] -[[package]] -name = "pycares" -version = "4.11.0" -source = { registry = "https://pypi.org/simple/" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8d/ad/9d1e96486d2eb5a2672c4d9a2dd372d015b8d7a332c6ac2722c4c8e6bbbf/pycares-4.11.0.tar.gz", hash = "sha256:c863d9003ca0ce7df26429007859afd2a621d3276ed9fef154a9123db9252557", size = 654473, upload-time = "2025-09-09T15:18:21.849Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/a9/62fea7ad72ac1fed2ac9dd8e9a7379b7eb0288bf2b3ea5731642c3a6f7de/pycares-4.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c296ab94d1974f8d2f76c499755a9ce31ffd4986e8898ef19b90e32525f7d84", size = 145909, upload-time = "2025-09-09T15:17:10.491Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/0317d6d0d3bd7599c53b8f1db09ad04260647d2f6842018e322584791fd5/pycares-4.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0fcd3a8bac57a0987d9b09953ba0f8703eb9dca7c77f7051d8c2ed001185be8", size = 141974, upload-time = "2025-09-09T15:17:11.634Z" }, - { url = "https://files.pythonhosted.org/packages/63/11/731b565ae1e81c43dac247a248ee204628186f6df97c9927bd06c62237f8/pycares-4.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:bac55842047567ddae177fb8189b89a60633ac956d5d37260f7f71b517fd8b87", size = 637796, upload-time = "2025-09-09T15:17:12.815Z" }, - { url = "https://files.pythonhosted.org/packages/f5/30/a2631fe2ffaa85475cdbff7df1d9376bc0b2a6ae77ca55d53233c937a5da/pycares-4.11.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:4da2e805ed8c789b9444ef4053f6ef8040cd13b0c1ca6d3c4fe6f9369c458cb4", size = 687734, upload-time = "2025-09-09T15:17:14.015Z" }, - { url = "https://files.pythonhosted.org/packages/a9/b7/b3a5f99d4ab776662e71d5a56e8f6ea10741230ff988d1f502a8d429236b/pycares-4.11.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:ea785d1f232b42b325578f0c8a2fa348192e182cc84a1e862896076a4a2ba2a7", size = 678320, upload-time = "2025-09-09T15:17:15.442Z" }, - { url = "https://files.pythonhosted.org/packages/ea/77/a00d962b90432993afbf3bd05da8fe42117e0d9037cd7fd428dc41094d7b/pycares-4.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:aa160dc9e785212c49c12bb891e242c949758b99542946cc8e2098ef391f93b0", size = 641012, upload-time = "2025-09-09T15:17:16.728Z" }, - { url = "https://files.pythonhosted.org/packages/c6/fb/9266979ba59d37deee1fd74452b2ae32a7395acafe1bee510ac023c6c9a5/pycares-4.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7830709c23bbc43fbaefbb3dde57bdd295dc86732504b9d2e65044df8fd5e9fb", size = 622363, upload-time = "2025-09-09T15:17:17.835Z" }, - { url = "https://files.pythonhosted.org/packages/91/c2/16dbc3dc33781a3c79cbdd76dd1cda808d98ba078d9a63a725d6a1fad181/pycares-4.11.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ef1ab7abbd238bb2dbbe871c3ea39f5a7fc63547c015820c1e24d0d494a1689", size = 670294, upload-time = "2025-09-09T15:17:19.214Z" }, - { url = "https://files.pythonhosted.org/packages/ff/75/f003905e55298a6dd5e0673a2dc11e31518a5141393b925dc05fcaba9fb4/pycares-4.11.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a4060d8556c908660512d42df1f4a874e4e91b81f79e3a9090afedc7690ea5ba", size = 652973, upload-time = "2025-09-09T15:17:20.388Z" }, - { url = "https://files.pythonhosted.org/packages/55/2a/eafb235c371979e11f8998d686cbaa91df6a84a34ffe4d997dfe57c45445/pycares-4.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a98fac4a3d4f780817016b6f00a8a2c2f41df5d25dfa8e5b1aa0d783645a6566", size = 629235, upload-time = "2025-09-09T15:17:21.92Z" }, - { url = "https://files.pythonhosted.org/packages/05/99/60f19eb1c8eb898882dd8875ea51ad0aac3aff5780b27247969e637cc26a/pycares-4.11.0-cp313-cp313-win32.whl", hash = "sha256:faa8321bc2a366189dcf87b3823e030edf5ac97a6b9a7fc99f1926c4bf8ef28e", size = 118918, upload-time = "2025-09-09T15:17:23.327Z" }, - { url = "https://files.pythonhosted.org/packages/2a/14/bc89ad7225cba73068688397de09d7cad657d67b93641c14e5e18b88e685/pycares-4.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:6f74b1d944a50fa12c5006fd10b45e1a45da0c5d15570919ce48be88e428264c", size = 144556, upload-time = "2025-09-09T15:17:24.341Z" }, - { url = "https://files.pythonhosted.org/packages/af/88/4309576bd74b5e6fc1f39b9bc5e4b578df2cadb16bdc026ac0cc15663763/pycares-4.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:4b6f7581793d8bb3014028b8397f6f80b99db8842da58f4409839c29b16397ad", size = 115692, upload-time = "2025-09-09T15:17:25.637Z" }, -] - [[package]] name = "pycparser" version = "3.0" diff --git a/python.lock.metadata b/python.lock.metadata index 9ed17cdd959..0762d330ddc 100644 --- a/python.lock.metadata +++ b/python.lock.metadata @@ -10,7 +10,6 @@ "SQLAlchemy[postgresql_asyncpg]~=2.0.45", "aioboto3~=15.0.0", "aiodataloader~=0.4.2", - "aiodns==3.2", "aiodocker==0.27.0", "aiofiles~=24.1.0", "aiohttp_cors~=0.8.1", @@ -79,7 +78,6 @@ "pexpect~=4.8", "prometheus-client~=0.21.1", "psutil~=7.0", - "pycares~=4.11.0", "pycryptodome>=3.20.0", "pydantic[email]~=2.11.3", "pyhumps~=3.8.0", diff --git a/requirements.txt b/requirements.txt index 9adbbae635f..b5ebe96092b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ aiohttp_cors~=0.8.1 aiohttp_remotes~=1.3.0 aiohttp_jinja2~=1.6 aiohttp_sse>=2.2 -aiodns==3.2 aiomonitor~=0.7.0 aioresponses>=0.7.3 aiosqlite~=0.21.0 @@ -63,7 +62,6 @@ opentelemetry-instrumentation-logging~=0.60b0 pexpect~=4.8 prometheus-client~=0.21.1 psutil~=7.0 -pycares~=4.11.0 pycryptodome>=3.20.0 habana-pyhlml~=1.22.1.6 pyhumps~=3.8.0 diff --git a/src/ai/backend/common/BUILD b/src/ai/backend/common/BUILD index 9798d45219b..12d86280fbb 100644 --- a/src/ai/backend/common/BUILD +++ b/src/ai/backend/common/BUILD @@ -4,7 +4,6 @@ python_sources( dependencies=[ ":resources", "stubs/trafaret:stubs", - "//:reqs#pycares", # aiodns runtime dependency - pin version explicitly ], ) diff --git a/src/ai/backend/common/identity.py b/src/ai/backend/common/identity.py index e7ba1732942..302a4e962ce 100644 --- a/src/ai/backend/common/identity.py +++ b/src/ai/backend/common/identity.py @@ -17,7 +17,6 @@ from pathlib import Path, PosixPath from typing import Any -import aiodns import aiohttp import ifaddr import psutil @@ -395,11 +394,16 @@ async def _get_instance_ip(_subnet_hint: BaseIPNetwork[Any] | None = None) -> st ) try: myself = socket.gethostname() - resolver = aiodns.DNSResolver() - result = await resolver.gethostbyname(myself, socket.AF_INET) - address: str = result.addresses[0] + loop = asyncio.get_running_loop() + addrinfo = await loop.getaddrinfo( + myself, + None, + family=socket.AF_INET, + type=socket.SOCK_STREAM, + ) + address: str = addrinfo[0][4][0] return address - except aiodns.error.DNSError: + except (socket.gaierror, IndexError): return "127.0.0.1" async def _get_instance_type() -> str: diff --git a/tests/unit/common/test_identity.py b/tests/unit/common/test_identity.py index 7b1dcab5801..cd8e6d32660 100644 --- a/tests/unit/common/test_identity.py +++ b/tests/unit/common/test_identity.py @@ -9,7 +9,6 @@ from typing import Any from unittest.mock import AsyncMock, MagicMock, patch -import aiodns import aiohttp import pytest from aioresponses import aioresponses @@ -159,28 +158,26 @@ async def test_get_instance_ip(provider: str | None) -> None: ret = await ai.backend.common.identity.get_instance_ip(None) assert ret == random_ip elif provider is None: - mocked_ares_host_result = MagicMock() - mocked_ares_host_result.addresses = ["10.1.2.3"] - mocked_resolver = MagicMock() - async def coro_return_mocked_result(*args: Any) -> MagicMock: - return mocked_ares_host_result + async def coro_return_addrinfo(*args: Any, **kwargs: Any) -> list[Any]: + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.1.2.3", 0))] - mocked_resolver.gethostbyname = coro_return_mocked_result + mocked_loop = MagicMock() + mocked_loop.getaddrinfo = coro_return_addrinfo with ( - patch("aiodns.DNSResolver", return_value=mocked_resolver), + patch("asyncio.get_running_loop", return_value=mocked_loop), patch("socket.gethostname", return_value="myname"), ): ret = await ai.backend.common.identity.get_instance_ip(None) assert ret == "10.1.2.3" - async def coro_raise_error(*args: Any) -> None: - raise aiodns.error.DNSError("domain not found") + async def coro_raise_error(*args: Any, **kwargs: Any) -> None: + raise socket.gaierror(socket.EAI_NONAME, "domain not found") - mocked_resolver = MagicMock() - mocked_resolver.gethostbyname = coro_raise_error + mocked_loop = MagicMock() + mocked_loop.getaddrinfo = coro_raise_error with ( - patch("aiodns.DNSResolver", return_value=mocked_resolver), + patch("asyncio.get_running_loop", return_value=mocked_loop), patch("socket.gethostname", return_value="myname"), ): ret = await ai.backend.common.identity.get_instance_ip(None)