chore: drop the now-unused aiodns/pycares dependencies (main only) (#12979)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Sanghun Lee <sanghun@lablup.com>

Author: 3 people

changes/12979.fix.md

@@ -0,0 +1 @@
+Fix a native memory and file-descriptor leak in all long-running components making aiohttp client requests: since aiodns 3.2, aiohttp defaulted to the aiodns/pycares resolver, and every ephemeral client session leaked a c-ares channel (native heap growth plus one `/dev/urandom` fd per channel, ~85k fds on 16-day-old agents). aiodns/pycares are removed so aiohttp uses its threaded resolver, and the boot-time self-IP lookup now uses the stdlib `loop.getaddrinfo()`.

python.lock

@@ -59,18 +59,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c6/29/80a0a91bd35b46bf31dc53a620780ee7c5b48b4dab7e60fdc4979641846a/aiodataloader-0.4.3-py3-none-any.whl", hash = "sha256:f2d57675e4c7a5cf7efc4c42697d307b951e1a9f40c22df3531a4b9cb7758229", size = 11029, upload-time = "2025-11-29T10:14:09.417Z" },
 ]
 
-[[package]]
-name = "aiodns"
-version = "3.2.0"
-source = { registry = "https://pypi.org/simple/" }
-dependencies = [
-    { name = "pycares" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e7/84/41a6a2765abc124563f5380e76b9b24118977729e25a84112f8dfb2b33dc/aiodns-3.2.0.tar.gz", hash = "sha256:62869b23409349c21b072883ec8998316b234c9a9e36675756e8e317e8768f72", size = 7823, upload-time = "2024-03-31T11:27:30.639Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/15/14/13c65b1bd59f7e707e0cc0964fbab45c003f90292ed267d159eeeeaa2224/aiodns-3.2.0-py3-none-any.whl", hash = "sha256:e443c0c27b07da3174a109fd9e736d69058d808f144d3c9d56dbd1776964c5f5", size = 5735, upload-time = "2024-03-31T11:27:28.615Z" },
-]
-
 [[package]]
 name = "aiodocker"
 version = "0.27.0"
@@ -1833,7 +1821,6 @@ source = { virtual = "." }
 dependencies = [
     { name = "aioboto3" },
     { name = "aiodataloader" },
-    { name = "aiodns" },
     { name = "aiodocker" },
     { name = "aiofiles" },
     { name = "aiohttp" },
@@ -1903,7 +1890,6 @@ dependencies = [
     { name = "pexpect" },
     { name = "prometheus-client" },
     { name = "psutil" },
-    { name = "pycares" },
     { name = "pycryptodome" },
     { name = "pydantic", extra = ["email"] },
     { name = "pyhumps" },
@@ -1965,7 +1951,6 @@ dependencies = [
 requires-dist = [
     { name = "aioboto3", specifier = "~=15.0.0" },
     { name = "aiodataloader", specifier = "~=0.4.2" },
-    { name = "aiodns", specifier = "==3.2" },
     { name = "aiodocker", specifier = "==0.27.0" },
     { name = "aiofiles", specifier = "~=24.1.0" },
     { name = "aiohttp", specifier = "~=3.13.3" },
@@ -2035,7 +2020,6 @@ requires-dist = [
     { name = "pexpect", specifier = "~=4.8" },
     { name = "prometheus-client", specifier = "~=0.21.1" },
     { name = "psutil", specifier = "~=7.0" },
-    { name = "pycares", specifier = "~=4.11.0" },
     { name = "pycryptodome", specifier = ">=3.20.0" },
     { name = "pydantic", extras = ["email"], specifier = "~=2.11.3" },
     { name = "pyhumps", specifier = "~=3.8.0" },
@@ -2270,30 +2254,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" },
 ]
 
-[[package]]
-name = "pycares"
-version = "4.11.0"
-source = { registry = "https://pypi.org/simple/" }
-dependencies = [
-    { name = "cffi" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/ad/9d1e96486d2eb5a2672c4d9a2dd372d015b8d7a332c6ac2722c4c8e6bbbf/pycares-4.11.0.tar.gz", hash = "sha256:c863d9003ca0ce7df26429007859afd2a621d3276ed9fef154a9123db9252557", size = 654473, upload-time = "2025-09-09T15:18:21.849Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/dc/a9/62fea7ad72ac1fed2ac9dd8e9a7379b7eb0288bf2b3ea5731642c3a6f7de/pycares-4.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c296ab94d1974f8d2f76c499755a9ce31ffd4986e8898ef19b90e32525f7d84", size = 145909, upload-time = "2025-09-09T15:17:10.491Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/ac/0317d6d0d3bd7599c53b8f1db09ad04260647d2f6842018e322584791fd5/pycares-4.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0fcd3a8bac57a0987d9b09953ba0f8703eb9dca7c77f7051d8c2ed001185be8", size = 141974, upload-time = "2025-09-09T15:17:11.634Z" },
-    { url = "https://files.pythonhosted.org/packages/63/11/731b565ae1e81c43dac247a248ee204628186f6df97c9927bd06c62237f8/pycares-4.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:bac55842047567ddae177fb8189b89a60633ac956d5d37260f7f71b517fd8b87", size = 637796, upload-time = "2025-09-09T15:17:12.815Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/30/a2631fe2ffaa85475cdbff7df1d9376bc0b2a6ae77ca55d53233c937a5da/pycares-4.11.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:4da2e805ed8c789b9444ef4053f6ef8040cd13b0c1ca6d3c4fe6f9369c458cb4", size = 687734, upload-time = "2025-09-09T15:17:14.015Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/b7/b3a5f99d4ab776662e71d5a56e8f6ea10741230ff988d1f502a8d429236b/pycares-4.11.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:ea785d1f232b42b325578f0c8a2fa348192e182cc84a1e862896076a4a2ba2a7", size = 678320, upload-time = "2025-09-09T15:17:15.442Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/77/a00d962b90432993afbf3bd05da8fe42117e0d9037cd7fd428dc41094d7b/pycares-4.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:aa160dc9e785212c49c12bb891e242c949758b99542946cc8e2098ef391f93b0", size = 641012, upload-time = "2025-09-09T15:17:16.728Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/fb/9266979ba59d37deee1fd74452b2ae32a7395acafe1bee510ac023c6c9a5/pycares-4.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7830709c23bbc43fbaefbb3dde57bdd295dc86732504b9d2e65044df8fd5e9fb", size = 622363, upload-time = "2025-09-09T15:17:17.835Z" },
-    { url = "https://files.pythonhosted.org/packages/91/c2/16dbc3dc33781a3c79cbdd76dd1cda808d98ba078d9a63a725d6a1fad181/pycares-4.11.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ef1ab7abbd238bb2dbbe871c3ea39f5a7fc63547c015820c1e24d0d494a1689", size = 670294, upload-time = "2025-09-09T15:17:19.214Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/75/f003905e55298a6dd5e0673a2dc11e31518a5141393b925dc05fcaba9fb4/pycares-4.11.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a4060d8556c908660512d42df1f4a874e4e91b81f79e3a9090afedc7690ea5ba", size = 652973, upload-time = "2025-09-09T15:17:20.388Z" },
-    { url = "https://files.pythonhosted.org/packages/55/2a/eafb235c371979e11f8998d686cbaa91df6a84a34ffe4d997dfe57c45445/pycares-4.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a98fac4a3d4f780817016b6f00a8a2c2f41df5d25dfa8e5b1aa0d783645a6566", size = 629235, upload-time = "2025-09-09T15:17:21.92Z" },
-    { url = "https://files.pythonhosted.org/packages/05/99/60f19eb1c8eb898882dd8875ea51ad0aac3aff5780b27247969e637cc26a/pycares-4.11.0-cp313-cp313-win32.whl", hash = "sha256:faa8321bc2a366189dcf87b3823e030edf5ac97a6b9a7fc99f1926c4bf8ef28e", size = 118918, upload-time = "2025-09-09T15:17:23.327Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/14/bc89ad7225cba73068688397de09d7cad657d67b93641c14e5e18b88e685/pycares-4.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:6f74b1d944a50fa12c5006fd10b45e1a45da0c5d15570919ce48be88e428264c", size = 144556, upload-time = "2025-09-09T15:17:24.341Z" },
-    { url = "https://files.pythonhosted.org/packages/af/88/4309576bd74b5e6fc1f39b9bc5e4b578df2cadb16bdc026ac0cc15663763/pycares-4.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:4b6f7581793d8bb3014028b8397f6f80b99db8842da58f4409839c29b16397ad", size = 115692, upload-time = "2025-09-09T15:17:25.637Z" },
-]
-
 [[package]]
 name = "pycparser"
 version = "3.0"

python.lock.metadata

@@ -10,7 +10,6 @@
     "SQLAlchemy[postgresql_asyncpg]~=2.0.45",
     "aioboto3~=15.0.0",
     "aiodataloader~=0.4.2",
-    "aiodns==3.2",
     "aiodocker==0.27.0",
     "aiofiles~=24.1.0",
     "aiohttp_cors~=0.8.1",
@@ -79,7 +78,6 @@
     "pexpect~=4.8",
     "prometheus-client~=0.21.1",
     "psutil~=7.0",
-    "pycares~=4.11.0",
     "pycryptodome>=3.20.0",
     "pydantic[email]~=2.11.3",
     "pyhumps~=3.8.0",

requirements.txt

@@ -7,7 +7,6 @@ aiohttp_cors~=0.8.1
 aiohttp_remotes~=1.3.0
 aiohttp_jinja2~=1.6
 aiohttp_sse>=2.2
-aiodns==3.2
 aiomonitor~=0.7.0
 aioresponses>=0.7.3
 aiosqlite~=0.21.0
@@ -63,7 +62,6 @@ opentelemetry-instrumentation-logging~=0.60b0
 pexpect~=4.8
 prometheus-client~=0.21.1
 psutil~=7.0
-pycares~=4.11.0
 pycryptodome>=3.20.0
 habana-pyhlml~=1.22.1.6
 pyhumps~=3.8.0

src/ai/backend/common/BUILD

@@ -4,7 +4,6 @@ python_sources(
     dependencies=[
         ":resources",
         "stubs/trafaret:stubs",
-        "//:reqs#pycares",  # aiodns runtime dependency - pin version explicitly
     ],
 )
 

src/ai/backend/common/identity.py

@@ -17,7 +17,6 @@
 from pathlib import Path, PosixPath
 from typing import Any
 
-import aiodns
 import aiohttp
 import ifaddr
 import psutil
@@ -395,11 +394,16 @@ async def _get_instance_ip(_subnet_hint: BaseIPNetwork[Any] | None = None) -> st
                     )
                 try:
                     myself = socket.gethostname()
-                    resolver = aiodns.DNSResolver()
-                    result = await resolver.gethostbyname(myself, socket.AF_INET)
-                    address: str = result.addresses[0]
+                    loop = asyncio.get_running_loop()
+                    addrinfo = await loop.getaddrinfo(
+                        myself,
+                        None,
+                        family=socket.AF_INET,
+                        type=socket.SOCK_STREAM,
+                    )
+                    address: str = addrinfo[0][4][0]
                     return address
-                except aiodns.error.DNSError:
+                except (socket.gaierror, IndexError):
                     return "127.0.0.1"
 
             async def _get_instance_type() -> str:

tests/unit/common/test_identity.py

@@ -9,7 +9,6 @@
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
-import aiodns
 import aiohttp
 import pytest
 from aioresponses import aioresponses
@@ -159,28 +158,26 @@ async def test_get_instance_ip(provider: str | None) -> None:
             ret = await ai.backend.common.identity.get_instance_ip(None)
             assert ret == random_ip
         elif provider is None:
-            mocked_ares_host_result = MagicMock()
-            mocked_ares_host_result.addresses = ["10.1.2.3"]
-            mocked_resolver = MagicMock()
 
-            async def coro_return_mocked_result(*args: Any) -> MagicMock:
-                return mocked_ares_host_result
+            async def coro_return_addrinfo(*args: Any, **kwargs: Any) -> list[Any]:
+                return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.1.2.3", 0))]
 
-            mocked_resolver.gethostbyname = coro_return_mocked_result
+            mocked_loop = MagicMock()
+            mocked_loop.getaddrinfo = coro_return_addrinfo
             with (
-                patch("aiodns.DNSResolver", return_value=mocked_resolver),
+                patch("asyncio.get_running_loop", return_value=mocked_loop),
                 patch("socket.gethostname", return_value="myname"),
             ):
                 ret = await ai.backend.common.identity.get_instance_ip(None)
                 assert ret == "10.1.2.3"
 
-            async def coro_raise_error(*args: Any) -> None:
-                raise aiodns.error.DNSError("domain not found")
+            async def coro_raise_error(*args: Any, **kwargs: Any) -> None:
+                raise socket.gaierror(socket.EAI_NONAME, "domain not found")
 
-            mocked_resolver = MagicMock()
-            mocked_resolver.gethostbyname = coro_raise_error
+            mocked_loop = MagicMock()
+            mocked_loop.getaddrinfo = coro_raise_error
             with (
-                patch("aiodns.DNSResolver", return_value=mocked_resolver),
+                patch("asyncio.get_running_loop", return_value=mocked_loop),
                 patch("socket.gethostname", return_value="myname"),
             ):
                 ret = await ai.backend.common.identity.get_instance_ip(None)