Run ty on typeshed stubs in CI (#16013)

jelle-openai · web-flow · commit c6d107067232 · 2026-07-15T11:22:16.000-07:00
Add a pinned ty check for the standard-library and third-party stubs across Python 3.10-3.14 and the supported target platforms. The runner honors stdlib/VERSIONS, resolves checked-in stub packages and their external dependencies, and avoids duplicate published stub packages that shadow local sources. Match pyright's policy for non-actionable override and deprecation diagnostics, add narrowly scoped ignores beside existing checker exceptions, and exclude only the obsolete requests and legacy distutils stubs. Check geopandas, seaborn, and shapely on every target version, with two existing pandas-stubs type-bound exceptions mirrored for ty. Also fix the remaining Windows-only dateutil builtin-name collision discovered by the new check. Closes #15999.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -82,6 +82,39 @@ jobs:
           --all \
           --verbosity=QUIET
 
+  ty:
+    name: "ty: Check stubs"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: ["linux", "win32", "darwin"]
+        # TODO: Add 3.15 once third-party runtime dependencies provide compatible wheels.
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - uses: astral-sh/setup-uv@v8.2.0
+        with:
+          version-file: "requirements-tests.txt"
+      - name: Install typeshed test-suite requirements
+        run: uv pip install -r requirements-tests.txt --system
+      - name: Create an isolated venv for testing
+        run: uv venv .venv
+      - name: Install third-party stub dependencies
+        run: |
+          PACKAGES=$(python tests/get_external_stub_requirements.py)
+          if [ -n "$PACKAGES" ]; then
+            uv pip install --python-version ${{ matrix.python-version }} $PACKAGES
+          fi
+          # Published stub packages can shadow the checked-in stubs when ty
+          # resolves their relative imports.
+          uv pip uninstall types-PyYAML types-pytz
+      - name: Run ty on all stubs
+        run: python tests/ty_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} --python=.venv
+
   pyright:
     name: "pyright: Run test cases"
     runs-on: ubuntu-latest
diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -2,6 +2,7 @@
 # be pinned to a specific version to make failure reproducible.
 mypy==2.3.0
 pyright==1.1.411
+ty==0.0.59
 
 # Libraries used by our various scripts.
 aiohttp==3.14.1
diff --git a/stdlib/_asyncio.pyi b/stdlib/_asyncio.pyi
@@ -53,7 +53,7 @@ else:
 # since the only reason why `asyncio.Future` is invariant is the `set_result()` method,
 # and `asyncio.Task.set_result()` always raises.
 @disjoint_base
-class Task(Future[_T_co]):  # type: ignore[type-var]  # pyright: ignore[reportInvalidTypeArguments]
+class Task(Future[_T_co]):  # type: ignore[type-var]  # pyright: ignore[reportInvalidTypeArguments]  # ty:ignore[invalid-generic-class]
     if sys.version_info >= (3, 12):
         def __init__(
             self,
diff --git a/stdlib/collections/__init__.pyi b/stdlib/collections/__init__.pyi
@@ -364,17 +364,17 @@ class _OrderedDictValuesView(ValuesView[_VT_co]):
 # pyright doesn't have a specific error code for subclassing error!
 @final
 @type_check_only
-class _odict_keys(dict_keys[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class _odict_keys(dict_keys[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[subclass-of-final-class]
     def __reversed__(self) -> Iterator[_KT_co]: ...
 
 @final
 @type_check_only
-class _odict_items(dict_items[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class _odict_items(dict_items[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[subclass-of-final-class]
     def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ...
 
 @final
 @type_check_only
-class _odict_values(dict_values[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class _odict_values(dict_values[_KT_co, _VT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[subclass-of-final-class]
     def __reversed__(self) -> Iterator[_VT_co]: ...
 
 @disjoint_base
diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi
@@ -43,7 +43,7 @@ _CM_EF = TypeVar("_CM_EF", bound=AbstractContextManager[Any, Any] | _ExitFunc)
 # At runtime it inherits from ABC and is not a Protocol, but it is on the
 # allowlist for use as a Protocol.
 @runtime_checkable
-class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[invalid-protocol]
     __slots__ = ()
     def __enter__(self) -> _T_co: ...
     @abstractmethod
@@ -55,7 +55,7 @@ class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ignore[m
 # At runtime it inherits from ABC and is not a Protocol, but it is on the
 # allowlist for use as a Protocol.
 @runtime_checkable
-class AbstractAsyncContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class AbstractAsyncContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[invalid-protocol]
     __slots__ = ()
     async def __aenter__(self) -> _T_co: ...
     @abstractmethod
diff --git a/stdlib/dataclasses.pyi b/stdlib/dataclasses.pyi
@@ -298,7 +298,7 @@ def fields(class_or_instance: DataclassInstance | type[DataclassInstance]) -> tu
 
 # HACK: `obj: Never` typing matches if object argument is using `Any` type.
 @overload
-def is_dataclass(obj: Never) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ...  # type: ignore[narrowed-type-not-subtype]  # pyright: ignore[reportGeneralTypeIssues]
+def is_dataclass(obj: Never) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ...  # type: ignore[narrowed-type-not-subtype]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[invalid-type-guard-definition]
 @overload
 def is_dataclass(obj: type) -> TypeIs[type[DataclassInstance]]: ...
 @overload
@@ -308,13 +308,17 @@ class FrozenInstanceError(AttributeError): ...
 
 class InitVar(Generic[_T]):
     __slots__ = ("type",)
-    type: Type[_T]
+    type: Type[_T]  # ty:ignore[unbound-type-variable]
     def __init__(self, type: Type[_T]) -> None: ...
 
     @overload
-    def __class_getitem__(cls, type: Type[_T]) -> InitVar[_T]: ...  # pyright: ignore[reportInvalidTypeForm]
+    def __class_getitem__(
+        cls, type: Type[_T]
+    ) -> InitVar[_T]: ...  # pyright: ignore[reportInvalidTypeForm]  # ty:ignore[invalid-type-form]
     @overload
-    def __class_getitem__(cls, type: Any) -> InitVar[Any]: ...  # pyright: ignore[reportInvalidTypeForm]
+    def __class_getitem__(
+        cls, type: Any
+    ) -> InitVar[Any]: ...  # pyright: ignore[reportInvalidTypeForm]  # ty:ignore[invalid-type-form]
 
 if sys.version_info >= (3, 14):
     def make_dataclass(
diff --git a/stdlib/os/__init__.pyi b/stdlib/os/__init__.pyi
@@ -940,7 +940,7 @@ In the future, this property will contain the last metadata change time.""")
 # At runtime it inherits from ABC and is not a Protocol, but it will be
 # on the allowlist for use as a Protocol starting in 3.14.
 @runtime_checkable
-class PathLike(ABC, Protocol[AnyStr_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class PathLike(ABC, Protocol[AnyStr_co]):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[invalid-protocol]
     __slots__ = ()
     @abstractmethod
     def __fspath__(self) -> AnyStr_co: ...
diff --git a/stdlib/platform.pyi b/stdlib/platform.pyi
@@ -44,15 +44,15 @@ if sys.version_info >= (3, 12):
 
         def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ...
         @property
-        def processor(self) -> str: ...
+        def processor(self) -> str: ...  # ty:ignore[invalid-named-tuple-override]
 
 else:
     @disjoint_base
     class uname_result(_uname_result_base):
         __match_args__ = ("system", "node", "release", "version", "machine")  # pyright: ignore[reportAssignmentType]
         def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ...
         @property
-        def processor(self) -> str: ...
+        def processor(self) -> str: ...  # ty:ignore[invalid-named-tuple-override]
 
 def uname() -> uname_result: ...
 def system() -> str: ...
diff --git a/stdlib/ssl.pyi b/stdlib/ssl.pyi
@@ -262,8 +262,18 @@ class Purpose(_ASN1Object, enum.Enum):
     # because this is an enum, the inherited __new__ is replaced at runtime with
     # Enum.__new__.
     def __new__(cls, value: object) -> Self: ...
-    SERVER_AUTH = (129, "serverAuth", "TLS Web Server Authentication", "1.3.6.1.5.5.7.3.2")  # pyright: ignore[reportCallIssue]
-    CLIENT_AUTH = (130, "clientAuth", "TLS Web Client Authentication", "1.3.6.1.5.5.7.3.1")  # pyright: ignore[reportCallIssue]
+    SERVER_AUTH = (  # ty:ignore[invalid-assignment]
+        129,
+        "serverAuth",
+        "TLS Web Server Authentication",
+        "1.3.6.1.5.5.7.3.2",
+    )  # pyright: ignore[reportCallIssue]
+    CLIENT_AUTH = (  # ty:ignore[invalid-assignment]
+        130,
+        "clientAuth",
+        "TLS Web Client Authentication",
+        "1.3.6.1.5.5.7.3.1",
+    )  # pyright: ignore[reportCallIssue]
 
 class SSLSocket(socket.socket):
     context: SSLContext
diff --git a/stdlib/types.pyi b/stdlib/types.pyi
@@ -268,7 +268,7 @@ class CodeType:
         __replace__ = replace
 
 @final
-class MappingProxyType(Mapping[_KT_co, _VT_co]):  # type: ignore[type-var]  # pyright: ignore[reportInvalidTypeArguments]
+class MappingProxyType(Mapping[_KT_co, _VT_co]):  # type: ignore[type-var]  # pyright: ignore[reportInvalidTypeArguments]  # ty:ignore[invalid-generic-class]
     __hash__: ClassVar[None]  # type: ignore[assignment]
     def __new__(cls, mapping: SupportsKeysAndGetItem[_KT_co, _VT_co]) -> Self: ...
     def __getitem__(self, key: _KT_co, /) -> _VT_co: ...  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
diff --git a/stdlib/typing.pyi b/stdlib/typing.pyi
@@ -1063,13 +1063,13 @@ class NamedTuple(tuple[Any, ...]):
 
     @final
     @classmethod
-    def _make(cls, iterable: Iterable[Any]) -> typing_extensions.Self: ...
+    def _make(cls, iterable: Iterable[Any]) -> typing_extensions.Self: ...  # ty:ignore[invalid-type-form]
     @final
     def _asdict(self) -> dict[str, Any]: ...
     @final
-    def _replace(self, **kwargs: Any) -> typing_extensions.Self: ...
+    def _replace(self, **kwargs: Any) -> typing_extensions.Self: ...  # ty:ignore[invalid-type-form]
     if sys.version_info >= (3, 13):
-        def __replace__(self, **kwargs: Any) -> typing_extensions.Self: ...
+        def __replace__(self, **kwargs: Any) -> typing_extensions.Self: ...  # ty:ignore[invalid-type-form]
 
 # Internal mypy fallback type for all typed dicts (does not exist at runtime)
 # N.B. Keep this mostly in sync with typing_extensions._TypedDict/mypy_extensions._TypedDict
diff --git a/stdlib/typing_extensions.pyi b/stdlib/typing_extensions.pyi
@@ -356,9 +356,9 @@ else:
         def __init__(self, typename: str, fields: None = None, **kwargs: Any) -> None: ...
 
         @classmethod
-        def _make(cls, iterable: Iterable[Any]) -> Self: ...
+        def _make(cls, iterable: Iterable[Any]) -> Self: ...  # ty:ignore[invalid-type-form]
         def _asdict(self) -> dict[str, Any]: ...
-        def _replace(self, **kwargs: Any) -> Self: ...
+        def _replace(self, **kwargs: Any) -> Self: ...  # ty:ignore[invalid-type-form]
 
     class NewType:
         def __init__(self, name: str, tp: AnnotationForm) -> None: ...
@@ -389,7 +389,7 @@ else:
     # At runtime it inherits from ABC and is not a Protocol, but it is on the
     # allowlist for use as a Protocol.
     @runtime_checkable
-    class Buffer(Protocol, abc.ABC):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+    class Buffer(Protocol, abc.ABC):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[invalid-protocol]
         # Not actually a Protocol at runtime; see
         # https://github.com/python/typeshed/issues/10224 for why we're defining it this way
         def __buffer__(self, flags: int, /) -> memoryview: ...
diff --git a/stubs/JACK-Client/jack/__init__.pyi b/stubs/JACK-Client/jack/__init__.pyi
@@ -12,7 +12,7 @@ from numpy.typing import NDArray
 # This is not a real subclassing. Just ensuring type-checkers sees this type as compatible with _CDataBase
 # pyright has no error code for subclassing final
 @type_check_only
-class _JackPositionT(_CDataBase):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+class _JackPositionT(_CDataBase):  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]  # ty:ignore[subclass-of-final-class]
     audio_frames_per_video_frame: float
     bar: int
     bar_start_tick: float
diff --git a/stubs/WebOb/webob/cachecontrol.pyi b/stubs/WebOb/webob/cachecontrol.pyi
@@ -38,7 +38,7 @@ class exists_property(Generic[_ScopeT]):
     def __delete__(self, obj: CacheControl[_ScopeT]) -> None: ...
 
 class value_property(Generic[_T, _DefaultT, _NoneLiteral, _ScopeT]):
-    def __init__(self, prop: str, default: _DefaultT = None, none: _NoneLiteral = None, type: _ScopeT = None) -> None: ...  # type: ignore[assignment]
+    def __init__(self, prop: str, default: _DefaultT = None, none: _NoneLiteral = None, type: _ScopeT = None) -> None: ...  # type: ignore[assignment]  # ty:ignore[invalid-parameter-default]
 
     @overload
     def __get__(self, obj: None, type: type[CacheControl[Any]] | None = None) -> Self: ...
diff --git a/stubs/geopandas/geopandas/geoseries.pyi b/stubs/geopandas/geopandas/geoseries.pyi
@@ -19,7 +19,7 @@ from .explore import _explore_geoseries
 from .io._geoarrow import GeoArrowArray
 from .plotting import plot_series
 
-class GeoSeries(GeoPandasBase, pd.Series[BaseGeometry]):  # type: ignore[type-var,misc]  # pyright: ignore[reportInvalidTypeArguments]
+class GeoSeries(GeoPandasBase, pd.Series[BaseGeometry]):  # type: ignore[type-var,misc]  # pyright: ignore[reportInvalidTypeArguments]  # ty:ignore[invalid-type-arguments]
     # Override the weird annotation of Series.__new__ in pandas-stubs
     def __new__(
         self,
diff --git a/stubs/geopandas/geopandas/plotting.pyi b/stubs/geopandas/geopandas/plotting.pyi
@@ -193,7 +193,7 @@ class GeoplotAccessor(PlotAccessor):
         include_bool: bool = False,
         backend: str | None = None,
         **kwargs,
-    ) -> pd.Series[Axes]: ...  # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments]
+    ) -> pd.Series[Axes]: ...  # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments]  # ty:ignore[invalid-type-arguments]
     @overload
     def __call__(
         self,
diff --git a/stubs/pyogrio/pyogrio/raw.pyi b/stubs/pyogrio/pyogrio/raw.pyi
@@ -3,7 +3,7 @@ from contextlib import AbstractContextManager
 from typing import Any, Literal, TypedDict, overload, type_check_only
 
 import numpy as np
-import pyarrow as pa  # type: ignore[import-not-found]  # pyright: ignore[reportMissingImports]
+import pyarrow as pa  # type: ignore[import-not-found]  # pyright: ignore[reportMissingImports]  # ty:ignore[unresolved-import]
 import shapely as shp
 
 from ._typing import Array1D, ArrayLikeInt, ReadPathOrBuffer, SupportsArrowCStream, WritePathOrBuffer
diff --git a/stubs/python-dateutil/dateutil/tz/__init__.pyi b/stubs/python-dateutil/dateutil/tz/__init__.pyi
@@ -1,3 +1,4 @@
+import builtins
 import sys
 from datetime import datetime
 from typing_extensions import Self
@@ -25,7 +26,7 @@ if sys.platform == "win32":
         hasdst: bool
         def __eq__(self, other: tzwinbase) -> bool: ...  # type: ignore[override]
         @staticmethod
-        def list() -> list[str]: ...
+        def list() -> builtins.list[str]: ...
         def display(self) -> str | None: ...
         def transitions(self, year: int) -> tuple[datetime, datetime] | None: ...
 
diff --git a/stubs/setuptools/setuptools/__init__.pyi b/stubs/setuptools/setuptools/__init__.pyi
@@ -155,7 +155,7 @@ def setup(
     install_requires: str | Iterable[str] = [],
     extras_require: _DictLike[Incomplete, Incomplete] = {},
     # kwargs used directly in distutils.core.setup
-    distclass: type[_DistributionT] = Distribution,  # type: ignore[assignment] # noqa: Y011
+    distclass: type[_DistributionT] = Distribution,  # type: ignore[assignment] # noqa: Y011  # ty:ignore[invalid-parameter-default]
     # Custom Distributions could accept more params
     **attrs: Any,
 ) -> _DistributionT: ...
diff --git a/stubs/yt-dlp/yt_dlp/extractor/common.pyi b/stubs/yt-dlp/yt_dlp/extractor/common.pyi
@@ -943,7 +943,7 @@ class InfoExtractor:
         self,
         url: str,
         *args: Unused,
-        info_dict: _InfoDict = {},  # type: ignore[typeddict-item]  # pyright: ignore[reportArgumentType]
+        info_dict: _InfoDict = {},  # type: ignore[typeddict-item]  # pyright: ignore[reportArgumentType]  # ty:ignore[missing-typed-dict-key]
         note: str = "Extracting generic embeds",
         **kwargs: Unused,
     ) -> list[dict[str, Any]]: ...
diff --git a/tests/README.md b/tests/README.md
@@ -3,6 +3,7 @@ This directory contains several tests:
 tests the stubs with [mypy](https://github.com/python/mypy/)
 - `tests/pyright_test.py` tests the stubs with
 [pyright](https://github.com/microsoft/pyright).
+- `tests/ty_test.py` tests the stubs with [ty](https://github.com/astral-sh/ty).
 - `tests/regr_test.py` runs mypy against the test cases for typeshed's
 stubs, guarding against accidental regressions.
 - `tests/check_typeshed_structure.py` checks that typeshed's directory
@@ -81,6 +82,22 @@ checks that would typically fail on incomplete stubs (such as `Unknown` checks).
 In typeshed's CI, pyright is run with these configuration settings on a subset of
 the stubs in typeshed (including the standard library).
 
+## ty\_test.py
+
+Run using:
+```bash
+(.venv)$ python3 tests/ty_test.py --python=.venv
+(.venv)$ python3 tests/ty_test.py stubs/PySocks --python=.venv
+```
+
+This test checks the stdlib and third-party stubs with ty, using the configuration
+in `ty.toml`. It selects the stdlib modules supported by the requested Python
+version using `stdlib/VERSIONS` and adds the third-party stub roots to ty's module
+search path. Unlike pyright, it checks `geopandas`, `seaborn`, and `shapely` on
+every target version; only the obsolete `requests` and legacy `distutils` stubs
+are excluded. Run
+`python tests/ty_test.py --help` for the supported options.
+
 ## regr\_test.py
 
 This test runs mypy against the test cases for typeshed's stdlib and third-party
diff --git a/tests/runtests.py b/tests/runtests.py
@@ -3,6 +3,7 @@
 
 import argparse
 import json
+import os
 import re
 import subprocess
 import sys
@@ -21,6 +22,19 @@
 _FAILED = colored("Failed", "red")
 
 
+def _pythonpath_env() -> dict[str, str]:
+    pythonpath = os.environ.get("PYTHONPATH")
+    env = os.environ.copy()
+    env["PYTHONPATH"] = "lib" if not pythonpath else os.pathsep.join(("lib", pythonpath))
+    return env
+
+
+def _ty_platform() -> str:
+    if sys.platform.startswith("win"):
+        return "win32"
+    return sys.platform
+
+
 def _parse_jsonc(json_text: str) -> str:
     # strip comments from the file
     lines = [line for line in json_text.split("\n") if not line.strip().startswith("//")]
@@ -100,6 +114,23 @@ def main() -> None:
         pyright_returncode = pyright_result.returncode
         pyright_skipped = False
 
+    print(f"\nRunning ty for Python {python_version}...")
+    ty_result = subprocess.run(
+        [
+            sys.executable,
+            "tests/ty_test.py",
+            path,
+            "--python-version",
+            python_version,
+            "--platform",
+            _ty_platform(),
+            "--python",
+            sys.executable,
+        ],
+        env=_pythonpath_env(),
+        check=False,
+    )
+
     print(f"\nRunning mypy for Python {python_version}...")
     mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", python_version], check=False)
     # If mypy failed, stubtest will fail without any helpful error
@@ -171,6 +202,7 @@ def main() -> None:
             pre_commit_result.returncode,
             check_structure_result.returncode,
             pyright_returncode,
+            ty_result.returncode,
             mypy_result.returncode,
             getattr(stubtest_result, "returncode", 0),
             pyright_testcases_returncode,
@@ -197,6 +229,7 @@ def main() -> None:
         print("Pyright:", _SKIPPED)
     else:
         print("Pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
+    print("ty:", _SUCCESS if ty_result.returncode == 0 else _FAILED)
     print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
     if stubtest_result is None:
         print("stubtest:", _SKIPPED)
diff --git a/tests/ty_test.py b/tests/ty_test.py
diff --git a/ty.toml b/ty.toml
