Merge branch 'main' into etree-iterparse

Author: srittau

.flake8

@@ -1,7 +1,6 @@
 [flake8]
 # Y: Flake8 is only used to run flake8-pyi, everything else is in Ruff
-# F821: Typeshed is a testing ground for flake8-pyi, which monkeypatches F821
-select = Y, F821
+select = Y
 # Ignore rules normally excluded by default
 extend-ignore = Y090,Y091
 per-file-ignores =

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
         args: [--fix=lf]
       - id: check-case-conflict
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.4 # must match requirements-tests.txt
+    rev: v0.12.2 # must match requirements-tests.txt
     hooks:
       - id: ruff
         name: Run ruff on stubs, tests and scripts
@@ -31,7 +31,7 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 7.2.0
+    rev: 7.3.0
     hooks:
       - id: flake8
         language: python

CONTRIBUTING.md

@@ -77,9 +77,9 @@ Note that some tests require extra setup steps to install the required dependenc
   Run the following commands from a Windows terminal to install all requirements:
 
   ```powershell
-  > python -m venv .venv
+  > py -m venv .venv
   > .venv\Scripts\activate
-  (.venv) > pip install -U pip
+  (.venv) > python -m pip install -U pip
   (.venv) > pip install -r requirements-tests.txt
   ```
 
@@ -447,7 +447,7 @@ If a package ships its own `py.typed` file, please follow these steps:
 
 1. Open an issue with the earliest month of removal in the subject.
 2. A maintainer will add the
-   ["stubs: removal" label](https://github.com/python/typeshed/labels/stubs%3A%20removal).
+   ["stubs: removal" label](https://github.com/python/typeshed/labels/%22stubs%3A%20removal%22).
 3. Open a PR that sets the `obsolete_since` field in the `METADATA.toml`
    file to the first version of the package that shipped `py.typed`.
 4. After at least six months, open a PR to remove the stubs.
@@ -457,18 +457,19 @@ steps:
 
 1. Open an issue explaining why the stubs should be removed.
 2. A maintainer will add the
-   ["stubs: removal" label](https://github.com/python/typeshed/labels/stubs%3A%20removal).
+   ["stubs: removal" label](https://github.com/python/typeshed/labels/%22stubs%3A%20removal%22).
 3. Open a PR that sets the `no_longer_updated` field in the `METADATA.toml`
    file to `true`.
 4. When a new version of the package was automatically uploaded to PyPI
    (which can take up to a day), open a PR to remove the stubs.
 
-If feeling kindly, please update [mypy](https://github.com/python/mypy/blob/master/mypy/stubinfo.py)
+Don't forget to make sure the library is not in the [`pyrightconfig.stricter.json`](./pyrightconfig.stricter.json)
+exclusion list. If feeling kindly, please update [mypy](https://github.com/python/mypy/blob/master/mypy/stubinfo.py)
 for any stub obsoletions or removals.
 
 ### Marking PRs as "deferred"
 
-We sometimes use the ["status: deferred" label](https://github.com/python/typeshed/labels/status%3A%20deferred)
+We sometimes use the ["status: deferred" label](https://github.com/python/typeshed/labels/%22status%3A%20deferred%22)
 to mark PRs and issues that we'd like to accept, but that are blocked by some
 external factor. Blockers can include:
 

lib/ts_utils/metadata.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import datetime
 import functools
 import re
 import urllib.parse
@@ -18,6 +19,7 @@
 import tomlkit
 from packaging.requirements import Requirement
 from packaging.specifiers import Specifier
+from tomlkit.items import String
 
 from .paths import PYPROJECT_PATH, STUBS_PATH, distribution_path
 
@@ -26,6 +28,7 @@
     "PackageDependencies",
     "StubMetadata",
     "StubtestSettings",
+    "get_oldest_supported_python",
     "get_recursive_requirements",
     "read_dependencies",
     "read_metadata",
@@ -139,6 +142,13 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     )
 
 
+@final
+@dataclass(frozen=True)
+class ObsoleteMetadata:
+    since_version: Annotated[str, "A string representing a specific version"]
+    since_date: Annotated[datetime.date, "A date when the package became obsolete"]
+
+
 @final
 @dataclass(frozen=True)
 class StubMetadata:
@@ -153,7 +163,7 @@ class StubMetadata:
     extra_description: str | None
     stub_distribution: Annotated[str, "The name under which the distribution is uploaded to PyPI"]
     upstream_repository: Annotated[str, "The URL of the upstream repository"] | None
-    obsolete_since: Annotated[str, "A string representing a specific version"] | None
+    obsolete: Annotated[ObsoleteMetadata, "Metadata indicating when the stubs package became obsolete"] | None
     no_longer_updated: bool
     uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"]
     partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."]
@@ -162,7 +172,7 @@ class StubMetadata:
 
     @property
     def is_obsolete(self) -> bool:
-        return self.obsolete_since is not None
+        return self.obsolete is not None
 
 
 _KNOWN_METADATA_FIELDS: Final = frozenset(
@@ -213,27 +223,27 @@ def read_metadata(distribution: str) -> StubMetadata:
     """
     try:
         with metadata_path(distribution).open("rb") as f:
-            data: dict[str, object] = tomli.load(f)
+            data = tomlkit.load(f)
     except FileNotFoundError:
         raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None
 
     unknown_metadata_fields = data.keys() - _KNOWN_METADATA_FIELDS
     assert not unknown_metadata_fields, f"Unexpected keys in METADATA.toml for {distribution!r}: {unknown_metadata_fields}"
 
     assert "version" in data, f"Missing 'version' field in METADATA.toml for {distribution!r}"
-    version = data["version"]
+    version: object = data.get("version")  # pyright: ignore[reportUnknownMemberType]
     assert isinstance(version, str) and len(version) > 0, f"Invalid 'version' field in METADATA.toml for {distribution!r}"
     # Check that the version spec parses
     if version[0].isdigit():
         version = f"=={version}"
     version_spec = Specifier(version)
     assert version_spec.operator in {"==", "~="}, f"Invalid 'version' field in METADATA.toml for {distribution!r}"
 
-    requires_s: object = data.get("requires", [])
+    requires_s: object = data.get("requires", [])  # pyright: ignore[reportUnknownMemberType]
     assert isinstance(requires_s, list)
     requires = [parse_requires(distribution, req) for req in requires_s]
 
-    extra_description: object = data.get("extra_description")
+    extra_description: object = data.get("extra_description")  # pyright: ignore[reportUnknownMemberType]
     assert isinstance(extra_description, (str, type(None)))
 
     if "stub_distribution" in data:
@@ -243,7 +253,7 @@ def read_metadata(distribution: str) -> StubMetadata:
     else:
         stub_distribution = f"types-{distribution}"
 
-    upstream_repository: object = data.get("upstream_repository")
+    upstream_repository: object = data.get("upstream_repository")  # pyright: ignore[reportUnknownMemberType]
     assert isinstance(upstream_repository, (str, type(None)))
     if isinstance(upstream_repository, str):
         parsed_url = urllib.parse.urlsplit(upstream_repository)
@@ -267,21 +277,28 @@ def read_metadata(distribution: str) -> StubMetadata:
             )
             assert num_url_path_parts == 2, bad_github_url_msg
 
-    obsolete_since: object = data.get("obsolete_since")
-    assert isinstance(obsolete_since, (str, type(None)))
-    no_longer_updated: object = data.get("no_longer_updated", False)
+    obsolete_since: object = data.get("obsolete_since")  # pyright: ignore[reportUnknownMemberType]
+    assert isinstance(obsolete_since, (String, type(None)))
+    if obsolete_since:
+        comment = obsolete_since.trivia.comment
+        since_date_string = comment.removeprefix("# Released on ")
+        since_date = datetime.date.fromisoformat(since_date_string)
+        obsolete = ObsoleteMetadata(since_version=obsolete_since, since_date=since_date)
+    else:
+        obsolete = None
+    no_longer_updated: object = data.get("no_longer_updated", False)  # pyright: ignore[reportUnknownMemberType]
     assert type(no_longer_updated) is bool
-    uploaded_to_pypi: object = data.get("upload", True)
+    uploaded_to_pypi: object = data.get("upload", True)  # pyright: ignore[reportUnknownMemberType]
     assert type(uploaded_to_pypi) is bool
-    partial_stub: object = data.get("partial_stub", True)
+    partial_stub: object = data.get("partial_stub", True)  # pyright: ignore[reportUnknownMemberType]
     assert type(partial_stub) is bool
-    requires_python_str: object = data.get("requires_python")
+    requires_python_str: object = data.get("requires_python")  # pyright: ignore[reportUnknownMemberType]
     oldest_supported_python = get_oldest_supported_python()
     oldest_supported_python_specifier = Specifier(f">={oldest_supported_python}")
     if requires_python_str is None:
         requires_python = oldest_supported_python_specifier
     else:
-        assert type(requires_python_str) is str
+        assert isinstance(requires_python_str, str)
         requires_python = Specifier(requires_python_str)
         assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant'
         # Check minimum Python version is not less than the oldest version of Python supported by typeshed
@@ -291,7 +308,7 @@ def read_metadata(distribution: str) -> StubMetadata:
         assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'"
 
     empty_tools: dict[object, object] = {}
-    tools_settings: object = data.get("tool", empty_tools)
+    tools_settings: object = data.get("tool", empty_tools)  # pyright: ignore[reportUnknownMemberType]
     assert isinstance(tools_settings, dict)
     assert tools_settings.keys() <= _KNOWN_METADATA_TOOL_FIELDS.keys(), f"Unrecognised tool for {distribution!r}"
     for tool, tk in _KNOWN_METADATA_TOOL_FIELDS.items():
@@ -307,7 +324,7 @@ def read_metadata(distribution: str) -> StubMetadata:
         extra_description=extra_description,
         stub_distribution=stub_distribution,
         upstream_repository=upstream_repository,
-        obsolete_since=obsolete_since,
+        obsolete=obsolete,
         no_longer_updated=no_longer_updated,
         uploaded_to_pypi=uploaded_to_pypi,
         partial_stub=partial_stub,

lib/ts_utils/paths.py

@@ -12,6 +12,7 @@
 PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml"
 REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt"
 GITIGNORE_PATH: Final = TS_BASE_PATH / ".gitignore"
+PYRIGHT_CONFIG: Final = TS_BASE_PATH / "pyrightconfig.stricter.json"
 
 TESTS_DIR: Final = "@tests"
 TEST_CASES_DIR: Final = "test_cases"

pyproject.toml

@@ -159,6 +159,8 @@ ignore = [
     # Used for direct, non-subclass type comparison, for example: `type(val) is str`
     # see https://github.com/astral-sh/ruff/issues/6465
     "E721", # Do not compare types, use `isinstance()`
+    # Highly opinionated, and it's often necessary to violate it
+    "PLC0415",  # `import` should be at the top-level of a file
     # Leave the size and complexity of tests to human interpretation
     "PLR09", # Too many ...
     # Too many magic number "2" that are preferable inline. https://github.com/astral-sh/ruff/issues/10009

pyrightconfig.stricter.json

@@ -18,14 +18,11 @@
         "stdlib/optparse.pyi",
         "stdlib/_tkinter.pyi",
         "stdlib/tkinter/__init__.pyi",
-        "stdlib/tkinter/commondialog.pyi",
         "stdlib/tkinter/filedialog.pyi",
         "stdlib/tkinter/dialog.pyi",
-        "stdlib/tkinter/messagebox.pyi",
         "stdlib/tkinter/scrolledtext.pyi",
         "stdlib/tkinter/tix.pyi",
         "stdlib/tkinter/ttk.pyi",
-        "stubs/aiofiles",
         "stubs/antlr4-python3-runtime",
         "stubs/auth0-python",
         "stubs/Authlib",
@@ -34,12 +31,10 @@
         "stubs/boltons",
         "stubs/braintree",
         "stubs/cffi",
-        "stubs/click-web",
         "stubs/dateparser",
         "stubs/defusedxml",
         "stubs/docker",
         "stubs/docutils",
-        "stubs/flake8-typing-imports",
         "stubs/Flask-SocketIO",
         "stubs/fpdf2",
         "stubs/gdb",
@@ -55,11 +50,10 @@
         "stubs/httplib2",
         "stubs/hvac",
         "stubs/icalendar",
-        "stubs/jmespath",
         "stubs/jsonschema",
         "stubs/jwcrypto",
         "stubs/ldap3",
-        "stubs/m3u8",
+        "stubs/m3u8/m3u8/model.pyi",
         "stubs/Markdown",
         "stubs/mock/mock/mock.pyi",
         "stubs/mysqlclient",
@@ -76,21 +70,18 @@
         "stubs/peewee",
         "stubs/pexpect",
         "stubs/pika",
+        "stubs/pony",
         "stubs/protobuf",
         "stubs/psutil",
         "stubs/psycopg2",
         "stubs/pyasn1",
         "stubs/pycurl",
         "stubs/Pygments",
         "stubs/PyMySQL",
-        "stubs/python-crontab",
-        "stubs/python-datemath",
         "stubs/python-dateutil",
-        "stubs/python-http-client",
         "stubs/python-jose",
         "stubs/pytz/pytz/lazy.pyi",
         "stubs/pywin32",
-        "stubs/pyxdg",
         "stubs/PyYAML",
         "stubs/reportlab",
         "stubs/requests",
@@ -102,7 +93,6 @@
         "stubs/tqdm",
         "stubs/vobject",
         "stubs/workalendar",
-        "stubs/wurlitzer",
     ],
     "typeCheckingMode": "strict",
     // TODO: Complete incomplete stubs

requirements-tests.txt

@@ -1,30 +1,30 @@
 # Type checkers that we test our stubs against. These should always
 # be pinned to a specific version to make failure reproducible.
-mypy==1.16.0
-pyright==1.1.400
+mypy==1.16.1
+pyright==1.1.403
 # pytype can be installed on Windows, but requires building wheels, let's not do that on the CI
 pytype==2024.10.11; platform_system != "Windows" and python_version >= "3.10" and python_version < "3.13"
 
 # Libraries used by our various scripts.
 # TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at
 # the moment.
-aiohttp==3.11.15; python_version < "3.14"
+aiohttp==3.12.14; python_version < "3.14"
 # TODO (2025-05-09): No wheels exist for Python 3.14 yet, slowing down CI
 # considerably and prone to fail.
 grpcio-tools>=1.66.2; python_version < "3.14" # For grpc_tools.protoc
 mypy-protobuf==3.6.0
-packaging==24.2
+packaging==25.0
 pathspec>=0.11.1
 pre-commit
 # Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml.
-ruff==0.11.4
+ruff==0.12.2
 # TODO (2025-05-07): Dependency libcst doesn't support Python 3.14 yet.
 stubdefaulter==0.1.0; python_version < "3.14"
 termcolor>=2.3
 tomli==2.2.1
-tomlkit==0.13.2
+tomlkit==0.13.3
 typing_extensions>=4.14.0rc1
-uv==0.7.4
+uv==0.7.19
 
 # Utilities for typeshed infrastructure scripts.
 ts_utils @ file:lib

scripts/create_baseline_stubs.py

@@ -23,9 +23,7 @@
 import aiohttp
 import termcolor
 
-from ts_utils.paths import STDLIB_PATH, STUBS_PATH
-
-PYRIGHT_CONFIG = Path("pyrightconfig.stricter.json")
+from ts_utils.paths import PYRIGHT_CONFIG, STDLIB_PATH, STUBS_PATH
 
 
 def search_pip_freeze_output(project: str, output: str) -> tuple[str, str] | None: