Use dev dependency-group

Avasam · Avasam · commit e5cb0ea42220 · 2025-05-16T16:25:22.000-04:00
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -6,7 +6,7 @@ on:
     - cron: "0 0 * * *"
   pull_request:
     paths:
-      - "requirements-tests.txt"
+      - "pyproject.toml"
       - ".github/workflows/daily.yml"
 
 # Please keep the permissions minimal, as stubtest runs arbitrary code from pypi.
@@ -45,11 +45,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
-          cache-dependency-path: requirements-tests.txt
+          cache-dependency-path: pyproject.toml
           allow-prereleases: true
           check-latest: true
       - name: Install dependencies
-        run: pip install -r requirements-tests.txt
+        run: pip install --group=dev
       - name: Run stubtest
         run: python tests/stubtest_stdlib.py
 
@@ -69,10 +69,10 @@ jobs:
           python-version: "3.13"
           cache: pip
           cache-dependency-path: |
-            requirements-tests.txt
+            pyproject.toml
             stubs/**/METADATA.toml
       - name: Install dependencies
-        run: pip install -r requirements-tests.txt
+        run: pip install --group=dev
       - name: Run stubtest
         shell: bash
         run: |
diff --git a/.github/workflows/meta_tests.yml b/.github/workflows/meta_tests.yml
@@ -12,7 +12,6 @@ on:
       - "tests/**"
       - "lib/**"
       - ".github/workflows/meta_tests.yml"
-      - "requirements-tests.txt"
       - "pyproject.toml"
 
 permissions:
@@ -40,7 +39,7 @@ jobs:
         with:
           python-version: "3.13"
       - run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - run: python ./tests/typecheck_typeshed.py --platform=${{ matrix.platform }}
   pyright:
     name: Check scripts and tests with pyright
@@ -57,7 +56,7 @@ jobs:
           # pytype_test.py imports pytype, we need to use Python 3.12 for now.
           python-version: "3.12"
       - run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - name: Run pyright on typeshed
         uses: jakebailey/pyright-action@v2
         with:
@@ -78,5 +77,5 @@ jobs:
         run: |
           git config --global user.name stubsabot
           git config --global user.email '<>'
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - run: python scripts/stubsabot.py --action-level local
diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml
@@ -33,12 +33,12 @@ jobs:
         with:
           python-version: "3.13"
       - name: Install dependencies
-        run: pip install git+https://github.com/hauntsaninja/mypy_primer.git
+        run: pip install git+https://github.com/hauntsaninja/mypy_primer.git dependency-groups
       - name: Run mypy_primer
         shell: bash
         run: |
           cd typeshed_to_test
-          MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3)
+          MYPY_VERSION=$(python3 -m dependency_groups dev | grep mypy== | cut -d = -f3)
           echo "new commit"
           git rev-list --format=%s --max-count=1 $GITHUB_SHA
           git checkout -b upstream_main origin/main
diff --git a/.github/workflows/stubsabot.yml b/.github/workflows/stubsabot.yml
@@ -33,7 +33,7 @@ jobs:
           git config --global user.name stubsabot
           git config --global user.email '<>'
       - name: Install dependencies
-        run: uv pip install -r requirements-tests.txt --system
+        run: uv pip install --group=dev --system
       - name: Run stubsabot
         run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything
 
diff --git a/.github/workflows/stubtest_stdlib.yml b/.github/workflows/stubtest_stdlib.yml
@@ -41,10 +41,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
-          cache-dependency-path: requirements-tests.txt
+          cache-dependency-path: pyproject.toml
           allow-prereleases: true
           check-latest: true
       - name: Install dependencies
-        run: pip install -r requirements-tests.txt
+        run: pip install --group=dev
       - name: Run stubtest
         run: python tests/stubtest_stdlib.py
diff --git a/.github/workflows/stubtest_third_party.yml b/.github/workflows/stubtest_third_party.yml
@@ -44,10 +44,10 @@ jobs:
           python-version: "3.13"
           cache: pip
           cache-dependency-path: |
-            requirements-tests.txt
+            pyproject.toml
             stubs/**/METADATA.toml
       - name: Install dependencies
-        run: pip install -r requirements-tests.txt
+        run: pip install --group=dev
       - name: Run stubtest
         shell: bash
         run: |
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.13"
       - run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - run: python ./tests/check_typeshed_structure.py
 
   pytype:
@@ -44,7 +44,7 @@ jobs:
           # Max supported Python version as of pytype 2024.10.11
           python-version: "3.12"
       - uses: astral-sh/setup-uv@v6
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - name: Install external dependencies for 3rd-party stubs
         run: |
           DEPENDENCIES=$( python tests/get_external_stub_requirements.py )
@@ -102,7 +102,7 @@ jobs:
           # TODO: figure out why that is (#11590)
           python-version: "3.11"
       - run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - run: uv pip install -r requirements-tests.txt --system
+      - run: uv pip install --group=dev --system
       - run: python ./tests/regr_test.py --all --verbosity QUIET
 
   pyright:
@@ -121,7 +121,7 @@ jobs:
       - uses: astral-sh/setup-uv@v6
       - name: Install typeshed test-suite requirements
         # Install these so we can run `get_external_stub_requirements.py`
-        run: uv pip install -r requirements-tests.txt --system
+        run: uv pip install --group=dev --system
       - name: Install required APT packages
         run: |
           DEPENDENCIES=$( python tests/get_external_apt_dependencies.py )
@@ -187,5 +187,5 @@ jobs:
       - name: Run tests
         run: |
           cd stub_uploader
-          uv pip install -r requirements.txt --system
+          uv pip install --group=dev --system
           python -m pytest tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,6 +10,10 @@ repos:
       - id: mixed-line-ending
         args: [--fix=lf]
       - id: check-case-conflict
+  - repo: https://github.com/sirosen/dependency-groups
+    rev: 1.3.1 # must match pyproject.toml
+    hooks:
+      - id: lint-dependency-groups
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.11.4 # must match requirements-tests.txt
     hooks:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -63,8 +63,8 @@ Note that some tests require extra setup steps to install the required dependenc
   ```bash
   $ python3 -m venv .venv
   $ source .venv/bin/activate
-  (.venv)$ pip install -U pip
-  (.venv)$ pip install -r requirements-tests.txt
+  (.venv)$ pip install -U "pip>=25.1"
+  (.venv)$ pip install --group=dev
   ```
 
   </td>
@@ -79,8 +79,8 @@ Note that some tests require extra setup steps to install the required dependenc
   ```powershell
   > python -m venv .venv
   > .venv\Scripts\activate
-  (.venv) > pip install -U pip
-  (.venv) > pip install -r requirements-tests.txt
+  (.venv) > pip install -U "pip>=25.1"
+  (.venv) > pip install --group=dev
   ```
 
   To be able to run pytype tests, you'll also need to install it manually
@@ -100,8 +100,7 @@ as it's currently excluded from the requirements file:
   If you already have [uv](https://docs.astral.sh/uv/getting-started/installation/) installed, you can simply replace the commands above with:
 
   ```shell
-  uv venv
-  uv pip install -r requirements-tests.txt
+  uv pip install --group=dev
   ```
 
   ```shell
@@ -220,7 +219,7 @@ This has the following keys:
   in addition to the requirements in the `requires` field.
 * `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages
   that need to be installed for stubtest to run successfully.
-* `brew_dependencies` (default: `[]`): A list of MacOS Homebrew packages
+* `brew_dependencies` (default: `[]`): A list of macOS Homebrew packages
   that need to be installed for stubtest to run successfully
 * `choco_dependencies` (default: `[]`): A list of Windows Chocolatey packages
   that need to be installed for stubtest to run successfully
@@ -347,7 +346,7 @@ replacing `$INSERT_LIBRARY_NAME_HERE` with the name of the library:
 When the script has finished running, it will print instructions telling you what to do next.
 
 If it has been a while since you set up the virtualenv, make sure you have
-the latest mypy (`pip install -r requirements-tests.txt`) before running the script.
+the latest mypy (`pip install --group=dev`) before running the script.
 
 ### Supported type system features
 
diff --git a/lib/ts_utils/paths.py b/lib/ts_utils/paths.py
@@ -10,7 +10,6 @@
 STUBS_PATH: Final = TS_BASE_PATH / "stubs"
 
 PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml"
-REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt"
 GITIGNORE_PATH: Final = TS_BASE_PATH / ".gitignore"
 
 TESTS_DIR: Final = "@tests"
diff --git a/lib/ts_utils/utils.py b/lib/ts_utils/utils.py
@@ -13,9 +13,11 @@
 from typing_extensions import TypeAlias
 
 import pathspec
+import tomllib
+from dependency_groups import resolve
 from packaging.requirements import Requirement
 
-from .paths import GITIGNORE_PATH, REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path
+from .paths import GITIGNORE_PATH, PYPROJECT_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path
 
 if TYPE_CHECKING:
     from _typeshed import OpenTextMode
@@ -95,12 +97,12 @@ def venv_python(venv_dir: Path) -> Path:
 
 @functools.cache
 def parse_requirements() -> Mapping[str, Requirement]:
-    """Return a dictionary of requirements from the requirements file."""
-    with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file:
-        stripped_lines = map(strip_comments, requirements_file)
-        stripped_more = [li for li in stripped_lines if not li.startswith("-")]
-        requirements = map(Requirement, filter(None, stripped_more))
-        return {requirement.name: requirement for requirement in requirements}
+    """Return a dictionary of requirements found in the dev dependency group."""
+    with PYPROJECT_PATH.open("rb") as requirements_file:
+        pyproject = tomllib.load(requirements_file)
+
+    requirements = [Requirement(requirement) for requirement in resolve(pyproject["dependency-groups"], "dev")]
+    return {requirement.name: requirement for requirement in requirements}
 
 
 def get_mypy_req() -> str:
diff --git a/pyproject.toml b/pyproject.toml
@@ -5,6 +5,41 @@ name = "typeshed"
 version = "0"
 requires-python = ">=3.9" # Minimum version to run tests, used by uv run
 
+[dependency-groups]
+dev = [
+    # Type checkers that we test our stubs against. These should always
+    # be pinned to a specific version to make failure reproducible.
+    "mypy ==1.15.0",
+    "pyright ==1.1.400",
+    # pytype can be installed on Windows, but requires building wheels, let's not do that on the CI
+    "pytype ==2024.10.11; platform_system != 'Windows' and python_version >= '3.10' and python_version < '3.13'",
+
+    # Libraries used by our various scripts.
+    # TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at the moment.
+    "aiohttp ==3.11.15; python_version < '3.14'",
+    "dependency-groups ==1.3.1", # Must match .pre-commit-config.yaml.
+    # TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at the moment.
+    "grpcio-tools >=1.66.2; python_version < '3.14'", # For grpc_tools.protoc"
+    "mypy-protobuf ==3.6.0",
+    "packaging ==24.2",
+    "pathspec >=0.11.1",
+    "pre-commit",
+    "ruff ==0.11.4", # Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml.
+    # TODO (2025-05-07): Dependency libcst doesn't support Python 3.14 yet.
+    "stubdefaulter ==0.1.0; python_version < '3.14'",
+    "termcolor >=2.3",
+    "tomli ==2.2.1",
+    "tomlkit ==0.13.2",
+    "typing_extensions >=4.13.0rc1",
+    "uv ==0.7.4",
+
+    # Utilities for typeshed infrastructure scripts.
+    "ts_utils @ file:lib",
+]
+
+[tool.uv.sources]
+ts_utils = { path = "lib", editable = true }
+
 [tool.black]
 line-length = 130
 target-version = ["py310"]
diff --git a/requirements-tests.txt b/requirements-tests.txt
diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py
@@ -84,8 +84,8 @@ async def get_project_urls_from_pypi(project: str, session: aiohttp.ClientSessio
 
 
 async def get_upstream_repo_url(project: str) -> str | None:
-    # aiohttp is overkill here, but it would also just be silly
-    # to have both requests and aiohttp in our requirements-tests.txt file.
+    # aiohttp is overkill here, but it would also just be silly to have
+    # both requests and aiohttp in our dev dependency-group in pyproject.toml.
     async with aiohttp.ClientSession() as session:
         project_urls = await get_project_urls_from_pypi(project, session)
 
diff --git a/tests/check_typeshed_structure.py b/tests/check_typeshed_structure.py
@@ -12,7 +12,7 @@
 from pathlib import Path
 
 from ts_utils.metadata import read_metadata
-from ts_utils.paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path
+from ts_utils.paths import PYPROJECT_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path
 from ts_utils.utils import (
     get_all_testcase_directories,
     get_gitignore_spec,
@@ -166,10 +166,10 @@ def check_requirement_pins() -> None:
     """Check that type checkers and linters are pinned to an exact version."""
     requirements = parse_requirements()
     for package in linters:
-        assert package in requirements, f"type checker/linter '{package}' not found in {REQUIREMENTS_PATH.name}"
+        assert package in requirements, f"type checker/linter '{package}' not found in {PYPROJECT_PATH.name}"
         spec = requirements[package].specifier
-        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQUIREMENTS_PATH.name}"
-        msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQUIREMENTS_PATH.name}"
+        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {PYPROJECT_PATH.name}"
+        msg = f"type checker/linter '{package}' is not pinned to an exact version in {PYPROJECT_PATH.name}"
         assert str(spec).startswith("=="), msg
 
 
