fix: use requirements files with hashes instead of constraints

Author: parthea

packages/gapic-generator/_update_hashes.py

@@ -39,38 +39,12 @@ PREVIEW_PYTHON_VERSION = "3.15"
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
-if (CURRENT_DIRECTORY / "testing").exists():
-    LOWER_BOUND_CONSTRAINTS_FILE = (
-        CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt"
-    )
-else:
-    LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt"
+LOWER_BOUND_REQUIREMENTS_FILE = (
+    CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt"
+)
 PACKAGE_NAME = "{{ api.naming.warehouse_package_name }}"
 
-UNIT_TEST_STANDARD_DEPENDENCIES = [
-    "mock",
-    "asyncmock",
-    "pytest",
-    "pytest-cov",
-    "pytest-asyncio",
-]
-UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
-UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = []
-UNIT_TEST_DEPENDENCIES: List[str] = []
-UNIT_TEST_EXTRAS: List[str] = []
-UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}
-
 SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ALL_PYTHON
-SYSTEM_TEST_STANDARD_DEPENDENCIES = [
-    "mock",
-    "pytest",
-    "google-cloud-testutils",
-]
-SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
-SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = []
-SYSTEM_TEST_DEPENDENCIES: List[str] = []
-SYSTEM_TEST_EXTRAS: List[str] = []
-SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}
 
 nox.options.sessions = [
     "unit",
@@ -112,33 +86,48 @@ def mypy(session):
 
 @nox.session
 def update_lower_bounds(session):
-    """Update lower bounds in constraints.txt to match setup.py"""
+    """Update lower bounds in requirements.txt to match setup.py"""
     session.install("google-cloud-testutils")
     session.install(".")
 
+    requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"
+
     session.run(
         "lower-bound-checker",
         "update",
         "--package-name",
         PACKAGE_NAME,
         "--constraints-file",
-        str(LOWER_BOUND_CONSTRAINTS_FILE),
+        str(requirements_in),
+    )
+
+    session.install("pip-tools")
+    session.run(
+        "pip-compile",
+        "--allow-unsafe",
+        "--generate-hashes",
+        "--no-strip-extras",
+        "--output-file",
+        str(LOWER_BOUND_REQUIREMENTS_FILE),
+        str(requirements_in),
     )
 
 
 @nox.session
 def check_lower_bounds(session):
-    """Check lower bounds in setup.py are reflected in constraints file"""
+    """Check lower bounds in setup.py are reflected in requirements file"""
     session.install("google-cloud-testutils")
     session.install(".")
 
+    requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"
+
     session.run(
         "lower-bound-checker",
         "check",
         "--package-name",
         PACKAGE_NAME,
         "--constraints-file",
-        str(LOWER_BOUND_CONSTRAINTS_FILE),
+        str(requirements_in),
     )
 
 
@@ -219,32 +208,17 @@ def lint_setup_py(session):
     session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
 
 
-def install_unittest_dependencies(session, *constraints):
-    standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES
-    session.install(*standard_deps, *constraints)
-
-    if UNIT_TEST_EXTERNAL_DEPENDENCIES:
-        warnings.warn(
-            "'unit_test_external_dependencies' is deprecated. Instead, please "
-            "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.",
-            DeprecationWarning,
+def install_unittest_dependencies(session, prerelease=False):
+    if prerelease:
+        requirements_path = str(
+            CURRENT_DIRECTORY / "testing" / "requirements-prerelease.txt"
         )
-        session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints)
-
-    if UNIT_TEST_LOCAL_DEPENDENCIES:
-        session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints)
-
-    if UNIT_TEST_EXTRAS_BY_PYTHON:
-        extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
-    elif UNIT_TEST_EXTRAS:
-        extras = UNIT_TEST_EXTRAS
-    else:
-        extras = []
-
-    if extras:
-        session.install("-e", f".[{','.join(extras)}]", *constraints)
     else:
-        session.install("-e", ".", *constraints)
+        requirements_path = str(
+            CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
+        )
+    session.install("-r", requirements_path)
+    session.install("-e", ".", "--no-deps")
 
 
 @nox.session(python=ALL_PYTHON)
@@ -255,10 +229,7 @@ def install_unittest_dependencies(session, *constraints):
 def unit(session, protobuf_implementation):
     # Install all test dependencies, then install this package in-place.
 
-    constraints_path = str(
-        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
-    )
-    install_unittest_dependencies(session, "-c", constraints_path)
+    install_unittest_dependencies(session)
 
     # Run py.test against the unit tests.
     session.run(
@@ -284,42 +255,11 @@ def unit(session, protobuf_implementation):
 {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2201) Add a `unit_rest_async` nox session to run tests with [async_rest] extra installed. #}
 
 
-def install_systemtest_dependencies(session, *constraints):
-    {# Note that grpcio<=1.62.2 works on Python 3.11 but fails on 3.12+ because it requires pkg_resources for its source build. #}
-    if session.python >= "3.12":
-        session.install("--pre", "grpcio>=1.75.1")
-    else:
-        session.install("--pre", "grpcio<=1.62.2")
-
-    session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints)
-
-    if SYSTEM_TEST_EXTERNAL_DEPENDENCIES:
-        session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints)
-
-    if SYSTEM_TEST_LOCAL_DEPENDENCIES:
-        session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints)
-
-    if SYSTEM_TEST_DEPENDENCIES:
-        session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints)
-
-    if SYSTEM_TEST_EXTRAS_BY_PYTHON:
-        extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
-    elif SYSTEM_TEST_EXTRAS:
-        extras = SYSTEM_TEST_EXTRAS
-    else:
-        extras = []
-
-    if extras:
-        session.install("-e", f".[{','.join(extras)}]", *constraints)
-    else:
-        session.install("-e", ".", *constraints)
-
-
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
     """Run the system test suite."""
-    constraints_path = str(
-        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    requirements_path = str(
+        CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
     )
     system_test_path = os.path.join("tests", "system.py")
     system_test_folder_path = os.path.join("tests", "system")
@@ -337,7 +277,7 @@ def system(session):
     if not system_test_exists and not system_test_folder_exists:
         session.skip("System tests were not found")
 
-    install_systemtest_dependencies(session, "-c", constraints_path)
+    install_systemtest_dependencies(session, "-r", requirements_path)
 
     # Run py.test against the system tests.
     if system_test_exists:
@@ -463,32 +403,7 @@ def prerelease_deps(session, protobuf_implementation):
     `pip install --pre <package>`.
     """
 
-    # Install all dependencies
-    session.install("-e", ".")
-
-    # Install dependencies for the unit test environment
-    unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
-    session.install(*unit_deps_all)
-
-    # Because we test minimum dependency versions on the minimum Python
-    # version, the first version we test with in the unit tests sessions has a
-    # constraints file containing all dependencies and extras.
-    with open(
-        CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
-        encoding="utf-8",
-    ) as constraints_file:
-        constraints_text = constraints_file.read()
-
-    # Ignore leading whitespace and comment lines.
-    constraints_deps = [
-        match.group(1)
-        for match in re.finditer(
-            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
-        )
-    ]
-
-    # Install dependencies specified in `testing/constraints-X.txt`.
-    session.install(*constraints_deps)
+    install_unittest_dependencies(session, prerelease=True)
 
     # Note: If a dependency is added to the `prerel_deps` list,
     # the `core_dependencies_from_source` list in the `core_deps_from_source`
@@ -515,22 +430,6 @@ def prerelease_deps(session, protobuf_implementation):
         for dep in prerel_deps
     }
 
-    # Dynamically sort local packages vs PyPI dependencies
-    local_paths = []
-    pypi_deps = []
-    
-    for dep, pkg_name in parsed_deps.items():
-        if (deps_dir / pkg_name).exists():
-            local_paths.append(str(deps_dir / pkg_name))
-        else:
-            pypi_deps.append(dep)
-
-    # Batch pip installations to avoid sequential overhead
-    if local_paths:
-        session.install(*local_paths, "--no-deps", "--ignore-installed")
-    if pypi_deps:
-        session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
-
     # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
     # to the dictionary below once this bug is fixed.
     # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
@@ -565,6 +464,14 @@ def prerelease_deps(session, protobuf_implementation):
     )
 
 
+@nox.session(python=ALL_PYTHON[0])
+def security_check_minimum_version(session):
+    # Do not pin `pip-audit` since we need the latest version to be aware of new vulnerabilities
+    session.install("pip-audit")
+    session.run("pip-audit", "--version")
+    session.run("pip-audit", "-r", CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt")
+
+
 @nox.session(python=PREVIEW_PYTHON_VERSION)
 @nox.parametrize(
     "protobuf_implementation",
@@ -575,32 +482,7 @@ def core_deps_from_source(session, protobuf_implementation):
     rather than pulling the dependencies from PyPI.
     """
 
-    # Install all dependencies
-    session.install("-e", ".")
-
-    # Install dependencies for the unit test environment
-    unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
-    session.install(*unit_deps_all)
-
-    # Because we test minimum dependency versions on the minimum Python
-    # version, the first version we test with in the unit tests sessions has a
-    # constraints file containing all dependencies and extras.
-    with open(
-        CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
-        encoding="utf-8",
-    ) as constraints_file:
-        constraints_text = constraints_file.read()
-
-    # Ignore leading whitespace and comment lines.
-    constraints_deps = [
-        match.group(1)
-        for match in re.finditer(
-            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
-        )
-    ]
-
-    # Install dependencies specified in `testing/constraints-X.txt`.
-    session.install(*constraints_deps)
+    install_unittest_dependencies(session, prerelease=True)
 
     # TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and
     # `grpcio-status` should be added to the list below so that they are installed from source,

packages/gapic-generator/gapic/templates/noxfile.py.j2

@@ -46,7 +46,7 @@ dependencies = [
     {# Explicitly exclude protobuf versions mentioned in https://cloud.google.com/support/bulletins#GCP-2022-019 #}
     "protobuf >= 6.33.5, < 8.0.0",
     {% for package_tuple, package_info in pypi_packages.items() %}
-    {# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
+    {# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
     {% if api.naming.warehouse_package_name != package_info.package_name %}
     {% if api.requires_package(package_tuple) %}
     "{{ package_info.package_name }} >= {{ package_info.lower_bound }}, <{{ package_info.upper_bound }}",

packages/gapic-generator/gapic/templates/setup.py.j2

@@ -1,5 +1,6 @@
+{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %}
 {% from '_pypi_packages.j2' import pypi_packages %}
-# This constraints file is required for unit tests.
+# This requirements file is required for unit tests.
 # List all library dependencies and extras in this file.
 google-api-core
 google-auth
@@ -8,8 +9,17 @@ proto-plus
 protobuf
 # cryptography is a direct dependency of google-auth
 cryptography
+# pytest-asyncio is needed for tests otherwise async tests are skipped
+pytest-asyncio
+# pytest/pytest-cov are needed for tests
+pytest
+pytest-cov
+{% if rest_async_io_enabled %}
+aiohttp
+requests
+{% endif %}
 {% for package_tuple, package_info in pypi_packages.items() %}
-{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
+{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
 {% if api.naming.warehouse_package_name != package_info.package_name %}
 {% if api.requires_package(package_tuple) %}
 {{ package_info.package_name }}

packages/gapic-generator/gapic/templates/testing/_default_requirements.j2

@@ -1,7 +1,7 @@
 {% from '_pypi_packages.j2' import pypi_packages %}
-# We use the constraints file for the latest Python version
-# (currently this file) to check that the latest
-# major versions of dependencies are supported in setup.py.
+# We use the requirements file for the latest Python version
+# to check that the latest major versions of dependencies are supported 
+# in setup.py.
 # List all library dependencies and extras in this file.
 # Require the latest major version be installed for each dependency.
 # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
@@ -11,8 +11,13 @@ google-auth>=2
 grpcio>=1
 proto-plus>=1
 protobuf>=7
+# pytest-asyncio is needed for tests otherwise async tests are skipped
+pytest-asyncio>=1
+# pytest/pytest-cov are needed for tests
+pytest>=9
+pytest-cov>=7
 {% for package_tuple, package_info in pypi_packages.items() %}
-{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
+{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
 {% if api.naming.warehouse_package_name != package_info.package_name %}
 {% if api.requires_package(package_tuple) %}
 {{ package_info.package_name }}>={{ (package_info.upper_bound.split(".")[0] | int) - 1 }}