Skip to content

fix: use requirements files with hashes instead of constraints #17512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
parthea wants to merge 9 commits into
base: main
Choose a base branch
from
Open

fix: use requirements files with hashes instead of constraints #17512

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3aaff38
chore: rename constraints files to requirements templates
parthea Jun 18, 2026
d039a24
fix: use requirements files with hashes instead of constraints
parthea Jun 18, 2026
6002d40
Merge branch 'main' into requirements-prototype-2026-06-09
parthea Jun 23, 2026
d7bb111
attempt to speed up goldens-prerelease
parthea Jun 23, 2026
c6a6a4e
attempt to speed up goldens-prerelease
parthea Jun 23, 2026
6e1e4d4
attempt to speed up goldens-prerelease
parthea Jun 23, 2026
aaa45b9
attempt to speed up goldens-prerelease
parthea Jun 23, 2026
bd9e430
attempt to speed up goldens-prerelease
parthea Jun 23, 2026
0af6fff
revert
parthea Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
568 changes: 568 additions & 0 deletions packages/gapic-generator/_update_hashes.py
View file
Open in desktop

Large diffs are not rendered by default.

208 changes: 45 additions & 163 deletions packages/gapic-generator/gapic/templates/noxfile.py.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,38 +39,12 @@ PREVIEW_PYTHON_VERSION = "3.15"

CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()

if (CURRENT_DIRECTORY / "testing").exists():
LOWER_BOUND_CONSTRAINTS_FILE = (
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt"
)
else:
LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt"
LOWER_BOUND_REQUIREMENTS_FILE = (
CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt"
)
PACKAGE_NAME = "{{ api.naming.warehouse_package_name }}"

UNIT_TEST_STANDARD_DEPENDENCIES = [
"mock",
"asyncmock",
"pytest",
"pytest-cov",
"pytest-asyncio",
]
UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = []
UNIT_TEST_DEPENDENCIES: List[str] = []
UNIT_TEST_EXTRAS: List[str] = []
UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}

SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ALL_PYTHON
SYSTEM_TEST_STANDARD_DEPENDENCIES = [
"mock",
"pytest",
"google-cloud-testutils",
]
SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = []
SYSTEM_TEST_DEPENDENCIES: List[str] = []
SYSTEM_TEST_EXTRAS: List[str] = []
SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}

nox.options.sessions = [
"unit",
Expand Down Expand Up @@ -112,33 +86,48 @@ def mypy(session):

@nox.session
def update_lower_bounds(session):
"""Update lower bounds in constraints.txt to match setup.py"""
"""Update lower bounds in requirements.txt to match setup.py"""
session.install("google-cloud-testutils")
session.install(".")

requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"

session.run(
"lower-bound-checker",
"update",
"--package-name",
PACKAGE_NAME,
"--constraints-file",
str(LOWER_BOUND_CONSTRAINTS_FILE),
str(requirements_in),
)

session.install("pip-tools")
session.run(
"pip-compile",
"--allow-unsafe",
"--generate-hashes",
"--no-strip-extras",
"--output-file",
str(LOWER_BOUND_REQUIREMENTS_FILE),
str(requirements_in),
)


@nox.session
def check_lower_bounds(session):
"""Check lower bounds in setup.py are reflected in constraints file"""
"""Check lower bounds in setup.py are reflected in requirements file"""
session.install("google-cloud-testutils")
session.install(".")

requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"

session.run(
"lower-bound-checker",
"check",
"--package-name",
PACKAGE_NAME,
"--constraints-file",
str(LOWER_BOUND_CONSTRAINTS_FILE),
str(requirements_in),
)


Expand Down Expand Up @@ -219,32 +208,17 @@ def lint_setup_py(session):
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")


def install_unittest_dependencies(session, *constraints):
standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES
session.install(*standard_deps, *constraints)

if UNIT_TEST_EXTERNAL_DEPENDENCIES:
warnings.warn(
"'unit_test_external_dependencies' is deprecated. Instead, please "
"use 'unit_test_dependencies' or 'unit_test_local_dependencies'.",
DeprecationWarning,
def install_unittest_dependencies(session, prerelease=False):
if prerelease:
requirements_path = str(
CURRENT_DIRECTORY / "testing" / "requirements-prerelease.txt"
)
session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints)

if UNIT_TEST_LOCAL_DEPENDENCIES:
session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints)

if UNIT_TEST_EXTRAS_BY_PYTHON:
extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
elif UNIT_TEST_EXTRAS:
extras = UNIT_TEST_EXTRAS
else:
extras = []

if extras:
session.install("-e", f".[{','.join(extras)}]", *constraints)
else:
session.install("-e", ".", *constraints)
requirements_path = str(
CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
)
session.install("-r", requirements_path)
session.install("-e", ".", "--no-deps")


@nox.session(python=ALL_PYTHON)
Expand All @@ -255,10 +229,7 @@ def install_unittest_dependencies(session, *constraints):
def unit(session, protobuf_implementation):
# Install all test dependencies, then install this package in-place.

constraints_path = str(
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
)
install_unittest_dependencies(session, "-c", constraints_path)
install_unittest_dependencies(session)

# Run py.test against the unit tests.
session.run(
Expand All @@ -284,42 +255,11 @@ def unit(session, protobuf_implementation):
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2201) Add a `unit_rest_async` nox session to run tests with [async_rest] extra installed. #}


def install_systemtest_dependencies(session, *constraints):
{# Note that grpcio<=1.62.2 works on Python 3.11 but fails on 3.12+ because it requires pkg_resources for its source build. #}
if session.python >= "3.12":
session.install("--pre", "grpcio>=1.75.1")
else:
session.install("--pre", "grpcio<=1.62.2")

session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints)

if SYSTEM_TEST_EXTERNAL_DEPENDENCIES:
session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints)

if SYSTEM_TEST_LOCAL_DEPENDENCIES:
session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints)

if SYSTEM_TEST_DEPENDENCIES:
session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints)

if SYSTEM_TEST_EXTRAS_BY_PYTHON:
extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
elif SYSTEM_TEST_EXTRAS:
extras = SYSTEM_TEST_EXTRAS
else:
extras = []

if extras:
session.install("-e", f".[{','.join(extras)}]", *constraints)
else:
session.install("-e", ".", *constraints)


@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
def system(session):
"""Run the system test suite."""
constraints_path = str(
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
requirements_path = str(
CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
)
system_test_path = os.path.join("tests", "system.py")
system_test_folder_path = os.path.join("tests", "system")
Expand All @@ -337,7 +277,7 @@ def system(session):
if not system_test_exists and not system_test_folder_exists:
session.skip("System tests were not found")

install_systemtest_dependencies(session, "-c", constraints_path)
install_systemtest_dependencies(session, "-r", requirements_path)

# Run py.test against the system tests.
if system_test_exists:
Expand Down Expand Up @@ -463,32 +403,7 @@ def prerelease_deps(session, protobuf_implementation):
`pip install --pre <package>`.
"""

# Install all dependencies
session.install("-e", ".")

# Install dependencies for the unit test environment
unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
session.install(*unit_deps_all)

# Because we test minimum dependency versions on the minimum Python
# version, the first version we test with in the unit tests sessions has a
# constraints file containing all dependencies and extras.
with open(
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
encoding="utf-8",
) as constraints_file:
constraints_text = constraints_file.read()

# Ignore leading whitespace and comment lines.
constraints_deps = [
match.group(1)
for match in re.finditer(
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
)
]

# Install dependencies specified in `testing/constraints-X.txt`.
session.install(*constraints_deps)
install_unittest_dependencies(session, prerelease=True)

# Note: If a dependency is added to the `prerel_deps` list,
# the `core_dependencies_from_source` list in the `core_deps_from_source`
Expand All @@ -515,22 +430,6 @@ def prerelease_deps(session, protobuf_implementation):
for dep in prerel_deps
}

# Dynamically sort local packages vs PyPI dependencies
local_paths = []
pypi_deps = []

for dep, pkg_name in parsed_deps.items():
if (deps_dir / pkg_name).exists():
local_paths.append(str(deps_dir / pkg_name))
else:
pypi_deps.append(dep)

# Batch pip installations to avoid sequential overhead
if local_paths:
session.install(*local_paths, "--no-deps", "--ignore-installed")
if pypi_deps:
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")

# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
# to the dictionary below once this bug is fixed.
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
Expand Down Expand Up @@ -565,6 +464,14 @@ def prerelease_deps(session, protobuf_implementation):
)


@nox.session(python=ALL_PYTHON[0])
def security_check_minimum_version(session):
# Do not pin `pip-audit` since we need the latest version to be aware of new vulnerabilities
session.install("pip-audit")
session.run("pip-audit", "--version")
session.run("pip-audit", "-r", CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt")


@nox.session(python=PREVIEW_PYTHON_VERSION)
@nox.parametrize(
"protobuf_implementation",
Expand All @@ -575,32 +482,7 @@ def core_deps_from_source(session, protobuf_implementation):
rather than pulling the dependencies from PyPI.
"""

# Install all dependencies
session.install("-e", ".")

# Install dependencies for the unit test environment
unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
session.install(*unit_deps_all)

# Because we test minimum dependency versions on the minimum Python
# version, the first version we test with in the unit tests sessions has a
# constraints file containing all dependencies and extras.
with open(
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
encoding="utf-8",
) as constraints_file:
constraints_text = constraints_file.read()

# Ignore leading whitespace and comment lines.
constraints_deps = [
match.group(1)
for match in re.finditer(
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
)
]

# Install dependencies specified in `testing/constraints-X.txt`.
session.install(*constraints_deps)
install_unittest_dependencies(session, prerelease=True)

# TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and
# `grpcio-status` should be added to the list below so that they are installed from source,
Expand Down
2 changes: 1 addition & 1 deletion packages/gapic-generator/gapic/templates/setup.py.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies = [
{# Explicitly exclude protobuf versions mentioned in https://cloud.google.com/support/bulletins#GCP-2022-019 #}
"protobuf >= 6.33.5, < 8.0.0",
{% for package_tuple, package_info in pypi_packages.items() %}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
{% if api.naming.warehouse_package_name != package_info.package_name %}
{% if api.requires_package(package_tuple) %}
"{{ package_info.package_name }} >= {{ package_info.lower_bound }}, <{{ package_info.upper_bound }}",
Expand Down
14 changes: 12 additions & 2 deletions ...templates/testing/_default_constraints.j2 → ...emplates/testing/_default_requirements.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %}
{% from '_pypi_packages.j2' import pypi_packages %}
# This constraints file is required for unit tests.
# This requirements file is required for unit tests.
# List all library dependencies and extras in this file.
google-api-core
google-auth
Expand All @@ -8,8 +9,17 @@ proto-plus
protobuf
# cryptography is a direct dependency of google-auth
cryptography
# pytest-asyncio is needed for tests otherwise async tests are skipped
pytest-asyncio
# pytest/pytest-cov are needed for tests
pytest
pytest-cov
{% if rest_async_io_enabled %}
aiohttp
requests
{% endif %}
{% for package_tuple, package_info in pypi_packages.items() %}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
{% if api.naming.warehouse_package_name != package_info.package_name %}
{% if api.requires_package(package_tuple) %}
{{ package_info.package_name }}
Expand Down
13 changes: 9 additions & 4 deletions ...templates/testing/constraints-3.15.txt.j2 → ...tes/testing/_latest_major_requirements.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{% from '_pypi_packages.j2' import pypi_packages %}
# We use the constraints file for the latest Python version
# (currently this file) to check that the latest
# major versions of dependencies are supported in setup.py.
# We use the requirements file for the latest Python version
# to check that the latest major versions of dependencies are supported
# in setup.py.
# List all library dependencies and extras in this file.
# Require the latest major version be installed for each dependency.
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
Expand All @@ -11,8 +11,13 @@ google-auth>=2
grpcio>=1
proto-plus>=1
protobuf>=7
# pytest-asyncio is needed for tests otherwise async tests are skipped
pytest-asyncio>=1
# pytest/pytest-cov are needed for tests
pytest>=9
pytest-cov>=7
{% for package_tuple, package_info in pypi_packages.items() %}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
{% if api.naming.warehouse_package_name != package_info.package_name %}
{% if api.requires_package(package_tuple) %}
{{ package_info.package_name }}>={{ (package_info.upper_bound.split(".")[0] | int) - 1 }}
Expand Down
20 changes: 0 additions & 20 deletions packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2
View file
Open in desktop

This file was deleted.

4 changes: 0 additions & 4 deletions packages/gapic-generator/gapic/templates/testing/constraints-3.11.txt.j2
View file
Open in desktop

This file was deleted.

Loading
Loading