noxfile.py

# Copyright 2017, Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Helpful notes for local usage:
#   unset PYENV_VERSION
#   pyenv local 3.14.1 3.13.10 3.12.11 3.11.4 3.10.12 3.9.17
#   PIP_INDEX_URL=https://pypi.org/simple nox

from __future__ import absolute_import
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import os
import sys
import tempfile
import typing
import nox  # type: ignore

from contextlib import contextmanager
from os import path
import shutil


nox.options.error_on_missing_interpreters = True


showcase_version = os.environ.get("SHOWCASE_VERSION", "0.35.0")
ADS_TEMPLATES = path.join(path.dirname(__file__), "gapic", "ads-templates")
RUFF_VERSION = "ruff==0.14.14"
LINT_PATHS = ["docs", "gapic", "tests", "test_utils", "noxfile.py", "setup.py"]
# Ruff uses globs for excludes (different from Black's regex)
# .*golden.* -> *golden*
# .*pb2.py -> *pb2.py
RUFF_EXCLUDES = "*golden*,*pb2.py,*pb2.pyi"

ALL_PYTHON = (
    "3.9",
    "3.10",
    "3.11",
    "3.12",
    "3.13",
    "3.14",
)

NEWEST_PYTHON = ALL_PYTHON[-1]


@nox.session(python=ALL_PYTHON)
def unit(session):
    """Run the unit test suite."""
    session.install(
        # TODO(https://github.com/googleapis/gapic-generator-python/issues/2478):
        # Temporarily pin coverage to 7.11.0
        # See https://github.com/nedbat/coveragepy/issues/2077
        "coverage<=7.11.0",
        "pytest-cov",
        "pytest",
        "pytest-xdist",
        "pyfakefs",
        "grpcio-status",
        "proto-plus",
    )
    session.install("-e", ".")
    session.run(
        "py.test",
        *(
            session.posargs
            or [
                "-vv",
                "-n=auto",
                "--cov=gapic",
                "--cov-config=.coveragerc",
                "--cov-report=term",
                "--cov-fail-under=100",
                path.join("tests", "unit"),
            ]
        ),
    )


FRAG_DIR = Path("tests") / "fragments"
FRAGMENT_FILES = tuple(
    Path(dirname).relative_to(FRAG_DIR) / f
    for dirname, _, files in os.walk(FRAG_DIR)
    for f in files
    if os.path.splitext(f)[1] == ".proto" and f.startswith("test_")
)


# Note: this class lives outside 'fragment'
# so that, if necessary, it can be pickled for a ProcessPoolExecutor
# A callable class is necessary so that the session can be closed over
# instead of passed in, which simplifies the invocation via map.
class FragTester:
    def __init__(self, session, use_ads_templates):
        self.session = session
        self.use_ads_templates = use_ads_templates

    def __call__(self, frag):
        with tempfile.TemporaryDirectory() as tmp_dir:
            # Generate the fragment GAPIC.
            outputs = []
            templates = (
                path.join(path.dirname(__file__), "gapic", "ads-templates")
                if self.use_ads_templates
                else "DEFAULT"
            )
            maybe_old_naming = ",old-naming" if self.use_ads_templates else ""

            session_args = [
                "python",
                "-m",
                "grpc_tools.protoc",
                f"--proto_path={str(FRAG_DIR)}",
                f"--python_gapic_out={tmp_dir}",
                f"--python_gapic_opt=transport=grpc+rest,python-gapic-templates={templates}{maybe_old_naming}",
            ]

            outputs.append(
                self.session.run(
                    *session_args,
                    str(frag),
                    external=True,
                    silent=True,
                )
            )

            # Install the generated fragment library.
            if self.use_ads_templates:
                self.session.install(tmp_dir, "-e", ".", "-qqq")
            else:
                # Use the constraints file for the specific python runtime version.
                # We do this to make sure that we're testing against the lowest
                # supported version of a dependency.
                # This is needed to recreate the issue reported in
                # https://github.com/googleapis/gapic-generator-python/issues/1748
                # The ads templates do not have constraints files.
                constraints_path = str(
                    f"{tmp_dir}/testing/constraints-{self.session.python}.txt"
                )
                self.session.install(tmp_dir, "-e", ".", "-qqq", "-r", constraints_path)

            # Run the fragment's generated unit tests.
            # Don't bother parallelizing them: we already parallelize
            # the fragments, and there usually aren't too many tests per fragment.
            outputs.append(
                self.session.run(
                    "py.test",
                    "--quiet",
                    f"--cov-config={str(Path(tmp_dir) / '.coveragerc')}",
                    "--cov-report=term",
                    "--cov-fail-under=100",
                    str(Path(tmp_dir) / "tests" / "unit"),
                    silent=True,
                )
            )
            # Only run mypy on the latest python
            if self.session.python in (ALL_PYTHON[-1]):
                self.session.run(
                    "mypy",
                    str(tmp_dir),
                    "--check-untyped-defs",
                    "--exclude",
                    "build/",
                    "--exclude",
                    "tests/",
                )
            return "".join(outputs)


@nox.session(python=ALL_PYTHON)
def fragment(session, use_ads_templates=False):
    session.install(
        "coverage",
        "pytest",
        "pytest-cov",
        "pytest-xdist",
        "pytest-asyncio",
        "grpcio-tools",
        "mypy",
        "types-protobuf",
        "types-requests",
    )
    session.install("-e", ".")

    # The specific failure is `Plugin output is unparseable`
    if session.python in ("3.9", "3.10"):
        session.install("google-api-core<2.28")

    frag_files = (
        [Path(f) for f in session.posargs] if session.posargs else FRAGMENT_FILES
    )

    # Skip test_iam.proto when using ads templates
    # There are known issues with ads mixins
    # https://github.com/googleapis/gapic-generator-python/issues/2182
    # In addition, ads templates may be removed soon
    # https://github.com/googleapis/gapic-generator-python/issues/1994
    if use_ads_templates:
        frag_files = [f for f in frag_files if f.name != "test_iam.proto"]

    if os.environ.get("PARALLEL_FRAGMENT_TESTS", "false").lower() == "true":
        with ThreadPoolExecutor() as p:
            all_outs = p.map(FragTester(session, use_ads_templates), frag_files)

        output = "".join(all_outs)
        session.log(output)
    else:
        tester = FragTester(session, use_ads_templates)
        for frag in frag_files:
            session.log(tester(frag))


@nox.session(python=ALL_PYTHON)
def fragment_alternative_templates(session):
    fragment(session, use_ads_templates=True)


# `_add_python_settings` consumes a path to a temporary directory (str; i.e. tmp_dir) and
# python settings (Dict; i.e. python settings) and modifies the service yaml within
# tmp_dir to include python settings. The primary purpose of this function is to modify
# the service yaml and include `rest_async_io_enabled=True` to test the async rest
# optional feature.
def _add_python_settings(tmp_dir, python_settings):
    return f"""
import yaml
from pathlib import Path
temp_file_path = Path(f"{tmp_dir}/showcase_v1beta1.yaml")
with temp_file_path.open('r') as file:
    data = yaml.safe_load(file)
    data['publishing']['library_settings'] = {python_settings}

with temp_file_path.open('w') as file:
    yaml.safe_dump(data, file, default_flow_style=False, sort_keys=False)
"""


# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): `rest_async_io_enabled` must be removed once async rest is GA.
@contextmanager
def showcase_library(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
    include_service_yaml=True,
    retry_config=True,
    rest_async_io_enabled=False,
):
    """Install the generated library into the session for showcase tests."""

    session.log("-" * 70)
    session.log("Note: Showcase must be running for these tests to work.")
    session.log("See https://github.com/googleapis/gapic-showcase")
    session.log("-" * 70)

    # Install gapic-generator-python
    session.install("-e", ".")

    # Install grpcio-tools for protoc
    session.install("grpcio-tools")

    # TODO(https://github.com/googleapis/gapic-generator-python/issues/2473):
    # Warnings emitted from google-api-core starting in 2.28
    # appear to cause issues when running protoc.
    # The specific failure is `Plugin output is unparseable`
    if session.python in ("3.9", "3.10"):
        session.install("google-api-core<2.28")

    # Install a client library for Showcase.
    with tempfile.TemporaryDirectory() as tmp_dir:
        # Download the Showcase descriptor.
        session.run(
            "curl",
            "https://github.com/googleapis/gapic-showcase/releases/"
            f"download/v{showcase_version}/"
            f"gapic-showcase-{showcase_version}.desc",
            "-L",
            "--output",
            path.join(tmp_dir, "showcase.desc"),
            external=True,
            silent=True,
        )
        if include_service_yaml:
            session.run(
                "curl",
                "https://github.com/googleapis/gapic-showcase/releases/"
                f"download/v{showcase_version}/"
                f"showcase_v1beta1.yaml",
                "-L",
                "--output",
                path.join(tmp_dir, "showcase_v1beta1.yaml"),
                external=True,
                silent=True,
            )
            # TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): The section below updates the showcase service yaml
            # to test experimental async rest transport. It must be removed once support for async rest is GA.
            if rest_async_io_enabled:
                # Install pyYAML for yaml.
                session.install("pyYAML")

                python_settings = [
                    {
                        "version": "google.showcase.v1beta1",
                        "python_settings": {
                            "experimental_features": {"rest_async_io_enabled": True}
                        },
                    }
                ]
                update_service_yaml = _add_python_settings(tmp_dir, python_settings)
                session.run("python", "-c", f"{update_service_yaml}")
            # END TODO section to remove.
        if retry_config:
            session.run(
                "curl",
                "https://github.com/googleapis/gapic-showcase/releases/"
                f"download/v{showcase_version}/"
                f"showcase_grpc_service_config.json",
                "-L",
                "--output",
                path.join(tmp_dir, "showcase_grpc_service_config.json"),
                external=True,
                silent=True,
            )
        # Write out a client library for Showcase.
        template_opt = f"python-gapic-templates={templates}"
        opts = "--python_gapic_opt="
        if include_service_yaml and retry_config:
            opts += ",".join(
                other_opts
                + (
                    f"{template_opt}",
                    "transport=grpc+rest",
                    f"service-yaml={tmp_dir}/showcase_v1beta1.yaml",
                    f"retry-config={tmp_dir}/showcase_grpc_service_config.json",
                )
            )
        else:
            opts += ",".join(
                other_opts
                + (
                    f"{template_opt}",
                    "transport=grpc+rest",
                )
            )
        cmd_tup = (
            "python",
            "-m",
            "grpc_tools.protoc",
            f"--experimental_allow_proto3_optional",
            f"--descriptor_set_in={tmp_dir}{path.sep}showcase.desc",
            opts,
            f"--python_gapic_out={tmp_dir}",
            f"google/showcase/v1beta1/echo.proto",
            f"google/showcase/v1beta1/identity.proto",
            f"google/showcase/v1beta1/messaging.proto",
        )
        session.run(
            *cmd_tup,
            external=True,
        )

        # Install the generated showcase library.
        if templates == "DEFAULT":
            # Use the constraints file for the specific python runtime version.
            # We do this to make sure that we're testing against the lowest
            # supported version of a dependency.
            # This is needed to recreate the issue reported in
            # https://github.com/googleapis/google-cloud-python/issues/12254
            constraints_path = str(
                f"{tmp_dir}/testing/constraints-{session.python}.txt"
            )
            # Install the library with a constraints file.
            session.install(
                "-e",
                tmp_dir + ("[async_rest]" if rest_async_io_enabled else ""),
                "-r",
                constraints_path,
            )
            # Exclude `google-auth==2.40.0` which contains a regression
            # https://github.com/googleapis/gapic-generator-python/issues/2385
            session.install(
                "--no-cache-dir",
                "--force-reinstall",
                "--upgrade",
                "google-auth[aiohttp]!=2.40.0",
            )
        else:
            # The ads templates do not have constraints files.
            # See https://github.com/googleapis/gapic-generator-python/issues/1788
            # Install the library without a constraints file.
            session.install("-e", tmp_dir)

        yield tmp_dir


@nox.session(python=ALL_PYTHON)
def showcase(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
    env: typing.Optional[typing.Dict[str, str]] = {},
):
    """Run the Showcase test suite."""

    with showcase_library(session, templates=templates, other_opts=other_opts):
        # Use pytest-asyncio<1.0.0 while we investigate the recent failure described in
        # https://github.com/googleapis/gapic-generator-python/issues/2399
        session.install("pytest", "pytest-asyncio<1.0.0")
        test_directory = Path("tests", "system")
        ignore_file = env.get("IGNORE_FILE")
        pytest_command = [
            "py.test",
            "--quiet",
            *(session.posargs or [str(test_directory)]),
        ]
        if ignore_file:
            ignore_path = test_directory / ignore_file
            pytest_command.extend(["--ignore", str(ignore_path)])

        session.run(
            *pytest_command,
            env=env,
        )


@nox.session(python=ALL_PYTHON)
def showcase_w_rest_async(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
    env: typing.Optional[typing.Dict[str, str]] = {},
):
    """Run the Showcase test suite."""

    with showcase_library(
        session, templates=templates, other_opts=other_opts, rest_async_io_enabled=True
    ):
        # Use pytest-asyncio<1.0.0 while we investigate the recent failure described in
        # https://github.com/googleapis/gapic-generator-python/issues/2399
        session.install("pytest", "pytest-asyncio<1.0.0")
        test_directory = Path("tests", "system")
        ignore_file = env.get("IGNORE_FILE")
        pytest_command = [
            "py.test",
            "--quiet",
            *(session.posargs or [str(test_directory)]),
        ]
        if ignore_file:
            ignore_path = test_directory / ignore_file
            pytest_command.extend(["--ignore", str(ignore_path)])

        session.run(
            *pytest_command,
            env=env,
        )


@nox.session(python=NEWEST_PYTHON)
def showcase_mtls(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
    env: typing.Optional[typing.Dict[str, str]] = {},
):
    """Run the Showcase mtls test suite."""

    with showcase_library(session, templates=templates, other_opts=other_opts):
        session.install("pytest", "pytest-asyncio")
        test_directory = Path("tests", "system")
        ignore_file = env.get("IGNORE_FILE")
        pytest_command = [
            "py.test",
            "--quiet",
            "--mtls",
            *(session.posargs or [str(test_directory)]),
        ]
        if ignore_file:
            ignore_path = test_directory / ignore_file
            pytest_command.extend(["--ignore", str(ignore_path)])

        session.run(
            *pytest_command,
            env=env,
        )


def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False):
    session.install(
        "coverage",
        "pytest",
        "pytest-cov",
        "pytest-xdist",
        "pytest-asyncio",
    )
    # Run the tests.
    session.run(
        "py.test",
        *(
            session.posargs
            or [
                "-n=auto",
                "--quiet",
                "--cov=google",
                "--cov-append",
                f"--cov-fail-under={str(fail_under)}",
                path.join("tests", "unit"),
            ]
        ),
    )


@nox.session(python=ALL_PYTHON)
def showcase_unit(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
):
    """Run the generated unit tests against the Showcase library."""
    with showcase_library(session, templates=templates, other_opts=other_opts) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session)


# TODO: `showcase_unit_w_rest_async` nox session runs showcase unit tests with the
# experimental async rest transport and must be removed once support for async rest is GA.
# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2121.
@nox.session(python=ALL_PYTHON)
def showcase_unit_w_rest_async(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
):
    """Run the generated unit tests with async rest transport against the Showcase library."""
    with showcase_library(
        session, templates=templates, other_opts=other_opts, rest_async_io_enabled=True
    ) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session, rest_async_io_enabled=True)


@nox.session(python=ALL_PYTHON)
def showcase_unit_alternative_templates(session):
    with showcase_library(
        session, templates=ADS_TEMPLATES, other_opts=("old-naming",)
    ) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session)


@nox.session(python=NEWEST_PYTHON)
def showcase_unit_add_iam_methods(session):
    with showcase_library(session, other_opts=("add-iam-methods",)) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session, fail_under=100)


@nox.session(python=ALL_PYTHON)
def showcase_unit_mixins(session):
    with showcase_library(session, include_service_yaml=True) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session)


@nox.session(python=ALL_PYTHON)
def showcase_unit_alternative_templates_mixins(session):
    with showcase_library(
        session,
        templates=ADS_TEMPLATES,
        other_opts=("old-naming",),
        include_service_yaml=True,
    ) as lib:
        session.chdir(lib)
        run_showcase_unit_tests(session)


@nox.session(python=NEWEST_PYTHON)
def showcase_mypy(
    session,
    templates="DEFAULT",
    other_opts: typing.Iterable[str] = (),
):
    """Perform typecheck analysis on the generated Showcase library."""

    session.install(
        "mypy",
        "types-setuptools",
        "types-protobuf",
        "types-requests",
        "types-dataclasses",
    )

    with showcase_library(session, templates=templates, other_opts=other_opts) as lib:
        session.chdir(lib)

        # Run the tests.
        session.run("mypy", "-p", "google", "--check-untyped-defs")


@nox.session(python=NEWEST_PYTHON)
def showcase_mypy_alternative_templates(session):
    showcase_mypy(session, templates=ADS_TEMPLATES, other_opts=("old-naming",))


@nox.session(python=NEWEST_PYTHON)
def snippetgen(session):
    # Clone googleapis/api-common-protos which are referenced by the snippet
    # protos
    api_common_protos = "api-common-protos"
    try:
        session.run("git", "-C", api_common_protos, "pull", external=True)
    except nox.command.CommandFailed:
        session.run(
            "git",
            "clone",
            "--single-branch",
            f"https://github.com/googleapis/{api_common_protos}",
            external=True,
        )

    # Install gapic-generator-python
    session.install("-e", ".")

    session.install("grpcio-tools", "pytest", "pytest-asyncio")

    session.run("py.test", "-vv", "tests/snippetgen")


@nox.session(python="3.10")
def docs(session):
    """Build the docs for this generator."""

    session.install("-e", ".")

    session.install(
        # We need to pin to specific versions of the `sphinxcontrib-*` packages
        # which still support sphinx 4.x.
        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
        "sphinxcontrib-applehelp==1.0.4",
        "sphinxcontrib-devhelp==1.0.2",
        "sphinxcontrib-htmlhelp==2.0.1",
        "sphinxcontrib-qthelp==1.0.3",
        "sphinxcontrib-serializinghtml==1.1.5",
        "sphinx==4.5.0",
        "sphinx-rtd-theme",
    )

    shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
    session.run(
        "sphinx-build",
        "-W",  # warnings as errors
        "-T",  # show full traceback on exception
        "-N",  # no colors
        "-b",
        "html",  # builder
        "-d",
        os.path.join("docs", "_build", "doctrees", ""),  # cache directory
        # paths to build:
        os.path.join("docs", ""),
        os.path.join("docs", "_build", "html", ""),
    )


@nox.session(python="3.10")
def docfx(session):
    """Build the docfx yaml files for this library."""

    session.install("-e", ".")
    session.install(
        # We need to pin to specific versions of the `sphinxcontrib-*` packages
        # which still support sphinx 4.x.
        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
        "sphinxcontrib-applehelp==1.0.4",
        "sphinxcontrib-devhelp==1.0.2",
        "sphinxcontrib-htmlhelp==2.0.1",
        "sphinxcontrib-qthelp==1.0.3",
        "sphinxcontrib-serializinghtml==1.1.5",
        "gcp-sphinx-docfx-yaml",
        "sphinx-rtd-theme",
    )

    shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
    session.run(
        "sphinx-build",
        "-T",  # show full traceback on exception
        "-N",  # no colors
        "-D",  # Override configuration values set in the conf.py file
        (
            "extensions=sphinx.ext.autodoc,"
            "sphinx.ext.autosummary,"
            "docfx_yaml.extension,"
            "sphinx.ext.intersphinx,"
            "sphinx.ext.coverage,"
            "sphinx.ext.napoleon,"
            "sphinx.ext.todo,"
            "sphinx.ext.viewcode,"
            "recommonmark"
        ),
        "-b",
        "html",  # builder
        "-d",
        os.path.join("docs", "_build", "doctrees", ""),  # cache directory
        # paths to build:
        os.path.join("docs", ""),
        os.path.join("docs", "_build", "html", ""),
    )


@nox.session(python=ALL_PYTHON)
def mypy(session):
    """Perform typecheck analysis."""
    # Pin to click==8.1.3 to workaround https://github.com/pallets/click/issues/2558
    session.install(
        "mypy",
        "types-protobuf<=3.19.7",
        "types-PyYAML",
        "types-dataclasses",
        "click==8.1.3",
    )
    session.install(".")
    session.run("mypy", "-p", "gapic")


@nox.session(python=NEWEST_PYTHON)
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.install("flake8", RUFF_VERSION)

    # 2. Check formatting
    session.run(
        "ruff",
        "format",
        "--check",
        *LINT_PATHS,
        "--exclude",
        RUFF_EXCLUDES,
    )

    # 3. Run Flake8
    session.run(
        "flake8",
        "gapic",
        "tests",
    )


@nox.session(python="3.10")
def blacken(session):
    """Run ruff format.

    DEPRECATED: This session now uses Ruff instead of Black.
    It formats code style only (indentation, quotes, etc).
    """
    session.log(
        "WARNING: The 'blacken' session is deprecated and will be removed in the next release. Please use 'nox -s format' in the future."
    )

    session.install(RUFF_VERSION)

    # 1. Format Code (Replaces black)
    # We do NOT run 'ruff check --select I' here, preserving strict parity.
    session.run(
        "ruff",
        "format",
        "--line-length=88",  # Standard Black line length
        *LINT_PATHS,
        "--exclude",
        RUFF_EXCLUDES,
    )


@nox.session(python=NEWEST_PYTHON)
def format(session):
    """
    Run ruff to sort imports and format code.
    """
    # 1. Install ruff (skipped automatically if you run with --no-venv)
    session.install(RUFF_VERSION)

    # 2. Run Ruff to fix imports
    # check --select I: Enables strict import sorting
    # --fix: Applies the changes automatically
    session.run(
        "ruff",
        "check",
        "--select",
        "I",
        "--fix",
        "--line-length=88",  # Standard Black line length
        *LINT_PATHS,
    )

    # 3. Run Ruff to format code
    session.run(
        "ruff",
        "format",
        "--line-length=88",  # Standard Black line length
        *LINT_PATHS,
    )