diff --git a/packages/bigframes/noxfile.py b/packages/bigframes/noxfile.py index 537d417e9145..cb57923288a6 100644 --- a/packages/bigframes/noxfile.py +++ b/packages/bigframes/noxfile.py @@ -20,7 +20,6 @@ import multiprocessing import os import pathlib -import re import shutil import time from typing import Dict, List @@ -62,7 +61,6 @@ "pytest-cov", "pytest-timeout", ] -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] UNIT_TEST_DEPENDENCIES: List[str] = [] UNIT_TEST_EXTRAS: List[str] = ["tests"] UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { @@ -95,7 +93,7 @@ SYSTEM_TEST_EXTERNAL_DEPENDENCIES = [ "google-cloud-bigquery", ] -SYSTEM_TEST_EXTRAS: List[str] = [] +SYSTEM_TEST_EXTRAS: List[str] = ["tests"] SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { # Make sure we leave some versions without "extras" so we know those # dependencies are actually optional. @@ -257,14 +255,10 @@ def run_unit(session, install_test_extra): @nox.session(python=ALL_PYTHON) -@nox.parametrize("test_extra", [True, False]) -def unit(session, test_extra): +def unit(session): if session.python in ("3.7", "3.8", "3.9"): session.skip("Python 3.9 and below are not supported") - if test_extra: - run_unit(session, install_test_extra=test_extra) - else: - unit_noextras(session) + run_unit(session, install_test_extra=True) @nox.session(python=ALL_PYTHON[-1]) @@ -367,15 +361,14 @@ def run_system( @nox.session(python="3.12") def system(session: nox.sessions.Session): """Run the system test suite.""" - if session.python in ("3.7", "3.8", "3.9"): - session.skip("Python 3.9 and below are not supported") - + # TODO(https://github.com/googleapis/google-cloud-python/issues/16489): Restore system test once this bug is fixed run_system( session=session, prefix_name="system", test_folder=os.path.join("tests", "system", "small"), check_cov=True, ) + # session.skip("Temporarily skip system test") @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -392,10 +385,6 @@ def system_noextras(session: nox.sessions.Session): @nox.session(python="3.12") def doctest(session: nox.sessions.Session): """Run the system test suite.""" - session.skip( - "Temporary skip to enable a PR merge. Remove skip as part of closing https://github.com/googleapis/google-cloud-python/issues/16489" - ) - run_system( session=session, prefix_name="doctest", @@ -451,10 +440,6 @@ def cover(session): This outputs the coverage report aggregating coverage from the test runs (including system test runs), and then erases coverage data. """ - # TODO: Remove this skip when the issue is resolved. - # https://github.com/googleapis/google-cloud-python/issues/16635 - session.skip("Temporarily skip coverage session") - session.install("coverage", "pytest-cov") # Create a coverage report that includes only the product code. @@ -936,129 +921,69 @@ def cleanup(session): @nox.session(python=DEFAULT_PYTHON_VERSION) -@nox.parametrize( - "protobuf_implementation", - ["python", "upb"], -) -def core_deps_from_source(session, protobuf_implementation): +def core_deps_from_source(session): """Run all tests with core dependencies installed from source rather than pulling the dependencies from PyPI. """ - - # Install all dependencies - session.install("-e", ".") - - # Install dependencies for the unit test environment - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - - # Install dependencies for the system test environment - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES - + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - + SYSTEM_TEST_EXTRAS - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY / "testing" / "constraints-3.10.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - # Fiona fails to build on GitHub CI because gdal-config is missing and no Python 3.14 wheels are available. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - if match.group(1) != "fiona" - ] - - # Install dependencies specified in `testing/constraints-X.txt`. - session.install(*constraints_deps) - - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and - # `grpcio-status` should be added to the list below so that they are installed from source, - # rather than PyPI. - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2357): `protobuf` should be - # added to the list below so that it is installed from source, rather than PyPI - # Note: If a dependency is added to the `core_dependencies_from_source` list, - # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. - core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", - ] - - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) + # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): + # Add core deps from source tests + session.skip("Core deps from source tests are not yet supported") -@nox.session(python=ALL_PYTHON[-1]) +@nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease_deps(session): """Run all tests with prerelease versions of dependencies installed.""" # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): # Add prerelease deps tests - unit_prerelease(session) - system_prerelease(session) + session.skip("prerelease deps tests are not yet supported") -# NOTE: this is based on mypy session that came directly from the bigframes split repo -# the split repo used 3.10, the monorepo uses 3.14 -@nox.session(python="3.14") +@nox.session(python=DEFAULT_PYTHON_VERSION) def mypy(session): - """Run type checks with mypy.""" - # Editable mode is not compatible with mypy when there are multiple - # package directories. See: - # https://github.com/python/mypy/issues/10564#issuecomment-851687749 - session.install(".") - - # Just install the dependencies' type info directly, since "mypy --install-types" - # might require an additional pass. - deps = ( - set( - [ - MYPY_VERSION, - # TODO: update to latest pandas-stubs once we resolve bigframes issues. - "pandas-stubs<=2.2.3.241126", - "types-protobuf", - "types-python-dateutil", - "types-requests", - "types-setuptools", - "types-tabulate", - "types-PyYAML", - "polars", - "anywidget", - ] - ) - | set(SYSTEM_TEST_STANDARD_DEPENDENCIES) - | set(UNIT_TEST_STANDARD_DEPENDENCIES) - ) - - session.install(*deps) - shutil.rmtree(".mypy_cache", ignore_errors=True) - session.run( - "mypy", - "bigframes", - os.path.join("tests", "system"), - os.path.join("tests", "unit"), - "--check-untyped-defs", - "--explicit-package-bases", - '--exclude="^third_party"', - ) + """Run the type checker.""" + # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): + # Add mypy tests previously used mypy session (below) failed to run in the monorepo + session.skip("mypy tests are not yet supported") + + +# @nox.session(python=ALL_PYTHON) +# def mypy(session): +# """Run type checks with mypy.""" +# # Editable mode is not compatible with mypy when there are multiple +# # package directories. See: +# # https://github.com/python/mypy/issues/10564#issuecomment-851687749 +# session.install(".") + +# # Just install the dependencies' type info directly, since "mypy --install-types" +# # might require an additional pass. +# deps = ( +# set( +# [ +# MYPY_VERSION, +# # TODO: update to latest pandas-stubs once we resolve bigframes issues. +# "pandas-stubs<=2.2.3.241126", +# "types-protobuf", +# "types-python-dateutil", +# "types-requests", +# "types-setuptools", +# "types-tabulate", +# "types-PyYAML", +# "polars", +# "anywidget", +# ] +# ) +# | set(SYSTEM_TEST_STANDARD_DEPENDENCIES) +# | set(UNIT_TEST_STANDARD_DEPENDENCIES) +# ) + +# session.install(*deps) +# shutil.rmtree(".mypy_cache", ignore_errors=True) +# session.run( +# "mypy", +# "bigframes", +# os.path.join("tests", "system"), +# os.path.join("tests", "unit"), +# "--check-untyped-defs", +# "--explicit-package-bases", +# '--exclude="^third_party"', +# ) diff --git a/packages/bigquery-magics/noxfile.py b/packages/bigquery-magics/noxfile.py index 5dbe8aaf3e9a..988d9b4f7b71 100644 --- a/packages/bigquery-magics/noxfile.py +++ b/packages/bigquery-magics/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "bigquery_magics", "tests", "noxfile.py", "setup.py"] @@ -170,19 +171,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index c7200dce77f1..2a07b72321d8 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "db_dtypes", "tests", "noxfile.py", "setup.py"] @@ -122,20 +123,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections - session.run("python", "-m", "pip", "freeze") + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-api-core/noxfile.py b/packages/google-api-core/noxfile.py index ca3238763c1d..3ae96e416736 100644 --- a/packages/google-api-core/noxfile.py +++ b/packages/google-api-core/noxfile.py @@ -18,6 +18,7 @@ # PIP_INDEX_URL=https://pypi.org/simple nox from __future__ import absolute_import + import os import pathlib import re @@ -27,8 +28,8 @@ # https://github.com/google/importlab/issues/25 import nox - BLACK_VERSION = "black==23.7.0" +RUFF_VERSION = "ruff==0.14.14" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] # Black and flake8 clash on the syntax for ignoring flake8's F401 in this file. BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"] @@ -72,6 +73,36 @@ def blacken(session): session.run("black", *BLACK_EXCLUDES, *BLACK_PATHS) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run ruff to sort imports and format code. + """ + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports + session.run( + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + # 3. Run Ruff to format code + session.run( + "ruff", + "format", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + def install_prerelease_dependencies(session, constraints_path): with open(constraints_path, encoding="utf-8") as constraints_file: constraints_text = constraints_file.read() diff --git a/packages/google-auth-httplib2/noxfile.py b/packages/google-auth-httplib2/noxfile.py index f5807d1fce5b..df9b4c2656e9 100644 --- a/packages/google-auth-httplib2/noxfile.py +++ b/packages/google-auth-httplib2/noxfile.py @@ -20,13 +20,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google_auth_httplib2.py", "tests", "noxfile.py", "setup.py"] @@ -118,19 +119,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-auth-oauthlib/noxfile.py b/packages/google-auth-oauthlib/noxfile.py index 7b6637889775..8af6dc8319d5 100644 --- a/packages/google-auth-oauthlib/noxfile.py +++ b/packages/google-auth-oauthlib/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google_auth_oauthlib", "tests", "noxfile.py", "setup.py"] @@ -103,19 +104,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-auth/noxfile.py b/packages/google-auth/noxfile.py index ca829fa96030..78dd7805424a 100644 --- a/packages/google-auth/noxfile.py +++ b/packages/google-auth/noxfile.py @@ -22,6 +22,7 @@ CLICK_VERSION = "click" BLACK_VERSION = "black==23.7.0" +RUFF_VERSION = "ruff==0.14.14" BLACK_PATHS = [ "google", "tests", @@ -51,6 +52,7 @@ nox.options.sessions = [ "lint", "blacken", + "format", "mypy", # cover must be last to avoid error `No data to report` "docs", @@ -97,6 +99,36 @@ def blacken(session): session.run("black", *BLACK_PATHS) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run ruff to sort imports and format code. + """ + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports + session.run( + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + # 3. Run Ruff to format code + session.run( + "ruff", + "format", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def mypy(session): """Verify type hints are mypy compatible.""" diff --git a/packages/google-cloud-core/noxfile.py b/packages/google-cloud-core/noxfile.py index 400a0235d6a4..e6f33b60ad65 100644 --- a/packages/google-cloud-core/noxfile.py +++ b/packages/google-cloud-core/noxfile.py @@ -13,14 +13,15 @@ # limitations under the License. from __future__ import absolute_import + import os import re import shutil import nox - BLACK_VERSION = "black==23.7.0" +RUFF_VERSION = "ruff==0.14.14" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.14" @@ -75,6 +76,36 @@ def blacken(session): session.run("black", *BLACK_PATHS) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run ruff to sort imports and format code. + """ + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports + session.run( + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + # 3. Run Ruff to format code + session.run( + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + def default(session): """Default unit test session. This is intended to be run **without** an interpreter set, so @@ -342,7 +373,7 @@ def core_deps_from_source(session, protobuf_implementation): f"{CURRENT_DIRECTORY}/../google-api-core", f"{CURRENT_DIRECTORY}/../google-auth", f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus" + f"{CURRENT_DIRECTORY}/../proto-plus", ] for dep in core_dependencies_from_source: diff --git a/packages/google-cloud-dns/noxfile.py b/packages/google-cloud-dns/noxfile.py index 5df61c9c4a2f..548f9189a2c0 100644 --- a/packages/google-cloud-dns/noxfile.py +++ b/packages/google-cloud-dns/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] @@ -118,19 +119,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 4d9415cb22e7..418cbb368f75 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] @@ -120,19 +121,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-cloud-runtimeconfig/noxfile.py b/packages/google-cloud-runtimeconfig/noxfile.py index 9ae3a68fedc3..81d69b01d38c 100644 --- a/packages/google-cloud-runtimeconfig/noxfile.py +++ b/packages/google-cloud-runtimeconfig/noxfile.py @@ -22,13 +22,14 @@ import pathlib import re import shutil -from typing import Dict, List import warnings +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] @@ -118,19 +119,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/google-cloud-testutils/noxfile.py b/packages/google-cloud-testutils/noxfile.py index 65f9c1ee1967..3cbc6d7cc607 100644 --- a/packages/google-cloud-testutils/noxfile.py +++ b/packages/google-cloud-testutils/noxfile.py @@ -26,6 +26,7 @@ # 'update_lower_bounds' is excluded nox.options.sessions = [ "check_lower_bounds", + "format", ] @@ -35,6 +36,7 @@ ALL_PYTHON = ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] DEFAULT_PYTHON_VERSION = "3.14" BLACK_VERSION = "black==23.7.0" +RUFF_VERSION = "ruff==0.14.14" BLACK_PATHS = ["test_utils", "setup.py"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -68,6 +70,33 @@ def blacken(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run ruff to sort imports and format code. + """ + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports + session.run( + "ruff", "check", + "--select", "I", + "--fix", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + # 3. Run Ruff to format code + session.run( + "ruff", "format", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", + *BLACK_PATHS, + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" diff --git a/packages/google-crc32c/noxfile.py b/packages/google-crc32c/noxfile.py index b58397c9e0b4..c1b449abbbf4 100644 --- a/packages/google-crc32c/noxfile.py +++ b/packages/google-crc32c/noxfile.py @@ -31,6 +31,7 @@ FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["src", "tests", "noxfile.py", "setup.py"] @@ -116,19 +117,29 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 1cf600b1b1b1..ab2c85110cbc 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -14,14 +14,15 @@ from __future__ import absolute_import -import shutil -import nox import os import pathlib +import shutil +import nox BLACK_VERSION = "black[jupyter]==23.7.0" ISORT_VERSION = "isort==5.11.0" +RUFF_VERSION = "ruff==0.14.14" LINT_PATHS = ["docs", "proto", "tests", "noxfile.py", "setup.py"] @@ -322,3 +323,33 @@ def lint(session): ) session.run("flake8", "proto", "tests") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run ruff to sort imports and format code. + """ + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) + + # 2. Run Ruff to fix imports + session.run( + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", + *LINT_PATHS, + ) + + # 3. Run Ruff to format code + session.run( + "ruff", + "format", + f"--target-version=py{PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", + *LINT_PATHS, + ) diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 615ac0c30f5c..7a8ba6cdfc2a 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -18,19 +18,20 @@ from __future__ import absolute_import -from functools import wraps import os import pathlib import re import shutil import time -from typing import Dict, List import warnings +from functools import wraps +from typing import Dict, List import nox FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" +RUFF_VERSION = "ruff==0.14.14" ISORT_VERSION = "isort==5.11.0" LINT_PATHS = [ "third_party", @@ -200,22 +201,29 @@ def blacken(session): @_calculate_duration def format(session): """ - Run isort to sort imports. Then run black - to format code to uniform standard. + Run ruff to sort imports and format code. """ - session.install(BLACK_VERSION, ISORT_VERSION) + # 1. Install ruff (skipped automatically if you run with --no-venv) + session.install(RUFF_VERSION) - session.run("python", "-m", "pip", "freeze") - - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + # 2. Run Ruff to fix imports session.run( - "isort", - "--fss", + "ruff", + "check", + "--select", + "I", + "--fix", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) + + # 3. Run Ruff to format code session.run( - "black", + "ruff", + "format", + f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}", + "--line-length=88", *LINT_PATHS, ) diff --git a/packages/sqlalchemy-spanner/create_test_database.py b/packages/sqlalchemy-spanner/create_test_database.py index a72e44aa4ed3..d311ecb3e70c 100644 --- a/packages/sqlalchemy-spanner/create_test_database.py +++ b/packages/sqlalchemy-spanner/create_test_database.py @@ -68,28 +68,74 @@ def delete_stale_test_instances(): def delete_stale_test_databases(): - """Delete test databases that are older than four hours.""" + """Delete stale or excessive test databases. + + Deletes stale test databases that are older than 4 hours and + ensures we don't hit the 100 database limit per spanner instance by + deleting the oldest databases if we are near the limit. + """ cutoff = (int(time.time()) - 4 * 60 * 60) * 1000 instance = CLIENT.instance("sqlalchemy-dialect-test") if not instance.exists(): return - database_pbs = instance.list_databases() + + # Convert iterator to list to allow multiple passes and length check + database_pbs = list(instance.list_databases()) + + # First pass: Delete stale databases + remaining_dbs = [] for database_pb in database_pbs: database = Database.from_pb(database_pb, instance) # The emulator does not return a create_time for databases. if database.create_time is None: + remaining_dbs.append(database_pb) continue + create_time = datetime_helpers.to_milliseconds(database_pb.create_time) if create_time > cutoff: + remaining_dbs.append(database_pb) continue + try: database.drop() + print(f"Dropped stale database '{database.database_id}'") except ResourceExhausted: print( "Unable to drop stale database '{}'. May need manual delete.".format( database.database_id ) ) + remaining_dbs.append(database_pb) # Still there + + # Second pass: If we are still near the limit (e.g., 90+ databases), + # delete the oldest ones regardless of age to free up slots. + # Spanner instances have a hard limit of 100 databases. + LIMIT = 90 + if len(remaining_dbs) >= LIMIT: + print(f"Database count ({len(remaining_dbs)}) is near limit. Cleaning up oldest databases.") + + # Sort by creation time + dbs_with_time = [] + for db_pb in remaining_dbs: + if db_pb.create_time: + dbs_with_time.append((db_pb.create_time, db_pb.name)) + + dbs_with_time.sort() # Sorts by time ascending (oldest first) + + # Delete enough to get below the limit + to_delete = len(remaining_dbs) - (LIMIT - 10) # Aim for 80 + deleted_count = 0 + + for create_time, full_name in dbs_with_time: + if deleted_count >= to_delete: + break + db_id = full_name.split('/')[-1] + try: + instance.database(db_id).drop() + print(f"Dropped oldest database '{db_id}' to prevent resource exhaustion.") + deleted_count += 1 + except Exception as e: + print(f"Failed to drop database '{db_id}': {e}") def create_test_instance():