diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 4f118eca0..7e82d4f2a 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -19,7 +19,10 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.9', '3.10', "3.11", "3.12", "3.13"] + python: ["3.13"] # TODO: For alpha release purposes, microgenerator tests + # are limited to Python 3.13. By default this also limits + # GAPIC tests. Once the testing kinks are worked out we can + # explore a more comprehensive approach. steps: - name: Checkout uses: actions/checkout@v4 @@ -36,6 +39,7 @@ jobs: run: | python -m pip install --upgrade setuptools pip wheel python -m pip install nox + # These unit tests cover the GAPIC generated code. - name: Run unit tests env: COVERAGE_FILE: .coverage-${{ matrix.python }} @@ -44,6 +48,15 @@ jobs: PY_VERSION: ${{ matrix.python }} run: | nox -s unit-${{ matrix.python }} + # These unit tests cover the microgenerator module. + - name: Run microgenerator unit tests + env: + COVERAGE_FILE: .coverage-microgenerator-${{ matrix.python }} + BUILD_TYPE: presubmit + TEST_TYPE: unit + PY_VERSION: ${{ matrix.python }} + run: | + nox -f scripts/microgenerator/noxfile.py -s unit-${{ matrix.python }} - name: Upload coverage results uses: actions/upload-artifact@v4 with: @@ -84,4 +97,4 @@ jobs: run: | find .coverage-results -type f -name '*.zip' -exec unzip {} \; coverage combine .coverage-results/**/.coverage* - coverage report --show-missing --fail-under=100 \ No newline at end of file + coverage report --show-missing --fail-under=100 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/microgenerator/noxfile.py b/scripts/microgenerator/noxfile.py index be3870ba4..9f4cd7e2c 100644 --- a/scripts/microgenerator/noxfile.py +++ b/scripts/microgenerator/noxfile.py @@ -17,7 +17,6 @@ from functools import wraps import pathlib import os -import shutil import nox import time @@ -28,7 +27,6 @@ BLACK_PATHS = (".",) DEFAULT_PYTHON_VERSION = "3.9" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"] UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -56,14 +54,12 @@ def wrapper(*args, **kwargs): # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", - "system", "cover", "lint", "lint_setup_py", "blacken", "mypy", "pytype", - "docs", ] @@ -95,10 +91,8 @@ def default(session, install_extras=True): # If no, we use our own context object from magics.py. In order to exercise # that logic (and the associated tests) we avoid installing the [ipython] extra # which has a downstream effect of then avoiding installing bigquery_magics. - if install_extras and session.python == UNIT_TEST_PYTHON_VERSIONS[0]: - install_target = ".[bqstorage,pandas,ipywidgets,geopandas,matplotlib,tqdm,opentelemetry,bigquery_v2]" - elif install_extras: # run against all other UNIT_TEST_PYTHON_VERSIONS - install_target = ".[all]" + if install_extras: # run against all other UNIT_TEST_PYTHON_VERSIONS + install_target = "." else: install_target = "." session.install("-e", install_target, "-c", constraints_path) @@ -107,9 +101,6 @@ def default(session, install_extras=True): # directly. For example, pandas-gbq is recommended for pandas features, but # we want to test that we fallback to the previous behavior. For context, # see internal document go/pandas-gbq-and-bigframes-redundancy. - if session.python == UNIT_TEST_PYTHON_VERSIONS[0]: - session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y") - session.run("python", "-m", "pip", "freeze") # Run py.test against the unit tests. @@ -118,13 +109,12 @@ def default(session, install_extras=True): "-n=8", "--quiet", "-W default::PendingDeprecationWarning", - "--cov=google/cloud/bigquery", - "--cov=tests/unit", + "--cov=scripts.microgenerator", "--cov-append", "--cov-config=.coveragerc", "--cov-report=", "--cov-fail-under=0", - os.path.join("tests", "unit"), + "tests/unit", *session.posargs, ) @@ -142,7 +132,7 @@ def unit(session): def mypy(session): """Run type checks with mypy.""" - session.install("-e", ".[all]") + session.install("-e", ".") session.install(MYPY_VERSION) # Just install the dependencies' type info directly, since "mypy --install-types" @@ -166,77 +156,11 @@ def pytype(session): # https://github.com/googleapis/python-bigquery/issues/655 session.install("attrs==20.3.0") - session.install("-e", ".[all]") + session.install("-e", ".") session.install(PYTYPE_VERSION) session.run("python", "-m", "pip", "freeze") # See https://github.com/google/pytype/issues/464 - session.run("pytype", "-P", ".", "google/cloud/bigquery") - - -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -@_calculate_duration -def system(session): - """Run the system test suite.""" - - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - - # Sanity check: Only run system tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable.") - - # Use pre-release gRPC for system tests. - # Exclude version 1.49.0rc1 which has a known issue. - # See https://github.com/grpc/grpc/pull/30642 - session.install("--pre", "grpcio!=1.49.0rc1", "-c", constraints_path) - - # Install all test dependencies, then install local packages in place. - session.install( - "pytest", - "psutil", - "pytest-xdist", - "google-cloud-testutils", - "-c", - constraints_path, - ) - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": - # mTLS test requires pyopenssl and latest google-cloud-storage - session.install("google-cloud-storage", "pyopenssl") - else: - session.install("google-cloud-storage", "-c", constraints_path) - - # Data Catalog needed for the column ACL test with a real Policy Tag. - session.install("google-cloud-datacatalog", "-c", constraints_path) - - # Resource Manager needed for test with a real Resource Tag. - session.install("google-cloud-resource-manager", "-c", constraints_path) - - if session.python in ["3.11", "3.12"]: - extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]" - else: - extras = "[all]" - session.install("-e", f".{extras}", "-c", constraints_path) - - # Test with some broken "extras" in case the user didn't install the extra - # directly. For example, pandas-gbq is recommended for pandas features, but - # we want to test that we fallback to the previous behavior. For context, - # see internal document go/pandas-gbq-and-bigframes-redundancy. - if session.python == SYSTEM_TEST_PYTHON_VERSIONS[0]: - session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y") - - # print versions of all dependencies - session.run("python", "-m", "pip", "freeze") - - # Run py.test against the system tests. - session.run( - "py.test", - "-n=auto", - "--quiet", - "-W default::PendingDeprecationWarning", - os.path.join("tests", "system"), - *session.posargs, - ) + session.run("pytype", "-P", ".", "scripts") @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -266,10 +190,8 @@ def lint(session): session.install("flake8", BLACK_VERSION) session.install("-e", ".") session.run("python", "-m", "pip", "freeze") - session.run("flake8", os.path.join("google", "cloud", "bigquery")) + session.run("flake8", os.path.join("scripts")) session.run("flake8", "tests") - session.run("flake8", os.path.join("docs", "samples")) - session.run("flake8", os.path.join("docs", "snippets.py")) session.run("flake8", "benchmark") session.run("black", "--check", *BLACK_PATHS) @@ -294,89 +216,3 @@ def blacken(session): session.install(BLACK_VERSION) session.run("python", "-m", "pip", "freeze") session.run("black", *BLACK_PATHS) - - -@nox.session(python="3.10") -@_calculate_duration -def docs(session): - """Build the docs.""" - - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "sphinx==4.5.0", - "alabaster", - "recommonmark", - ) - session.install("google-cloud-storage") - session.install("-e", ".[all]") - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run("python", "-m", "pip", "freeze") - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python="3.10") -@_calculate_duration -def docfx(session): - """Build the docfx yaml files for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "gcp-sphinx-docfx-yaml", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run("python", "-m", "pip", "freeze") - session.run( - "sphinx-build", - "-T", # show full traceback on exception - "-N", # no colors - "-D", - ( - "extensions=sphinx.ext.autodoc," - "sphinx.ext.autosummary," - "docfx_yaml.extension," - "sphinx.ext.intersphinx," - "sphinx.ext.coverage," - "sphinx.ext.napoleon," - "sphinx.ext.todo," - "sphinx.ext.viewcode," - "recommonmark" - ), - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) diff --git a/scripts/microgenerator/setup.py b/scripts/microgenerator/setup.py new file mode 100644 index 000000000..0f52ef914 --- /dev/null +++ b/scripts/microgenerator/setup.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import setuptools # type: ignore + +name = "google-cloud-bigquery_v2-centralized-client" +description = "Google Cloud Bigquery V2 Centralized Client Microgenerator" +version = "0.1.0" + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ # TODO: break out development reqs from generation requirements + "PyYAML", + "Jinja2", +] +extras = {} + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + python_requires=">=3.12", + install_requires=dependencies, + extras_require=extras, + include_package_data=True, + zip_safe=False, +) diff --git a/scripts/microgenerator/tests/__init__.py b/scripts/microgenerator/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/microgenerator/tests/unit/__init__.py b/scripts/microgenerator/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/microgenerator/tests/unit/test_utils.py b/scripts/microgenerator/tests/unit/test_utils.py new file mode 100644 index 000000000..dbccba109 --- /dev/null +++ b/scripts/microgenerator/tests/unit/test_utils.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for utils.py.""" + +import pytest +from unittest.mock import Mock, patch + +from scripts.microgenerator import utils + + +def test_load_resource_success(): + """Tests that _load_resource returns the loader's result on success.""" + loader_func = Mock(return_value="Success") + result = utils._load_resource( + loader_func=loader_func, + path="/fake/path", + not_found_exc=FileNotFoundError, + parse_exc=ValueError, + resource_type_name="Fake Resource", + ) + assert result == "Success" + loader_func.assert_called_once() + + +def test_load_resource_not_found(capsys): + """Tests that _load_resource exits on not_found_exc.""" + loader_func = Mock(side_effect=FileNotFoundError) + with pytest.raises(SystemExit) as excinfo: + utils._load_resource( + loader_func=loader_func, + path="/fake/path", + not_found_exc=FileNotFoundError, + parse_exc=ValueError, + resource_type_name="Fake Resource", + ) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Error: Fake Resource '/fake/path' not found." in captured.err + + +def test_load_resource_parse_error(capsys): + """Tests that _load_resource exits on parse_exc.""" + error = ValueError("Invalid format") + loader_func = Mock(side_effect=error) + with pytest.raises(SystemExit) as excinfo: + utils._load_resource( + loader_func=loader_func, + path="/fake/path", + not_found_exc=FileNotFoundError, + parse_exc=ValueError, + resource_type_name="Fake Resource", + ) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert ( + "Error: Could not load fake resource from '/fake/path': Invalid format" + in captured.err + ) + + +def test_load_template_success(tmp_path): + """Tests that load_template successfully loads a Jinja2 template.""" + template_dir = tmp_path / "templates" + template_dir.mkdir() + template_path = template_dir / "test.j2" + template_path.write_text("Hello, {{ name }}!") + + template = utils.load_template(str(template_path)) + assert template.render(name="World") == "Hello, World!" + + +def test_load_template_not_found(capsys): + """Tests that load_template exits when the template is not found.""" + with pytest.raises(SystemExit) as excinfo: + utils.load_template("/non/existent/path.j2") + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Error: Template file '/non/existent/path.j2' not found." in captured.err + + +def test_load_template_parse_error(tmp_path, capsys): + """Tests that load_template exits on a template syntax error.""" + template_dir = tmp_path / "templates" + template_dir.mkdir() + template_path = template_dir / "test.j2" + template_path.write_text("Hello, {{ name }!") # Malformed template + + with pytest.raises(SystemExit) as excinfo: + utils.load_template(str(template_path)) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Could not load template file" in captured.err + + +def test_load_config_success(tmp_path): + """Tests that load_config successfully loads a YAML file.""" + config_dir = tmp_path / "configs" + config_dir.mkdir() + config_path = config_dir / "config.yaml" + config_path.write_text("key: value") + + config = utils.load_config(str(config_path)) + assert config == {"key": "value"} + + +def test_load_config_not_found(capsys): + """Tests that load_config exits when the config file is not + found.""" + with pytest.raises(SystemExit) as excinfo: + utils.load_config("/non/existent/path.yaml") + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert ( + "Error: Configuration file '/non/existent/path.yaml' not found." in captured.err + ) + + +def test_load_config_parse_error(tmp_path, capsys): + """Tests that load_config exits on a YAML syntax error.""" + config_dir = tmp_path / "configs" + config_dir.mkdir() + config_path = config_dir / "config.yaml" + config_path.write_text("key: value:") # Malformed YAML + + with pytest.raises(SystemExit) as excinfo: + utils.load_config(str(config_path)) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Could not load configuration file" in captured.err + + +def test_walk_codebase_finds_py_files(tmp_path): + """Tests that walk_codebase finds all .py files.""" + # Create a directory structure + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + (tmp_path / "a" / "c").mkdir() + + # Create some files + (tmp_path / "a" / "file1.py").touch() + (tmp_path / "a" / "c" / "file2.py").touch() + (tmp_path / "b" / "file3.py").touch() + (tmp_path / "a" / "file.txt").touch() # Should be ignored + (tmp_path / "b" / "script").touch() # Should be ignored + + result = sorted(list(utils.walk_codebase(str(tmp_path)))) + + expected = sorted( + [ + str(tmp_path / "a" / "file1.py"), + str(tmp_path / "a" / "c" / "file2.py"), + str(tmp_path / "b" / "file3.py"), + ] + ) + + assert result == expected + + +def test_walk_codebase_no_py_files(tmp_path): + """Tests that walk_codebase handles directories with no .py files.""" + (tmp_path / "a").mkdir() + (tmp_path / "a" / "file.txt").touch() + (tmp_path / "b").mkdir() + + result = list(utils.walk_codebase(str(tmp_path))) + assert result == [] + + +def test_walk_codebase_empty_directory(tmp_path): + """Tests that walk_codebase handles an empty directory.""" + result = list(utils.walk_codebase(str(tmp_path))) + assert result == [] + + +def test_walk_codebase_non_existent_path(): + """Tests that walk_codebase handles a non-existent path gracefully.""" + result = list(utils.walk_codebase("/non/existent/path")) + assert result == [] + + +def test_walk_codebase_with_file_path(tmp_path): + """Tests that walk_codebase handles being passed a file path.""" + file_path = tmp_path / "file.py" + file_path.touch() + # os.walk on a file yields nothing, so this should be empty. + result = list(utils.walk_codebase(str(file_path))) + assert result == [] + + +def test_write_code_to_file_creates_directory(tmp_path): + """Tests that write_code_to_file creates the directory if it doesn't exist.""" + output_dir = tmp_path / "new_dir" + output_path = output_dir / "test_file.py" + content = "import this" + + assert not output_dir.exists() + + utils.write_code_to_file(str(output_path), content) + + assert output_dir.is_dir() + assert output_path.read_text() == content + + +def test_write_code_to_file_existing_directory(tmp_path): + """Tests that write_code_to_file works when the directory already exists.""" + output_path = tmp_path / "test_file.py" + content = "import that" + + utils.write_code_to_file(str(output_path), content) + + assert output_path.read_text() == content + + +def test_write_code_to_file_no_directory(tmp_path, monkeypatch): + """Tests that write_code_to_file works for the current directory.""" + monkeypatch.chdir(tmp_path) + output_path = "test_file.py" + content = "import the_other" + + utils.write_code_to_file(output_path, content) + + assert (tmp_path / output_path).read_text() == content + + +@patch("os.path.isdir", return_value=False) +@patch("os.makedirs") +def test_write_code_to_file_dir_creation_fails( + mock_makedirs, mock_isdir, tmp_path, capsys +): + """Tests that write_code_to_file exits if directory creation fails.""" + output_path = tmp_path / "new_dir" / "test_file.py" + content = "import this" + + with pytest.raises(SystemExit) as excinfo: + utils.write_code_to_file(str(output_path), content) + + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Error: Output directory was not created." in captured.err + mock_makedirs.assert_called_once()