Merge branch 'main' into migrate-off-bazel

Author: parthea

.generator/Dockerfile

@@ -123,6 +123,10 @@ COPY .generator/requirements.in .
 RUN python3.9 -m pip install -r requirements.in
 RUN python3.9 -m pip install /synthtool
 
+# Install build which is used to get the metadata of package config files.
+COPY .generator/requirements.in .
+RUN python3.9 -m pip install -r requirements.in
+
 # Copy the CLI script into the container.
 COPY .generator/cli.py .
 RUN chmod a+rx ./cli.py

.generator/cli.py

@@ -23,12 +23,13 @@
 import subprocess
 import sys
 import tempfile
+import yaml
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, List
-
+import build.util
 import parse_googleapis_content
-import yaml
+
 
 try:
     import synthtool
@@ -334,7 +335,6 @@ def _run_nox_sessions(library_id: str, repo: str):
         "unit-3.9",
         "unit-3.13",
         "docs",
-        "system-3.13",
         "lint",
         "lint_setup_py",
         "mypy-3.13",
@@ -450,12 +450,54 @@ def _verify_library_namespace(library_id: str, repo: str):
             )
 
 
+def _get_library_dist_name(library_id: str, repo: str) -> str:
+    """
+    Gets the package name by programmatically building the metadata.
+
+    Args:
+        library_id: id of the library.
+        repo: This directory will contain all directories that make up a
+            library, the .librarian folder, and any global file declared in
+            the config.yaml.
+
+    Returns:
+        str: The library name string if found, otherwise None.
+    """
+    library_path = f"{repo}/packages/{library_id}"
+    metadata = build.util.project_wheel_metadata(library_path)
+    return metadata.get("name")
+
+
+def _verify_library_dist_name(library_id: str, repo: str):
+    """Verifies the library distribution name against its config files.
+
+    This function ensures that:
+    1. At least one of `setup.py` or `pyproject.toml` exists and is valid.
+    2. Any existing config file's 'name' property matches the `library_id`.
+
+    Args:
+        library_id: id of the library.
+        repo: This directory will contain all directories that make up a
+            library, the .librarian folder, and any global file declared in
+            the config.yaml.
+
+    Raises:
+        ValueError: If a name in an existing config file does not match the `library_id`.
+    """
+    dist_name = _get_library_dist_name(library_id, repo)
+    if dist_name != library_id:
+        raise ValueError(
+            f"The distribution name `{dist_name}` does not match the folder `{library_id}`."
+        )
+
+
 def handle_build(librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR):
     """The main coordinator for validating client library generation."""
     try:
         request_data = _read_json_file(f"{librarian}/{BUILD_REQUEST_FILE}")
         library_id = _get_library_id(request_data)
         _verify_library_namespace(library_id, repo)
+        _verify_library_dist_name(library_id, repo)
         _run_nox_sessions(library_id, repo)
     except Exception as e:
         raise ValueError("Build failed.") from e
@@ -750,6 +792,12 @@ def handle_release_init(
             the config.yaml.
         output(str): Path to the directory in the container where modified
             code should be placed.
+
+    Raises:
+        ValueError: if the version in `release-init-request.json` is
+            the same as the version in state.yaml or if the
+            `release-init-request.json` file in the given
+            librarian directory cannot be read.
     """
 
     try:
@@ -772,20 +820,25 @@ def handle_release_init(
             library_id = library_release_data["id"]
             library_changes = library_release_data["changes"]
             path_to_library = f"packages/{library_id}"
-            _update_version_for_library(repo, output, path_to_library, version)
 
             # Get previous version from state.yaml
             previous_version = _get_previous_version(library_id, librarian)
-            if previous_version != version:
-                _update_changelog_for_library(
-                    repo,
-                    output,
-                    library_changes,
-                    version,
-                    previous_version,
-                    library_id,
+            if previous_version == version:
+                raise ValueError(
+                    f"The version in {RELEASE_INIT_REQUEST_FILE} is the same as the version in {STATE_YAML_FILE}\n"
+                    f"{library_id} version: {previous_version}\n"
                 )
 
+            _update_version_for_library(repo, output, path_to_library, version)
+            _update_changelog_for_library(
+                repo,
+                output,
+                library_changes,
+                version,
+                previous_version,
+                library_id,
+            )
+
     except Exception as e:
         raise ValueError(f"Release init failed: {e}") from e
 

.generator/requirements-test.in

@@ -17,3 +17,4 @@ pytest-cov
 pytest-mock
 gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655
 starlark-pyo3>=2025.1
+build

.generator/requirements.in

@@ -2,3 +2,4 @@ click
 gapic-generator
 nox
 starlark-pyo3>=2025.1
+build

.generator/test_cli.py

@@ -36,6 +36,7 @@
     _clean_up_files_after_post_processing,
     _copy_files_needed_for_post_processing,
     _create_main_version_header,
+    _get_library_dist_name,
     _determine_library_namespace,
     _get_library_id,
     _get_libraries_to_prepare_for_release,
@@ -50,6 +51,7 @@
     _update_changelog_for_library,
     _update_global_changelog,
     _update_version_for_library,
+    _verify_library_dist_name,
     _verify_library_namespace,
     _write_json_file,
     _write_text_file,
@@ -364,7 +366,6 @@ def test_run_nox_sessions_success(mocker, mock_generate_request_data_for_nox):
         "unit-3.9",
         "unit-3.13",
         "docs",
-        "system-3.13",
         "lint",
         "lint_setup_py",
         "mypy-3.13",
@@ -377,7 +378,6 @@ def test_run_nox_sessions_success(mocker, mock_generate_request_data_for_nox):
             mocker.call("unit-3.9", "mock-library", "repo"),
             mocker.call("unit-3.13", "mock-library", "repo"),
             mocker.call("docs", "mock-library", "repo"),
-            mocker.call("system-3.13", "mock-library", "repo"),
             mocker.call("lint", "mock-library", "repo"),
             mocker.call("lint_setup_py", "mock-library", "repo"),
             mocker.call("mypy-3.13", "mock-library", "repo"),
@@ -430,7 +430,8 @@ def test_handle_build_success(caplog, mocker, mock_build_request_file):
     caplog.set_level(logging.INFO)
 
     mocker.patch("cli._run_nox_sessions")
-    mocker.patch("cli._verify_library_namespace", return_value=True)
+    mocker.patch("cli._verify_library_namespace")
+    mocker.patch("cli._verify_library_dist_name")
     handle_build()
 
     assert "'build' command executed." in caplog.text
@@ -507,14 +508,44 @@ def test_handle_release_init_success(mocker, mock_release_init_request_file):
     handle_release_init()
 
 
-def test_handle_release_init_fail():
+def test_handle_release_init_fail_value_error_file():
     """
     Tests that handle_release_init fails to read `librarian/release-init-request.json`.
     """
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match="No such file or directory"):
         handle_release_init()
 
 
+def test_handle_release_init_fail_value_error_version(mocker):
+    m = mock_open()
+
+    mock_release_init_request_content = {
+        "libraries": [
+            {
+                "id": "google-cloud-language",
+                "apis": [{"path": "google/cloud/language/v1"}],
+                "release_triggered": True,
+                "version": "1.2.2",
+                "changes": [],
+            },
+        ]
+    }
+    with unittest.mock.patch("cli.open", m):
+        mocker.patch(
+            "cli._get_libraries_to_prepare_for_release",
+            return_value=mock_release_init_request_content["libraries"],
+        )
+        mocker.patch("cli._get_previous_version", return_value="1.2.2")
+        mocker.patch("cli._process_changelog", return_value=None)
+        mocker.patch(
+            "cli._read_json_file", return_value=mock_release_init_request_content
+        )
+        with pytest.raises(
+            ValueError, match="is the same as the version in state.yaml"
+        ):
+            handle_release_init()
+
+
 def test_read_valid_text_file(mocker):
     """Tests reading a valid text file."""
     mock_content = "some text"
@@ -797,6 +828,26 @@ def test_determine_library_namespace_fails_not_subpath():
         _determine_library_namespace(gapic_parent_path, pkg_root_path)
 
 
+def test_get_library_dist_name_success(mocker):
+    mock_metadata = {"name": "my-lib", "version": "1.0.0"}
+    mocker.patch("build.util.project_wheel_metadata", return_value=mock_metadata)
+    assert _get_library_dist_name("my-lib", "repo") == "my-lib"
+
+
+def test_verify_library_dist_name_setup_success(mocker):
+    """Tests success when a library distribution name in setup.py is valid."""
+    mock_setup_file = mocker.patch("cli._get_library_dist_name", return_value="my-lib")
+    _verify_library_dist_name("my-lib", "repo")
+    mock_setup_file.assert_called_once_with("my-lib", "repo")
+
+
+def test_verify_library_dist_name_fail(mocker):
+    """Tests failure when a library-id does not match the libary distribution name."""
+    mocker.patch("cli._get_library_dist_name", return_value="invalid-lib")
+    with pytest.raises(ValueError):
+        _verify_library_dist_name("my-lib", "repo")
+
+
 def test_verify_library_namespace_success_valid(mocker, mock_path_class):
     """Tests success when a single valid namespace is found."""
     # 1. Get the mock instance from the mock class's return_value

.github/CODEOWNERS

@@ -1,7 +1,11 @@
 # Code owners file.
 # This file controls who is tagged for review for any given pull request.
 
-*                         @googleapis/python-core-client-libraries
+# Default owner for all directories not owned by others
+*                                             @googleapis/python-core-client-libraries @googleapis/cloud-sdk-librarian-team
 
-/.generator/ @googleapis/cloud-sdk-librarian-team
-/.librarian/ @googleapis/cloud-sdk-librarian-team
+# Packages onboarded to librarian require review from both groups
+/packages/google-cloud-dlp/                   @googleapis/python-core-client-libraries
+/packages/google-cloud-dlp/                   @googleapis/cloud-sdk-librarian-team
+
+/packages/google-cloud-bigquery-storage/      @googleapis/api-bigquery

.github/workflows/configure_release_please.yml

@@ -23,6 +23,9 @@ on:
     branches:
       - main
 
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:      
+
 permissions:
   contents: read
 

.kokoro/system.sh

@@ -38,21 +38,35 @@ pwd
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 
+# This is needed in order for `git diff` to succeed
+git config --global --add safe.directory $(realpath .)
+
 # Run system tests for each package with directory packages/*/tests/system
 for dir in `find 'packages' -type d -wholename 'packages/*/tests/system'`; do
   # Get the path to the package by removing the suffix /tests/system
   package=$(echo $dir | cut -f -2 -d '/')
-  echo "Running system tests for ${package}"
-  pushd ${package}
-  # Temporarily allow failure.
+
+  files_to_check=${package}/CHANGELOG.md
+
+  echo "checking changes with 'git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check}'"
   set +e
-  ${system_test_script}
-  ret=$?
+  package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check} | wc -l)
   set -e
-  if [ ${ret} -ne 0 ]; then
-      RETVAL=${ret}
+  if [[ "${package_modified}" -eq 0 ]]; then
+      echo "no change detected in ${files_to_check}, skipping"
+  else
+      echo "change detected in ${files_to_check}"
+      echo "Running system tests for ${package}"
+      pushd ${package}
+      # Temporarily allow failure.
+      set +e
+      ${system_test_script}
+      ret=$?
+      set -e
+      if [ ${ret} -ne 0 ]; then
+          RETVAL=${ret}
+      fi
+      popd
   fi
-  popd
 done
-
 exit ${RETVAL}

.librarian/generator-input/client-post-processing/add-missing-dependencies-to-setup-py-constraints.yaml

@@ -161,3 +161,43 @@ replacements:
           "google-cloud-org-policy >= 1.0.0, <2.0.0",
           "proto-plus >= 1.22.3, <2.0.0",
     count: 1
+  - paths: [
+      packages/google-shopping-merchant-notifications/setup.py
+    ]
+    before: |
+      dependencies = \[
+          "google-api-core\[grpc\] >= 1.34.1, <3.0.0,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
+          # Exclude incompatible versions of `google-auth`
+          # See https://github.com/googleapis/google-cloud-python/issues/12364
+          "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",
+          "proto-plus >= 1.22.3, <2.0.0",
+          "proto-plus >= 1.25.0, <2.0.0; python_version >= '3.13'",
+          "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+      \]
+    after:  |
+      dependencies = [
+          "google-api-core[grpc] >= 1.34.1, <3.0.0,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
+          # Exclude incompatible versions of `google-auth`
+          # See https://github.com/googleapis/google-cloud-python/issues/12364
+          "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",
+          "proto-plus >= 1.22.3, <2.0.0",
+          "proto-plus >= 1.25.0, <2.0.0; python_version >= '3.13'",
+          "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+          "google-shopping-type >= 1.0.0, <2.0.0",
+      ]
+    count: 1
+  - paths: [
+      packages/google-shopping-merchant-notifications/testing/constraints-3.7.txt
+    ]
+    before: |
+      google-api-core==1.34.1
+      google-auth==2.14.1
+      proto-plus==1.22.3
+      protobuf==3.20.2
+    after:  |
+      google-api-core==1.34.1
+      google-auth==2.14.1
+      proto-plus==1.22.3
+      google-shopping-type==1.0.0
+      protobuf==3.20.2
+    count: 1