From bfe9f2a010821618176d71c4efee83d0a37ce13b Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 15 Oct 2025 21:54:48 +0000 Subject: [PATCH 1/6] feat: add support to release handwritten clients --- .generator/cli.py | 45 +++++++++++++++++++++++++++--------------- .generator/test_cli.py | 2 ++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index a9b766f67e84..c3efda25bb50 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -1109,7 +1109,9 @@ def _process_version_file(content, version, version_path) -> str: Returns: A string with the modified content. """ - if version_path.name.endswith("gapic_version.py"): + if version_path.name.endswith("gapic_version.py") or version_path.name.endswith( + "version.py" + ): pattern = r"(__version__\s*=\s*[\"'])([^\"']+)([\"'].*)" else: pattern = r"(version\s*=\s*[\"'])([^\"']+)([\"'].*)" @@ -1125,7 +1127,7 @@ def _process_version_file(content, version, version_path) -> str: def _update_version_for_library( repo: str, output: str, path_to_library: str, version: str ): - """Updates the version string in `**/gapic_version.py`, `setup.py`, + """Updates the version string in `**/gapic_version.py`, `**/version.py`, `setup.py`, `pyproject.toml` and `samples/**/snippet_metadata.json` for a given library, if applicable. @@ -1139,12 +1141,15 @@ def _update_version_for_library( version(str): The new version of the library Raises: `ValueError` if a version string could not be located in `**/gapic_version.py` - within the given library. + or `**/version.py` within the given library. """ - # Find and update gapic_version.py files - version_files = list(Path(f"{repo}/{path_to_library}").rglob("**/gapic_version.py")) - if len(version_files) == 0: + # Find and update version.py or gapic_version.py files + search_base = Path(f"{repo}/{path_to_library}") + version_files = list(search_base.rglob("**/gapic_version.py")) + version_files.extend(list(search_base.glob("google/**/version.py"))) + + if not version_files: # Fallback to `pyproject.toml`` or `setup.py``. Proto-only libraries have # version information in `setup.py` or `pyproject.toml` instead of `gapic_version.py`. pyproject_toml = Path(f"{repo}/{path_to_library}/pyproject.toml") @@ -1160,7 +1165,7 @@ def _update_version_for_library( # Find and update snippet_metadata.json files snippet_metadata_files = Path(f"{repo}/{path_to_library}").rglob( - "samples/**/*.json" + "samples/**/*snippet*.json" ) for metadata_file in snippet_metadata_files: output_path = f"{output}/{metadata_file.relative_to(repo)}" @@ -1300,6 +1305,7 @@ def _update_changelog_for_library( version: str, previous_version: str, library_id: str, + relative_path: str, ): """Prepends a new release entry with multiple, grouped changes, to a changelog. @@ -1316,8 +1322,6 @@ def _update_changelog_for_library( library_id(str): The id of the library where the changelog should be updated. """ - - relative_path = f"packages/{library_id}/CHANGELOG.md" changelog_src = f"{repo}/{relative_path}" changelog_dest = f"{output}/{relative_path}" updated_content = _process_changelog( @@ -1357,19 +1361,21 @@ def handle_release_init( `release-init-request.json` file in the given librarian directory cannot be read. """ - try: + is_generated = Path(f"{repo}/packages").exists() + # Read a release-init-request.json file request_data = _read_json_file(f"{librarian}/{RELEASE_INIT_REQUEST_FILE}") libraries_to_prep_for_release = _get_libraries_to_prepare_for_release( request_data ) - _update_global_changelog( - f"{repo}/CHANGELOG.md", - f"{output}/CHANGELOG.md", - libraries_to_prep_for_release, - ) + if is_generated: + _update_global_changelog( + f"{repo}/CHANGELOG.md", + f"{output}/CHANGELOG.md", + libraries_to_prep_for_release, + ) # Prepare the release for each library by updating the # library specific version files and library specific changelog. @@ -1377,7 +1383,6 @@ def handle_release_init( version = library_release_data["version"] library_id = library_release_data["id"] library_changes = library_release_data["changes"] - path_to_library = f"packages/{library_id}" # Get previous version from state.yaml previous_version = _get_previous_version(library_id, librarian) @@ -1387,6 +1392,13 @@ def handle_release_init( f"{library_id} version: {previous_version}\n" ) + if is_generated: + path_to_library = f"packages/{library_id}" + changelog_relative_path = f"packages/{library_id}/CHANGELOG.md" + else: + path_to_library = "." + changelog_relative_path = "CHANGELOG.md" + _update_version_for_library(repo, output, path_to_library, version) _update_changelog_for_library( repo, @@ -1395,6 +1407,7 @@ def handle_release_init( version, previous_version, library_id, + relative_path=changelog_relative_path, ) except Exception as e: diff --git a/.generator/test_cli.py b/.generator/test_cli.py index a401229a6723..00869f4139a2 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -1108,6 +1108,7 @@ def test_update_changelog_for_library_success(mocker): "1.2.3", "1.2.2", "google-cloud-language", + "CHANGELOG.md", ) @@ -1157,6 +1158,7 @@ def test_update_changelog_for_library_failure(mocker): "1.2.3", "1.2.2", "google-cloud-language", + "CHANGELOG.md", ) From 7d9ed4189775107859193324004a9439f16aa1c4 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 15 Oct 2025 22:11:29 +0000 Subject: [PATCH 2/6] add missing coverage --- .generator/parse_googleapis_content.py | 1 + .generator/test_cli.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.generator/parse_googleapis_content.py b/.generator/parse_googleapis_content.py index a045dfda3a65..0afa485e58de 100644 --- a/.generator/parse_googleapis_content.py +++ b/.generator/parse_googleapis_content.py @@ -103,6 +103,7 @@ "glob", ) + def parse_content(content: str) -> dict: """Parses content from BUILD.bazel and returns a dictionary containing bazel rules and arguments. diff --git a/.generator/test_cli.py b/.generator/test_cli.py index 00869f4139a2..8b43cd875128 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -856,6 +856,24 @@ def test_handle_release_init_success(mocker, mock_release_init_request_file): handle_release_init() +def test_handle_release_init_is_generated_success( + mocker, mock_release_init_request_file +): + """ + Tests that `handle_release_init` calls `_update_global_changelog` when the + `packages` directory exists. + """ + mocker.patch("pathlib.Path.exists", return_value=True) + mock_update_global_changelog = mocker.patch("cli._update_global_changelog") + mocker.patch("cli._update_version_for_library") + mocker.patch("cli._get_previous_version", return_value="1.2.2") + mocker.patch("cli._update_changelog_for_library") + + handle_release_init() + + mock_update_global_changelog.assert_called_once() + + def test_handle_release_init_fail_value_error_file(): """ Tests that handle_release_init fails to read `librarian/release-init-request.json`. From bcc4f13d7844c1c34e82785ac80591cf8476c9d5 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 15 Oct 2025 22:18:15 +0000 Subject: [PATCH 3/6] refactor code --- .generator/cli.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.generator/cli.py b/.generator/cli.py index c3efda25bb50..bb8e3d801d0c 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -1334,6 +1334,19 @@ def _update_changelog_for_library( _write_text_file(changelog_dest, updated_content) +def _is_generated_library(repo: str) -> bool: + """Determines if a library is generated or handwritten. + + Args: + repo(str): This directory will contain all directories that make up a + library, the .librarian folder, and any global file declared in + the config.yaml. + + Returns: True if the library is generated, False otherwise. + """ + return Path(f"{repo}/packages").exists() + + def handle_release_init( librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR, output: str = OUTPUT_DIR ): @@ -1362,7 +1375,7 @@ def handle_release_init( librarian directory cannot be read. """ try: - is_generated = Path(f"{repo}/packages").exists() + is_generated = _is_generated_library(repo) # Read a release-init-request.json file request_data = _read_json_file(f"{librarian}/{RELEASE_INIT_REQUEST_FILE}") From d252f4d405d8a416ed82144097d9980065fcaea7 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 15 Oct 2025 23:31:30 +0000 Subject: [PATCH 4/6] address feedback --- .generator/cli.py | 18 +++++++++++++++++- .generator/test_cli.py | 21 ++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index bb8e3d801d0c..f341b46cccbf 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -1147,7 +1147,23 @@ def _update_version_for_library( # Find and update version.py or gapic_version.py files search_base = Path(f"{repo}/{path_to_library}") version_files = list(search_base.rglob("**/gapic_version.py")) - version_files.extend(list(search_base.glob("google/**/version.py"))) + excluded_dirs = { + ".nox", + ".venv", + "venv", + "site-packages", + ".git", + "build", + "dist", + "__pycache__", + } + version_files.extend( + [ + p + for p in search_base.rglob("**/version.py") + if not any(part in excluded_dirs for part in p.parts) + ] + ) if not version_files: # Fallback to `pyproject.toml`` or `setup.py``. Proto-only libraries have diff --git a/.generator/test_cli.py b/.generator/test_cli.py index 8b43cd875128..64a4bde106c2 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -988,9 +988,12 @@ def test_update_global_changelog(mocker, mock_release_init_request_file): def test_update_version_for_library_success_gapic(mocker): m = mock_open() - mock_rglob = mocker.patch( - "pathlib.Path.rglob", return_value=[pathlib.Path("repo/gapic_version.py")] - ) + mock_rglob = mocker.patch("pathlib.Path.rglob") + mock_rglob.side_effect = [ + [pathlib.Path("repo/gapic_version.py")], # 1st call (gapic_version.py) + [], # 2nd call (version.py) + [pathlib.Path("repo/samples/snippet_metadata.json")] # 3rd call (snippets) + ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = '__version__ = "1.2.2"' mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} @@ -1020,7 +1023,11 @@ def test_update_version_for_library_success_proto_only_setup_py(mocker): m = mock_open() mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [[], [pathlib.Path("repo/setup.py")]] + mock_rglob.side_effect = [ + [], + [pathlib.Path("repo/setup.py")], + [pathlib.Path("repo/samples/snippet_metadata.json")] + ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = 'version = "1.2.2"' mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} @@ -1051,7 +1058,11 @@ def test_update_version_for_library_success_proto_only_py_project_toml(mocker): mock_path_exists = mocker.patch("pathlib.Path.exists") mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [[], [pathlib.Path("repo/pyproject.toml")]] + mock_rglob.side_effect = [ + [], + [pathlib.Path("repo/pyproject.toml")], + [pathlib.Path("repo/samples/snippet_metadata.json")] + ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = 'version = "1.2.2"' mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} From a59600dd4918ba86f8555d99f2280a437c726839 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Mon, 20 Oct 2025 15:52:12 +0000 Subject: [PATCH 5/6] add code coverage --- .generator/test_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.generator/test_cli.py b/.generator/test_cli.py index 64a4bde106c2..a5e136ade5b8 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -1053,14 +1053,14 @@ def test_update_version_for_library_success_proto_only_setup_py(mocker): ) -def test_update_version_for_library_success_proto_only_py_project_toml(mocker): +def test_update_version_for_library_success_proto_only_pyproject_toml(mocker): m = mock_open() - mock_path_exists = mocker.patch("pathlib.Path.exists") + mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) mock_rglob = mocker.patch("pathlib.Path.rglob") mock_rglob.side_effect = [ - [], - [pathlib.Path("repo/pyproject.toml")], + [], # gapic_version.py + [], # version.py [pathlib.Path("repo/samples/snippet_metadata.json")] ] mock_shutil_copy = mocker.patch("shutil.copy") From 028447035c6d7793749c5c6decbcfcf54c536618 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Mon, 20 Oct 2025 16:08:04 +0000 Subject: [PATCH 6/6] address feedback --- .generator/cli.py | 14 ++++++++------ .generator/test_cli.py | 31 +++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index f341b46cccbf..e505debfa2e5 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -228,7 +228,7 @@ def handle_configure( source: str = SOURCE_DIR, repo: str = REPO_DIR, input: str = INPUT_DIR, - output: str = OUTPUT_DIR + output: str = OUTPUT_DIR, ): """Onboards a new library by completing its configuration. @@ -259,7 +259,7 @@ def handle_configure( # configure-request.json contains the library definitions. request_data = _read_json_file(f"{librarian}/{CONFIGURE_REQUEST_FILE}") new_library_config = _get_new_library_config(request_data) - + _update_global_changelog( f"{repo}/CHANGELOG.md", f"{output}/CHANGELOG.md", @@ -1350,7 +1350,7 @@ def _update_changelog_for_library( _write_text_file(changelog_dest, updated_content) -def _is_generated_library(repo: str) -> bool: +def _is_mono_repo(repo: str) -> bool: """Determines if a library is generated or handwritten. Args: @@ -1391,7 +1391,7 @@ def handle_release_init( librarian directory cannot be read. """ try: - is_generated = _is_generated_library(repo) + is_mono_repo = _is_mono_repo(repo) # Read a release-init-request.json file request_data = _read_json_file(f"{librarian}/{RELEASE_INIT_REQUEST_FILE}") @@ -1399,7 +1399,9 @@ def handle_release_init( request_data ) - if is_generated: + if is_mono_repo: + + # only a mono repo has a global changelog _update_global_changelog( f"{repo}/CHANGELOG.md", f"{output}/CHANGELOG.md", @@ -1421,7 +1423,7 @@ def handle_release_init( f"{library_id} version: {previous_version}\n" ) - if is_generated: + if is_mono_repo: path_to_library = f"packages/{library_id}" changelog_relative_path = f"packages/{library_id}/CHANGELOG.md" else: diff --git a/.generator/test_cli.py b/.generator/test_cli.py index a5e136ade5b8..9e1ad578f688 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -990,9 +990,9 @@ def test_update_version_for_library_success_gapic(mocker): mock_rglob = mocker.patch("pathlib.Path.rglob") mock_rglob.side_effect = [ - [pathlib.Path("repo/gapic_version.py")], # 1st call (gapic_version.py) - [], # 2nd call (version.py) - [pathlib.Path("repo/samples/snippet_metadata.json")] # 3rd call (snippets) + [pathlib.Path("repo/gapic_version.py")], # 1st call (gapic_version.py) + [], # 2nd call (version.py) + [pathlib.Path("repo/samples/snippet_metadata.json")], # 3rd call (snippets) ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = '__version__ = "1.2.2"' @@ -1026,7 +1026,7 @@ def test_update_version_for_library_success_proto_only_setup_py(mocker): mock_rglob.side_effect = [ [], [pathlib.Path("repo/setup.py")], - [pathlib.Path("repo/samples/snippet_metadata.json")] + [pathlib.Path("repo/samples/snippet_metadata.json")], ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = 'version = "1.2.2"' @@ -1061,7 +1061,7 @@ def test_update_version_for_library_success_proto_only_pyproject_toml(mocker): mock_rglob.side_effect = [ [], # gapic_version.py [], # version.py - [pathlib.Path("repo/samples/snippet_metadata.json")] + [pathlib.Path("repo/samples/snippet_metadata.json")], ] mock_shutil_copy = mocker.patch("shutil.copy") mock_content = 'version = "1.2.2"' @@ -1555,7 +1555,9 @@ def test_copy_readme_to_docs(mocker): mock_os_islink = mocker.patch("os.path.islink", return_value=False) mock_os_remove = mocker.patch("os.remove") mock_os_lexists = mocker.patch("os.path.lexists", return_value=True) - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data="dummy content")) + mock_open = mocker.patch( + "builtins.open", mocker.mock_open(read_data="dummy content") + ) output = "output" library_id = "google-cloud-language" @@ -1582,10 +1584,15 @@ def test_copy_readme_to_docs_handles_symlink(mocker): mock_os_islink = mocker.patch("os.path.islink") mock_os_remove = mocker.patch("os.remove") mock_os_lexists = mocker.patch("os.path.lexists", return_value=True) - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data="dummy content")) + mock_open = mocker.patch( + "builtins.open", mocker.mock_open(read_data="dummy content") + ) # Simulate docs_path being a symlink - mock_os_islink.side_effect = [False, True] # First call for destination_path, second for docs_path + mock_os_islink.side_effect = [ + False, + True, + ] # First call for destination_path, second for docs_path output = "output" library_id = "google-cloud-language" @@ -1612,7 +1619,9 @@ def test_copy_readme_to_docs_destination_path_is_symlink(mocker): mock_os_islink = mocker.patch("os.path.islink", return_value=True) mock_os_remove = mocker.patch("os.remove") mock_os_lexists = mocker.patch("os.path.lexists", return_value=True) - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data="dummy content")) + mock_open = mocker.patch( + "builtins.open", mocker.mock_open(read_data="dummy content") + ) output = "output" library_id = "google-cloud-language" @@ -1629,7 +1638,9 @@ def test_copy_readme_to_docs_source_not_exists(mocker): mock_os_islink = mocker.patch("os.path.islink") mock_os_remove = mocker.patch("os.remove") mock_os_lexists = mocker.patch("os.path.lexists", return_value=False) - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data="dummy content")) + mock_open = mocker.patch( + "builtins.open", mocker.mock_open(read_data="dummy content") + ) output = "output" library_id = "google-cloud-language"