Merge branch 'main' into impl/release-note-generator

Author: meltsufin

.github/release-note-generation/split_release_note.py

@@ -69,6 +69,7 @@ def detect_modules(root_directory: Path):
         tree = ET.parse(module_pom_xml)
         root = tree.getroot()
         version = root.find('mvn:version', POM_NAMESPACES).text
+        api_name = None
         if owlbot_yaml_path.exists():
             # If OwlBot configuration file exists (most cases), it's the better
             # source to get the OwlBot-generated pull request title prefix than
@@ -78,21 +79,21 @@ def detect_modules(root_directory: Path):
                 match = re.search(r'api-name: (.+)', owlbot_yaml_content)
                 if match:
                     api_name = match.group(1)
-                    modules.append(LibraryModule(module_path, api_name,
-                                                 version,
-                                                 changelog))
+
+        if not api_name:
+            # Fallback to repo-metadata.json (e.g. for vertexai or Spanner transitional state)
+            if repo_metadata_path.exists():
+                with open(repo_metadata_path, 'r') as file:
+                    repo_metadata = json.load(file)
+                    api_name = repo_metadata.get('api_shortname')
+
+        if api_name:
+            modules.append(LibraryModule(module_path, api_name,
+                                         version,
+                                         changelog))
         else:
-            # vertexai (handwritten) does not have OwlBot yaml file
-            with open(repo_metadata_path, 'r') as file:
-                repo_metadata = json.load(file)
-                api_name = repo_metadata['api_shortname']
-                if api_name:
-                    modules.append(LibraryModule(repo_metadata_path.parent, api_name,
-                                                 version,
-                                                 changelog))
-                else:
-                    raise Exception(f'repo_metadata_path {repo_metadata_path} does'
-                                    f' not have api_shortname field')
+            raise Exception(f'Could not determine api-name for {repo_metadata_path}')
+
 
     return modules
 
@@ -133,7 +134,7 @@ def group_changes_by_api(main_changes: [str]):
             elif section == BUG_FIXES_SECTION:
                 api_to_changelog[api_name].bug_fixes.append(note)
             elif section == DEPENDENCIES_SECTION:
-                api_to_changelog[api_name].dependencies.append(note)
+                api_to_changelog[api_name].dependency_upgrades.append(note)
     return api_to_changelog
 
 

.github/release-note-generation/unit_test.py

@@ -1,9 +1,11 @@
 import unittest
+import tempfile
+import json
+from pathlib import Path
 
 # Unit tests for split_release_note.py
 
-from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi
-from pathlib import Path
+from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi, detect_modules
 
 dummy_module = LibraryModule(
     Path('release-note-generation/test/java-analyics-admin'),
@@ -81,6 +83,31 @@ def test_group_changes_by_api(self):
                                      ['No change']),
                          ['No change'])
 
+    def test_detect_modules_fallback(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            tmp_path = Path(tmpdirname)
+            module_path = tmp_path / "java-spanner"
+            module_path.mkdir()
+            
+            # Create minimal pom.xml
+            pom_path = module_path / "pom.xml"
+            with open(pom_path, "w") as f:
+                f.write('<project xmlns:mvn="http://maven.apache.org/POM/4.0.0"><mvn:version>1.0.0</mvn:version></project>')
+                
+            # Create .repo-metadata.json with api_shortname
+            metadata_path = module_path / ".repo-metadata.json"
+            with open(metadata_path, "w") as f:
+                json.dump({"api_shortname": "spanner"}, f)
+                
+            # Create CHANGELOG.md
+            changelog_path = module_path / "CHANGELOG.md"
+            changelog_path.touch()
+            
+            modules = detect_modules(tmp_path)
+            self.assertEqual(len(modules), 1)
+            self.assertEqual(modules[0].api_name, "spanner")
+            self.assertEqual(modules[0].version, "1.0.0")
+
 
 if __name__ == "__main__":
     unittest.main()

.github/workflows/ci.yaml

@@ -34,20 +34,11 @@ jobs:
     - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
       id: filter
       with:
-        # we want to run tests if source code is changed or the scripts
-        # used to run the unit tests
         filters: |
           src:
-            - '**/*.java'
-            - '**/pom.xml'
-            - '!java-bigquery/**'
-            - '!java-bigquerystorage/**'
-            - '!java-datastore/**'
-            - '!java-logging-logback/**'
-            - '!java-logging/**'
-            - '!java-spanner/**'
-            - '!java-storage/**'
-            - '!google-auth-library-java/**'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/*.java'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/pom.xml'
+            - 'pom.xml'
           ci:
             - '.github/workflows/ci.yaml'
             - '.kokoro/**'
@@ -134,42 +125,49 @@ jobs:
           java-bigquery:
             - 'java-bigquery/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-bigquerystorage:
             - 'java-bigquerystorage/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-datastore:
             - 'java-datastore/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-logging-logback:
             - 'java-logging-logback/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-logging:
             - 'java-logging/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-spanner:
             - 'java-spanner/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
           java-storage:
             - 'java-storage/**'
             - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
             - 'sdk-platform-java/**/*.java'
             - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
             - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
@@ -246,8 +244,40 @@ jobs:
         JOB_TYPE: test
         JOB_NAME: units-8-runtime-${{matrix.java}}
       working-directory: ${{matrix.package}}
+  split-clirr:
+    runs-on: ubuntu-latest
+    needs: changes
+    strategy:
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
+        JOB_TYPE: clirr
+        JOB_NAME: clirr-${{matrix.package}}
+  split-dependencies:
+    runs-on: ubuntu-latest
+    needs: changes
+    strategy:
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: .kokoro/dependencies.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
   required:
-    needs: [ changes, split-units ]
+    needs: [ changes, split-units, split-clirr, split-dependencies ]
     name: conditional-required-check
     if: ${{ always() }} # Always run even if any "needs" jobs fail
     runs-on: ubuntu-22.04

.github/workflows/google-auth-library-java-ci.yaml

@@ -55,8 +55,35 @@ jobs:
         BUILD_SUBDIR: google-auth-library-java
         JOB_TYPE: test
         SUREFIRE_JVM_OPT: "-P '!slf4j2x,slf4j2x-test'"
+  clirr:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: clirr
+        BUILD_SUBDIR: google-auth-library-java
+  dependencies:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: .kokoro/dependencies.sh
+
   required:
-    needs: [ units-logging ]
+    needs: [ units-logging, clirr, dependencies ]
     name: conditional-required-check
     if: ${{ always() }} # Always run even if any "needs" jobs fail
     runs-on: ubuntu-22.04

.github/workflows/hermetic_library_generation.yaml

@@ -47,4 +47,5 @@ jobs:
         head_ref: ${{ github.head_ref }}
         token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}
         force_regenerate_all: ${{ github.event.pull_request.head.ref == 'generate-libraries-main' }}
+        showcase_mode: true
         image_tag: ${{ env.GENERATOR_VERSION }}

.github/workflows/java-spanner-jdbc-ci.yaml

@@ -35,6 +35,11 @@ jobs:
           library:
             - 'java-spanner-jdbc/**'
             - '.github/workflows/java-spanner-jdbc-ci.yaml'
+            - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
+            - 'sdk-platform-java/**/*.java'
+            - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
+            - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
   units:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -99,16 +104,12 @@ jobs:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [17]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
       with:
         distribution: temurin
-        java-version: ${{matrix.java}}
-    - run: java -version
+        java-version: 17
     - run: .kokoro/dependencies.sh
   javadoc:
     needs: filter

.github/workflows/java-storage-nio-ci.yaml

@@ -35,6 +35,11 @@ jobs:
           library:
             - 'java-storage-nio/**'
             - '.github/workflows/java-storage-nio-ci.yaml'
+            - 'google-auth-library-java/**/*.java'
+            - 'google-auth-library-java/**/pom.xml'
+            - 'sdk-platform-java/**/*.java'
+            - 'sdk-platform-java/java-shared-dependencies/**/pom.xml'
+            - 'sdk-platform-java/gapic-generator-java-pom-parent/pom.xml'
   units:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -99,16 +104,12 @@ jobs:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [17]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
       with:
         distribution: temurin
-        java-version: ${{matrix.java}}
-    - run: java -version
+        java-version: 17
     - run: .kokoro/dependencies.sh
   javadoc:
     needs: filter