Merge branch 'main' into rm-managed-identities-v1beta1

Author: zhumin8

.github/release-note-generation/split_release_note.py

@@ -69,6 +69,7 @@ def detect_modules(root_directory: Path):
         tree = ET.parse(module_pom_xml)
         root = tree.getroot()
         version = root.find('mvn:version', POM_NAMESPACES).text
+        api_name = None
         if owlbot_yaml_path.exists():
             # If OwlBot configuration file exists (most cases), it's the better
             # source to get the OwlBot-generated pull request title prefix than
@@ -78,21 +79,21 @@ def detect_modules(root_directory: Path):
                 match = re.search(r'api-name: (.+)', owlbot_yaml_content)
                 if match:
                     api_name = match.group(1)
-                    modules.append(LibraryModule(module_path, api_name,
-                                                 version,
-                                                 changelog))
+
+        if not api_name:
+            # Fallback to repo-metadata.json (e.g. for vertexai or Spanner transitional state)
+            if repo_metadata_path.exists():
+                with open(repo_metadata_path, 'r') as file:
+                    repo_metadata = json.load(file)
+                    api_name = repo_metadata.get('api_shortname')
+
+        if api_name:
+            modules.append(LibraryModule(module_path, api_name,
+                                         version,
+                                         changelog))
         else:
-            # vertexai (handwritten) does not have OwlBot yaml file
-            with open(repo_metadata_path, 'r') as file:
-                repo_metadata = json.load(file)
-                api_name = repo_metadata['api_shortname']
-                if api_name:
-                    modules.append(LibraryModule(repo_metadata_path.parent, api_name,
-                                                 version,
-                                                 changelog))
-                else:
-                    raise Exception(f'repo_metadata_path {repo_metadata_path} does'
-                                    f' not have api_shortname field')
+            raise Exception(f'Could not determine api-name for {repo_metadata_path}')
+
 
     return modules
 
@@ -133,7 +134,7 @@ def group_changes_by_api(main_changes: [str]):
             elif section == BUG_FIXES_SECTION:
                 api_to_changelog[api_name].bug_fixes.append(note)
             elif section == DEPENDENCIES_SECTION:
-                api_to_changelog[api_name].dependencies.append(note)
+                api_to_changelog[api_name].dependency_upgrades.append(note)
     return api_to_changelog
 
 

.github/release-note-generation/unit_test.py

@@ -1,9 +1,11 @@
 import unittest
+import tempfile
+import json
+from pathlib import Path
 
 # Unit tests for split_release_note.py
 
-from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi
-from pathlib import Path
+from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi, detect_modules
 
 dummy_module = LibraryModule(
     Path('release-note-generation/test/java-analyics-admin'),
@@ -81,6 +83,31 @@ def test_group_changes_by_api(self):
                                      ['No change']),
                          ['No change'])
 
+    def test_detect_modules_fallback(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            tmp_path = Path(tmpdirname)
+            module_path = tmp_path / "java-spanner"
+            module_path.mkdir()
+            
+            # Create minimal pom.xml
+            pom_path = module_path / "pom.xml"
+            with open(pom_path, "w") as f:
+                f.write('<project xmlns:mvn="http://maven.apache.org/POM/4.0.0"><mvn:version>1.0.0</mvn:version></project>')
+                
+            # Create .repo-metadata.json with api_shortname
+            metadata_path = module_path / ".repo-metadata.json"
+            with open(metadata_path, "w") as f:
+                json.dump({"api_shortname": "spanner"}, f)
+                
+            # Create CHANGELOG.md
+            changelog_path = module_path / "CHANGELOG.md"
+            changelog_path.touch()
+            
+            modules = detect_modules(tmp_path)
+            self.assertEqual(len(modules), 1)
+            self.assertEqual(modules[0].api_name, "spanner")
+            self.assertEqual(modules[0].version, "1.0.0")
+
 
 if __name__ == "__main__":
     unittest.main()

.github/workflows/ci.yaml

@@ -31,23 +31,14 @@ jobs:
       ci: ${{ steps.filter.outputs.ci }}
     steps:
     - uses: actions/checkout@v4
-    - uses: dorny/paths-filter@v3
+    - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
       id: filter
       with:
-        # we want to run tests if source code is changed or the scripts
-        # used to run the unit tests
         filters: |
           src:
-            - '**/*.java'
-            - '**/pom.xml'
-            - '!java-bigquery/**'
-            - '!java-bigquerystorage/**'
-            - '!java-datastore/**'
-            - '!java-logging-logback/**'
-            - '!java-logging/**'
-            - '!java-spanner/**'
-            - '!java-storage/**'
-            - '!google-auth-library-java/**'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/*.java'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/pom.xml'
+            - 'pom.xml'
           ci:
             - '.github/workflows/ci.yaml'
             - '.kokoro/**'
@@ -121,7 +112,7 @@ jobs:
     outputs:
       packages: ${{ steps.filter.outputs.changes }}
     steps:
-    - uses: dorny/paths-filter@v4
+    - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
       id: filter
       with:
         # For each library, run CI in split repos where there are changes in:
@@ -246,6 +237,37 @@ jobs:
         JOB_TYPE: test
         JOB_NAME: units-8-runtime-${{matrix.java}}
       working-directory: ${{matrix.package}}
+  split-clirr:
+    runs-on: ubuntu-latest
+    needs: changes
+    strategy:
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+    steps:
+    - name: Get current week within the year
+      id: date
+      run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
+        JOB_TYPE: clirr
+        JOB_NAME: clirr-${{matrix.package}}
+  required:
+    needs: [ changes, split-units, split-clirr ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ needs.changes.outputs.packages != '[]' && contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"
   windows:
     runs-on: windows-latest
     steps:
@@ -321,41 +343,14 @@ jobs:
     - name: validate generation configuration
       shell: bash
       run: |
-        docker run \
-          --rm \
+        bash generation/run_generator_docker.sh "${library_generation_image_tag}" "${{ github.base_ref || 'main' }}" \
+          -e GENERATOR_VERSION="${library_generation_image_tag}" \
           --quiet \
           -u "$(id -u):$(id -g)" \
           -v "$(pwd):${workspace_name}" \
           --entrypoint python \
-          gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \
+          -- \
           /src/library_generation/cli/entry_point.py validate-generation-config
       env:
         library_generation_image_tag: 2.68.0
         workspace_name: /workspace
-
-# TODO: Uncomment the needed Github Actions
-#  dependencies:
-#    runs-on: ubuntu-latest
-#    strategy:
-#      matrix:
-#        java: [8, 11, 17]
-#    steps:
-#    - uses: actions/checkout@v3
-#    - uses: actions/setup-java@v3
-#      with:
-#        distribution: zulu
-#        java-version: ${{matrix.java}}
-#    - run: java -version
-#    - run: .kokoro/dependencies.sh
-#  clirr:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v3
-#    - uses: actions/setup-java@v3
-#      with:
-#        distribution: zulu
-#        java-version: 8
-#    - run: java -version
-#    - run: .kokoro/build.sh
-#      env:
-#        JOB_TYPE: clirr

.github/workflows/generated_files_sync.yaml

@@ -27,13 +27,12 @@ jobs:
     - name: Generate root pom.xml file
       shell: bash
       run: |
-        docker run \
-          --rm \
+        bash generation/run_generator_docker.sh "${library_generation_image_tag}" "${{ github.base_ref }}" \
           --quiet \
           -u "$(id -u):$(id -g)" \
           -v "$(pwd):/workspace" \
           --entrypoint python \
-          gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \
+          -- \
           /src/library_generation/cli/generate_monorepo_root_pom.py \
           generate \
           --repository-path=/workspace
@@ -48,13 +47,12 @@ jobs:
     - name: Generate gapic-libraries-bom/pom.xml
       shell: bash
       run: |
-        docker run \
-          --rm \
+        bash generation/run_generator_docker.sh "${library_generation_image_tag}" "${{ github.base_ref }}" \
           --quiet \
           -u "$(id -u):$(id -g)" \
           -v "$(pwd):/workspace" \
           --entrypoint python \
-          gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \
+          -- \
           /src/library_generation/cli/generate_monorepo_gapic_bom.py \
           generate \
           --repository-path=/workspace \

.github/workflows/google-auth-library-java-ci.yaml

@@ -28,12 +28,13 @@ jobs:
       library: ${{ steps.filter.outputs.library }}
     steps:
     - uses: actions/checkout@v4
-    - uses: dorny/paths-filter@v3
+    - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
       id: filter
       with:
         filters: |
           library:
             - 'google-auth-library-java/**'
+            - '.github/workflows/google-auth-library-java-ci.yaml'
   units-logging:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -54,3 +55,30 @@ jobs:
         BUILD_SUBDIR: google-auth-library-java
         JOB_TYPE: test
         SUREFIRE_JVM_OPT: "-P '!slf4j2x,slf4j2x-test'"
+  clirr:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: clirr
+        BUILD_SUBDIR: google-auth-library-java
+
+  required:
+    needs: [ units-logging, clirr ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"