Merge branch 'observability/test/compute-integration-test' of https://github.com/googleapis/google-cloud-java into observability/test/compute-integration-test

Author: diegomarquezp

.github/release-note-generation/split_release_note.py

@@ -69,6 +69,7 @@ def detect_modules(root_directory: Path):
         tree = ET.parse(module_pom_xml)
         root = tree.getroot()
         version = root.find('mvn:version', POM_NAMESPACES).text
+        api_name = None
         if owlbot_yaml_path.exists():
             # If OwlBot configuration file exists (most cases), it's the better
             # source to get the OwlBot-generated pull request title prefix than
@@ -78,21 +79,21 @@ def detect_modules(root_directory: Path):
                 match = re.search(r'api-name: (.+)', owlbot_yaml_content)
                 if match:
                     api_name = match.group(1)
-                    modules.append(LibraryModule(module_path, api_name,
-                                                 version,
-                                                 changelog))
+
+        if not api_name:
+            # Fallback to repo-metadata.json (e.g. for vertexai or Spanner transitional state)
+            if repo_metadata_path.exists():
+                with open(repo_metadata_path, 'r') as file:
+                    repo_metadata = json.load(file)
+                    api_name = repo_metadata.get('api_shortname')
+
+        if api_name:
+            modules.append(LibraryModule(module_path, api_name,
+                                         version,
+                                         changelog))
         else:
-            # vertexai (handwritten) does not have OwlBot yaml file
-            with open(repo_metadata_path, 'r') as file:
-                repo_metadata = json.load(file)
-                api_name = repo_metadata['api_shortname']
-                if api_name:
-                    modules.append(LibraryModule(repo_metadata_path.parent, api_name,
-                                                 version,
-                                                 changelog))
-                else:
-                    raise Exception(f'repo_metadata_path {repo_metadata_path} does'
-                                    f' not have api_shortname field')
+            raise Exception(f'Could not determine api-name for {repo_metadata_path}')
+
 
     return modules
 
@@ -133,7 +134,7 @@ def group_changes_by_api(main_changes: [str]):
             elif section == BUG_FIXES_SECTION:
                 api_to_changelog[api_name].bug_fixes.append(note)
             elif section == DEPENDENCIES_SECTION:
-                api_to_changelog[api_name].dependencies.append(note)
+                api_to_changelog[api_name].dependency_upgrades.append(note)
     return api_to_changelog
 
 

.github/release-note-generation/unit_test.py

@@ -1,9 +1,11 @@
 import unittest
+import tempfile
+import json
+from pathlib import Path
 
 # Unit tests for split_release_note.py
 
-from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi
-from pathlib import Path
+from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi, detect_modules
 
 dummy_module = LibraryModule(
     Path('release-note-generation/test/java-analyics-admin'),
@@ -81,6 +83,31 @@ def test_group_changes_by_api(self):
                                      ['No change']),
                          ['No change'])
 
+    def test_detect_modules_fallback(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            tmp_path = Path(tmpdirname)
+            module_path = tmp_path / "java-spanner"
+            module_path.mkdir()
+            
+            # Create minimal pom.xml
+            pom_path = module_path / "pom.xml"
+            with open(pom_path, "w") as f:
+                f.write('<project xmlns:mvn="http://maven.apache.org/POM/4.0.0"><mvn:version>1.0.0</mvn:version></project>')
+                
+            # Create .repo-metadata.json with api_shortname
+            metadata_path = module_path / ".repo-metadata.json"
+            with open(metadata_path, "w") as f:
+                json.dump({"api_shortname": "spanner"}, f)
+                
+            # Create CHANGELOG.md
+            changelog_path = module_path / "CHANGELOG.md"
+            changelog_path.touch()
+            
+            modules = detect_modules(tmp_path)
+            self.assertEqual(len(modules), 1)
+            self.assertEqual(modules[0].api_name, "spanner")
+            self.assertEqual(modules[0].version, "1.0.0")
+
 
 if __name__ == "__main__":
     unittest.main()

.github/workflows/ci.yaml

@@ -34,20 +34,11 @@ jobs:
     - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
       id: filter
       with:
-        # we want to run tests if source code is changed or the scripts
-        # used to run the unit tests
         filters: |
           src:
-            - '**/*.java'
-            - '**/pom.xml'
-            - '!java-bigquery/**'
-            - '!java-bigquerystorage/**'
-            - '!java-datastore/**'
-            - '!java-logging-logback/**'
-            - '!java-logging/**'
-            - '!java-spanner/**'
-            - '!java-storage/**'
-            - '!google-auth-library-java/**'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/*.java'
+            - '!(java-bigquery|java-bigquerystorage|java-datastore|java-logging-logback|java-logging|java-spanner|java-storage|google-auth-library-java)/**/pom.xml'
+            - 'pom.xml'
           ci:
             - '.github/workflows/ci.yaml'
             - '.kokoro/**'
@@ -246,6 +237,37 @@ jobs:
         JOB_TYPE: test
         JOB_NAME: units-8-runtime-${{matrix.java}}
       working-directory: ${{matrix.package}}
+  split-clirr:
+    runs-on: ubuntu-latest
+    needs: changes
+    strategy:
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+    steps:
+    - name: Get current week within the year
+      id: date
+      run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
+        JOB_TYPE: clirr
+        JOB_NAME: clirr-${{matrix.package}}
+  required:
+    needs: [ changes, split-units, split-clirr ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ needs.changes.outputs.packages != '[]' && contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"
   windows:
     runs-on: windows-latest
     steps:
@@ -332,30 +354,3 @@ jobs:
       env:
         library_generation_image_tag: 2.68.0
         workspace_name: /workspace
-
-# TODO: Uncomment the needed Github Actions
-#  dependencies:
-#    runs-on: ubuntu-latest
-#    strategy:
-#      matrix:
-#        java: [8, 11, 17]
-#    steps:
-#    - uses: actions/checkout@v3
-#    - uses: actions/setup-java@v3
-#      with:
-#        distribution: zulu
-#        java-version: ${{matrix.java}}
-#    - run: java -version
-#    - run: .kokoro/dependencies.sh
-#  clirr:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v3
-#    - uses: actions/setup-java@v3
-#      with:
-#        distribution: zulu
-#        java-version: 8
-#    - run: java -version
-#    - run: .kokoro/build.sh
-#      env:
-#        JOB_TYPE: clirr

.github/workflows/google-auth-library-java-ci.yaml

@@ -34,6 +34,7 @@ jobs:
         filters: |
           library:
             - 'google-auth-library-java/**'
+            - '.github/workflows/google-auth-library-java-ci.yaml'
   units-logging:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -54,3 +55,30 @@ jobs:
         BUILD_SUBDIR: google-auth-library-java
         JOB_TYPE: test
         SUREFIRE_JVM_OPT: "-P '!slf4j2x,slf4j2x-test'"
+  clirr:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 11
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: clirr
+        BUILD_SUBDIR: google-auth-library-java
+
+  required:
+    needs: [ units-logging, clirr ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"

…form-java-verify_library_generation.yaml …workflows/hermetic-build-scripts-ci.yaml.github/workflows/sdk-platform-java-verify_library_generation.yaml renamed to .github/workflows/hermetic-build-scripts-ci.yaml .github/workflows/sdk-platform-java-verify_library_generation.yaml renamed to .github/workflows/hermetic-build-scripts-ci.yaml

@@ -1,9 +1,7 @@
 on:
   pull_request:
 
-name: sdk-platform-java verify_library_generation
-env:
-  BUILD_SUBDIR: sdk-platform-java
+name: hermetic build scripts
 jobs:
   filter:
     runs-on: ubuntu-latest
@@ -16,48 +14,14 @@ jobs:
       with:
         filters: |
           library:
-            - 'sdk-platform-java/**'
-  should-run-library-generation-tests:
-    needs: filter
-    if: ${{ needs.filter.outputs.library == 'true' }}
-    runs-on: ubuntu-22.04
-    outputs:
-      should_run: ${{ steps.get_changed_directories.outputs.should_run }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: get changed directories in the pull request
-      id: get_changed_directories
-      shell: bash
-      run: |
-        set -ex
-        # PRs that come from a fork need to be handled differently
-        if [[ ${head_repo_name} == ${base_repo} ]]; then
-          git checkout ${base_ref}
-          git checkout ${head_ref}
-          changed_directories="$(git diff --name-only ${base_ref} ${head_ref})"
-        else
-          git remote add fork ${head_repo_url}
-          git fetch fork # create a mapping of the fork
-          git checkout -b "${head_ref}" fork/${head_ref}
-          changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")"
-        fi
-        if [[ ${changed_directories} =~ "sdk-platform-java/hermetic_build/" ]] || [[ ${changed_directories} =~ "sdk-platform-java/.cloudbuild/library_generation/" ]]; then
-          echo "should_run=true" >> $GITHUB_OUTPUT
-        else
-          echo "should_run=false" >> $GITHUB_OUTPUT
-        fi
-      env:
-        base_ref: ${{ github.event.pull_request.base.ref }}
-        head_ref: ${{ github.event.pull_request.head.ref }}
-        head_repo_url: ${{ github.event.pull_request.head.repo.html_url }}
-        head_repo_name: ${{ github.event.pull_request.head.repo.full_name  }}
-        base_repo: ${{ github.repository }}
+            - 'sdk-platform-java/hermetic_build/**'
+            - 'sdk-platform-java/.cloudbuild/library_generation/**'
+            - '.github/workflows/hermetic-build-scripts-ci.yaml'
   library-generation-unit-tests:
-    needs: [filter, should-run-library-generation-tests]
-    if: needs.filter.outputs.library == 'true' && needs.should-run-library-generation-tests.outputs.should_run == 'true'
+    needs: filter
+    if: needs.filter.outputs.library == 'true'
     runs-on: ubuntu-22.04
+    name: hermetic build units (python)
     defaults:
       run:
         working-directory: sdk-platform-java
@@ -86,9 +50,10 @@ jobs:
         set -x
         python -m unittest discover -s hermetic_build -p "*unit_tests.py"
   library-generation-lint-shell:
-    needs: [filter, should-run-library-generation-tests]
-    if: needs.filter.outputs.library == 'true' && needs.should-run-library-generation-tests.outputs.should_run == 'true'
+    needs: filter
+    if: needs.filter.outputs.library == 'true'
     runs-on: ubuntu-22.04
+    name: hermetic build lint (shell)
     defaults:
       run:
         working-directory: sdk-platform-java
@@ -103,9 +68,10 @@ jobs:
         ignore_paths: 
           .kokoro
   library-generation-lint-python:
-    needs: [filter, should-run-library-generation-tests]
-    if: needs.filter.outputs.library == 'true' && needs.should-run-library-generation-tests.outputs.should_run == 'true'
+    needs: filter
+    if: needs.filter.outputs.library == 'true'
     runs-on: ubuntu-22.04
+    name: hermetic build lint (python)
     defaults:
       run:
         working-directory: sdk-platform-java
@@ -131,3 +97,14 @@ jobs:
         # exclude generated golden files
         # exclude owlbot until further refaction
         black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)"
+  required:
+    needs: [ library-generation-unit-tests, library-generation-lint-shell, library-generation-lint-python ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"

.github/workflows/hermetic_library_generation.yaml

@@ -47,4 +47,5 @@ jobs:
         head_ref: ${{ github.head_ref }}
         token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}
         force_regenerate_all: ${{ github.event.pull_request.head.ref == 'generate-libraries-main' }}
+        showcase_mode: true
         image_tag: ${{ env.GENERATOR_VERSION }}

.github/workflows/java-spanner-jdbc-ci.yaml

@@ -34,6 +34,7 @@ jobs:
         filters: |
           library:
             - 'java-spanner-jdbc/**'
+            - '.github/workflows/java-spanner-jdbc-ci.yaml'
   units:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -141,3 +142,14 @@ jobs:
         JOB_TYPE: lint
         HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         BASE_SHA: ${{ github.event.pull_request.base.sha }}
+  required:
+    needs: [ units, units-java8, windows, dependencies, javadoc, lint ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"

.github/workflows/java-storage-nio-ci.yaml

@@ -34,6 +34,7 @@ jobs:
         filters: |
           library:
             - 'java-storage-nio/**'
+            - '.github/workflows/java-storage-nio-ci.yaml'
   units:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -141,3 +142,14 @@ jobs:
         JOB_TYPE: lint
         HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         BASE_SHA: ${{ github.event.pull_request.base.sha }}
+  required:
+    needs: [ units, units-java8, windows, dependencies, javadoc, lint ]
+    name: conditional-required-check
+    if: ${{ always() }} # Always run even if any "needs" jobs fail
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fail if any previous failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+      - name: Success otherwise
+        run: echo "Success!"