Merge branch 'main' into remove-dev-gate

Author: ldetmer

.github/release-please.yml

@@ -2,6 +2,7 @@ primaryBranch: main
 releaseType: java-yoshi
 manifest: true
 handleGHRelease: true
+onDemand: true
 branches:
     - branch: 1.0.x
       releaseType: java-backport

.github/workflows/ci.yaml

@@ -20,8 +20,33 @@ on:
   pull_request:
 name: ci
 jobs:
+  # detect whether or note we should run "bulk" (non-handwritten) unit tests
+  bulk-filter:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+      ci: ${{ steps.filter.outputs.ci }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        # we want to run tests if source code is changed or the scripts
+        # used to run the unit tests
+        filters: |
+          src:
+            - '**/*.java'
+            - '**/pom.xml'
+          ci:
+            - '.github/workflows/ci.yaml'
+            - '.kokoro/**'
+  # these unit tests are "bulk" (non-handwritten) libraries
   units:
     runs-on: ubuntu-latest
+    needs: bulk-filter
     strategy:
       fail-fast: false
       matrix:
@@ -42,11 +67,13 @@ jobs:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
     - run: .kokoro/build.sh
+      if: ${{ needs.bulk-filter.outputs.src == 'true' || needs.bulk-filter.outputs.ci == 'true' }}
       env:
         JOB_TYPE: test
         JOB_NAME: units-${{matrix.java}}
   units-8-runtime:
     runs-on: ubuntu-latest
+    needs: bulk-filter
     name: "units (8)"
     steps:
     - name: Get current week within the year
@@ -73,6 +100,7 @@ jobs:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
     - run: .kokoro/build.sh
+      if: ${{ needs.bulk-filter.outputs.src == 'true' || needs.bulk-filter.outputs.ci == 'true' }}
       shell: bash      
       env:
         JOB_TYPE: test

.github/workflows/generated_files_sync.yaml

@@ -17,7 +17,7 @@ on:
   pull_request:
 name: generation diff
 env:
-  library_generation_image_tag: 2.68.1-SNAPSHOT  # {x-version-update:gapic-generator-java:current}
+  library_generation_image_tag: 2.69.0  # {x-version-update:gapic-generator-java:current}
 jobs:
   root-pom:
     # root pom.xml does not have diff from generated one

.github/workflows/hermetic_library_generation.yaml

@@ -43,4 +43,5 @@ jobs:
         base_ref: ${{ github.base_ref }}
         head_ref: ${{ github.head_ref }}
         token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}
-        image_tag: 2.68.1-SNAPSHOT  # {x-version-update:gapic-generator-java:current}
+        force_regenerate_all: ${{ github.event.pull_request.head.ref == 'generate-libraries-main' }}
+        image_tag: 2.69.0  # {x-version-update:gapic-generator-java:current}

.github/workflows/sdk-platform-java-ci.yaml

@@ -33,10 +33,11 @@ jobs:
           cache: maven
       - run: java -version
       - name: Unit Tests
-        run: |
-          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
-              -Dfmt.skip -DenableTestCoverage
-        working-directory: sdk-platform-java
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test
       # The `envVarTest` profile runs tests that require an environment variable
       - name: Env Var Tests
         run: |
@@ -49,10 +50,12 @@ jobs:
           GOOGLE_SDK_JAVA_LOGGING: true
         working-directory: sdk-platform-java
       - run: bazelisk version
-      - name: Install Maven modules
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-        working-directory: sdk-platform-java
+      - name: Install all modules using Java 11
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - name: Integration Tests
         run: |
           bazelisk --batch test //sdk-platform-java/test/integration/...
@@ -89,26 +92,29 @@ jobs:
           cache: maven
       - run: java -version
       - name: Unit Tests
-        run: |
-          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
-              -Dfmt.skip -DenableTestCoverage
-        working-directory: sdk-platform-java
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test
       # The `envVarTest` profile runs tests that require an environment variable
       - name: Env Var Tests
-        run: |
-          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
-              -Dfmt.skip -DenableTestCoverage -PenvVarTest
-        # Set the Env Var for this step only
+        shell: bash
+        run: .kokoro/build.sh
         env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test
+          SUREFIRE_JVM_OPT: '-PenvVarTest'
           GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com
           GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS: true
           GOOGLE_SDK_JAVA_LOGGING: true
-        working-directory: sdk-platform-java
       - run: bazelisk version
-      - name: Install Maven modules
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
-        working-directory: sdk-platform-java
+      - name: Install all modules using Java 11
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - name: Integration Tests
         run: |
           bazelisk --batch test //sdk-platform-java/test/integration/...
@@ -127,26 +133,29 @@ jobs:
           cache: maven
       - run: java -version
       - name: Unit Tests
-        run: |
-          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
-              -Dfmt.skip -DenableTestCoverage
-        working-directory: sdk-platform-java
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test
       # The `envVarTest` profile runs tests that require an environment variable
       - name: Env Var Tests
-        run: |
-          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
-              -Dfmt.skip -DenableTestCoverage -PenvVarTest
-        # Set the Env Var for this step only
+        shell: bash
+        run: .kokoro/build.sh
         env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test
+          SUREFIRE_JVM_OPT: '-PenvVarTest'
           GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com
           GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS: true
           GOOGLE_SDK_JAVA_LOGGING: true
-        working-directory: sdk-platform-java
       - run: bazelisk version
-      - name: Install Maven modules
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-        working-directory: sdk-platform-java
+      - name: Install all modules using Java 11
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - name: Integration Tests
         # note need to set shouldInstallTestSecurityManager=false due to https://github.com/bazelbuild/bazel/issues/24354
         run: |
@@ -164,12 +173,17 @@ jobs:
           java-version: 8
           distribution: temurin
           cache: maven
+      - name: Compile with Java 17 and run tests with Java 8
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV
       - uses: actions/setup-java@v3
         with:
           java-version: 17
           distribution: temurin
-      - name: Compile with Java 17 and run tests with Java 8
         shell: bash
         run: |
           set -x
@@ -215,9 +229,10 @@ jobs:
           cache: maven
       - name: Install all modules using Java 11
         shell: bash
-        run: |
-          mvn -V -B -ntp clean install -DskipTests -Dfmt.skip -Dcheckstyle.skip
-        working-directory: sdk-platform-java
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - uses: actions/setup-java@v3
         with:
           java-version: 8
@@ -265,9 +280,10 @@ jobs:
           cache: maven
       - name: Install all modules using Java 11
         shell: bash
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-        working-directory: sdk-platform-java
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - uses: actions/setup-java@v3
         with:
           java-version: 8
@@ -339,9 +355,11 @@ jobs:
           distribution: temurin
       - run: mvn -version
       - name: Install Maven modules
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-        working-directory: sdk-platform-java        
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: test      
       - name: Showcase golden tests
         working-directory: sdk-platform-java/java-showcase
         run: |
@@ -418,9 +436,11 @@ jobs:
           distribution: temurin
           cache: maven
       - name: Install Maven modules
-        run: |
-          mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-        working-directory: sdk-platform-java
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - name: Install showcase to local maven repository
         run: |
           mvn install -B -ntp -T 1C -DskipTests
@@ -454,20 +474,11 @@ jobs:
         distribution: temurin
         cache: maven
     - name: Install Maven modules to local Maven repository
-      run: |
-        mvn install \
-          -B -ntp \
-          -Dorg.slf4j.simpleLogger.showDateTime=true \
-          -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-          -DskipTests \
-          -Dclirr.skip=true \
-          -Denforcer.skip=true \
-          -Dcheckstyle.skip=true \
-          -Dflatten.skip=true \
-          -Danimal.sniffer.skip=true \
-          -Dmaven.wagon.http.retryHandler.count=5 \
-          -T 1C
-      working-directory: sdk-platform-java
+      shell: bash
+      run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: sdk-platform-java
+        JOB_TYPE: install
     - name: Validate gapic-generator-java-bom
       uses: googleapis/java-cloud-bom/tests/validate-bom@47ad868794c5d38ee7eb95d69371a7a79f84d890
       with:
@@ -491,9 +502,11 @@ jobs:
         mvn test --batch-mode --no-transfer-progress
       working-directory: sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check
     - name: Install Maven modules
-      run: |
-        mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip -Dfmt.skip
-      working-directory: sdk-platform-java
+      shell: bash
+      run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: sdk-platform-java
+        JOB_TYPE: test
     - name: Unmanaged dependency check
       uses: ./sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check
       with:

.github/workflows/sdk-platform-java-dependency_compatibility_test.yaml

@@ -50,6 +50,13 @@ jobs:
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dependencies-list != '' }}
         run: echo "DEPENDENCIES_LIST=${{ github.event.inputs.dependencies-list }}" >> $GITHUB_ENV
 
+      - name: Install sdk-platform-java modules
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
+
       # Run in the root module which should test for everything except for showcase
       - name: Perform Dependency Compatibility Unit Testing
         shell: bash
@@ -61,12 +68,6 @@ jobs:
           fi
         working-directory: sdk-platform-java
 
-      # Install the modules for showcase (Shared-Deps is required to run showcase)
-      - name: Install sdk-platform-java's modules
-        # gapic-generator-java requires Java 8 and is irrelevant for this CI
-        run: mvn -q -B -ntp install --projects '!gapic-generator-java' -Dcheckstyle.skip -Dfmt.skip -DskipTests -Dclirr.skip -T 1C
-        working-directory: sdk-platform-java
-
       # Set up local showcase server to run the showcase ITs
       - name: Parse showcase version
         working-directory: sdk-platform-java/java-showcase/gapic-showcase

.github/workflows/sdk-platform-java-downstream.yaml

@@ -32,14 +32,9 @@ jobs:
       matrix:
         repo:
           - java-bigtable
-          - java-bigquery
           - java-firestore
           - java-pubsub
           - java-pubsublite
-          - java-spanner-jdbc
-          - java-spanner
-          - java-storage
-          - java-storage-nio
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
@@ -52,9 +47,9 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install libxml2-utils
       - name: Test helper scripts
-        run: ./sdk-platform-java/.kokoro/presubmit/common_test.sh
+        run: .kokoro/common_test.sh
       - name: Perform downstream compatibility testing
-        run: REPOS_UNDER_TEST="${{ matrix.repo }}" ./sdk-platform-java/.kokoro/presubmit/downstream-compatibility.sh
+        run: REPOS_UNDER_TEST="${{ matrix.repo }}" .kokoro/downstream-compatibility.sh
   downstream-compatibility-spring-generator:
     needs: filter
     if: ${{ needs.filter.outputs.library == 'true' }}
@@ -73,4 +68,4 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install libxml2-utils
       - name: Perform downstream compatibility testing
-        run: ./sdk-platform-java/.kokoro/presubmit/downstream-compatibility-spring.sh
+        run: .kokoro/downstream-compatibility-spring.sh