Merge branch 'main' into datastore-csm-impl-1

Author: lqiu96

.github/CODEOWNERS

@@ -13,3 +13,4 @@
 /java-spanner-jdbc/ @googleapis/spanner-team @googleapis/cloud-sdk-java-team
 /google-auth-library-java/ @googleapis/cloud-sdk-auth-team @googleapis/cloud-sdk-java-team
 /java-storage/ @googleapis/gcs-team @googleapis/cloud-sdk-java-team
+/java-storage-nio/ @googleapis/gcs-team @googleapis/cloud-sdk-java-team

.github/header-checker-lint.yml

@@ -1,3 +1,4 @@
 allowedCopyrightHolders:
 - 'Google LLC'
+- 'Google Inc'
 - 'The Grafeas Authors'

.github/workflows/ci.yaml

@@ -20,8 +20,33 @@ on:
   pull_request:
 name: ci
 jobs:
+  # detect whether or note we should run "bulk" (non-handwritten) unit tests
+  bulk-filter:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+      ci: ${{ steps.filter.outputs.ci }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        # we want to run tests if source code is changed or the scripts
+        # used to run the unit tests
+        filters: |
+          src:
+            - '**/*.java'
+            - '**/pom.xml'
+          ci:
+            - '.github/workflows/ci.yaml'
+            - '.kokoro/**'
+  # these unit tests are "bulk" (non-handwritten) libraries
   units:
     runs-on: ubuntu-latest
+    needs: bulk-filter
     strategy:
       fail-fast: false
       matrix:
@@ -42,11 +67,13 @@ jobs:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
     - run: .kokoro/build.sh
+      if: ${{ needs.bulk-filter.outputs.src == 'true' || needs.bulk-filter.outputs.ci == 'true' }}
       env:
         JOB_TYPE: test
         JOB_NAME: units-${{matrix.java}}
   units-8-runtime:
     runs-on: ubuntu-latest
+    needs: bulk-filter
     name: "units (8)"
     steps:
     - name: Get current week within the year
@@ -73,6 +100,7 @@ jobs:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
     - run: .kokoro/build.sh
+      if: ${{ needs.bulk-filter.outputs.src == 'true' || needs.bulk-filter.outputs.ci == 'true' }}
       shell: bash      
       env:
         JOB_TYPE: test
@@ -220,8 +248,9 @@ jobs:
         cache: maven
     - run: java -version
     - name: Install Maven modules to local Maven repository
-      run: |
-        mvn install -B -ntp  -T 1C -DskipTests -Dclirr.skip -Dcheckstyle.skip
+      run: .kokoro/build.sh
+      env:
+        JOB_TYPE: install
     - run: java -version
     - run: mvn -B -ntp enforcer:enforce@enforce -T 1C
   gapic-libraries-bom:
@@ -234,8 +263,9 @@ jobs:
         distribution: temurin
         cache: maven
     - name: Install Maven modules to local Maven repository
-      run: |
-        mvn install -B -ntp  -T 1C -DskipTests -Dclirr.skip -Dcheckstyle.skip -Denforcer.skip
+      run: .kokoro/build.sh
+      env:
+        JOB_TYPE: install
     - name: Validate gapic-libraries-bom
       uses: googleapis/java-cloud-bom/tests/validate-bom@v26.79.0
       with:

.github/workflows/hermetic_library_generation.yaml

@@ -43,4 +43,5 @@ jobs:
         base_ref: ${{ github.base_ref }}
         head_ref: ${{ github.head_ref }}
         token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}
+        force_regenerate_all: ${{ github.event.pull_request.head.ref == 'generate-libraries-main' }}
         image_tag: 2.68.1-SNAPSHOT  # {x-version-update:gapic-generator-java:current}

.github/workflows/java-storage-nio-ci.yaml

@@ -0,0 +1,143 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+name: java-storage-nio ci
+env:
+  BUILD_SUBDIR: java-storage-nio
+jobs:
+  filter:
+    runs-on: ubuntu-latest
+    outputs:
+      library: ${{ steps.filter.outputs.library }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          library:
+            - 'java-storage-nio/**'
+  units:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [11, 17, 21, 25]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{matrix.java}}
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: test
+  units-java8:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    # Building using Java 17 and run the tests with Java 8 runtime
+    name: "units (8)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 8
+          distribution: temurin
+      - name: "Set jvm system property environment variable for surefire plugin (unit tests)"
+        # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests.
+        # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
+        run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java -P !java17" >> $GITHUB_ENV
+        shell: bash
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - run: .kokoro/build.sh
+        env:
+          JOB_TYPE: test
+  windows:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: windows-latest
+    steps:
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 8
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: test
+  dependencies:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [17]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{matrix.java}}
+    - run: java -version
+    - run: .kokoro/dependencies.sh
+  javadoc:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: javadoc
+  lint:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: lint
+        HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        BASE_SHA: ${{ github.event.pull_request.base.sha }}

.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml

@@ -12,44 +12,25 @@ on:
   schedule:
     - cron: '0 1 * * *' # Nightly at 1am
 
+# This job intends to test the compatibility of the Protobuf version against repos not in the
+# monorepo. As repos are migrated to the monorepo, this job will be obsolete and turned off.
 name: sdk-platform-java Downstream Protobuf Compatibility Check Nightly
-env:
-  BUILD_SUBDIR: sdk-platform-java
 jobs:
-  filter:
-    runs-on: ubuntu-latest
-    outputs:
-      library: ${{ steps.filter.outputs.library }}
-    steps:
-    - uses: actions/checkout@v4
-    - uses: dorny/paths-filter@v3
-      id: filter
-      with:
-        filters: |
-          library:
-            - 'sdk-platform-java/**'
   downstream-protobuf-test:
-    needs: filter
     # This job runs if any of the three conditions match:
     # 1. PR is raised from Release-Please (PR comes from branch: release-please--branches-main)
     # 2. Job is invoked by the nightly job (scheduled event)
     # 3. Job is manually invoked via Github UI (workflow_dispatch event)
-    if: needs.filter.outputs.library == 'true' && (github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+    if: github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
         repo:
-          - google-cloud-java
           - java-bigtable
-          - java-bigquery
           - java-firestore
           - java-pubsub
           - java-pubsublite
-          - java-spanner-jdbc
-          - java-spanner
-          - java-storage
-          - java-storage-nio
         # Default Protobuf-Java versions to use are specified here. Without this, the nightly workflow won't know
         # which values to use and would resolve to ''.
         protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"4.33.5"')) }}
@@ -70,7 +51,7 @@ jobs:
       - name: Print Protobuf-Java testing version
         run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}"
       - name: Perform downstream source compatibility testing
-        run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh
+        run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh
       - name: Perform downstream binary compatibility testing
-        run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh
+        run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh
 

.github/workflows/sdk-platform-java-nightly.yaml

@@ -6,24 +6,8 @@ on:
 defaults:
   run:
     shell: bash
-env:
-  BUILD_SUBDIR: sdk-platform-java
 jobs:
-  filter:
-    runs-on: ubuntu-latest
-    outputs:
-      library: ${{ steps.filter.outputs.library }}
-    steps:
-    - uses: actions/checkout@v4
-    - uses: dorny/paths-filter@v3
-      id: filter
-      with:
-        filters: |
-          library:
-            - 'sdk-platform-java/**'
   nightly:
-    needs: filter
-    if: ${{ needs.filter.outputs.library == 'true' }}
     strategy:
       fail-fast: false
       matrix:
@@ -39,8 +23,15 @@ jobs:
           distribution: temurin
           cache: maven
       - run: mvn -version
+      - name: Install all modules first
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - name: Install
         run: mvn install --errors --batch-mode --no-transfer-progress -Dcheckstyle.skip -Dfmt.skip
+        working-directory: sdk-platform-java
       - name: Create issue if previous step fails
         if: ${{ failure() }}
         env:
@@ -50,8 +41,6 @@ jobs:
             --title "Nightly build for Java ${{ matrix.java }} on ${{ matrix.os }} failed." \
             --body "The build has failed : https://github.com/googleapis/google-cloud-java/actions/runs/${GITHUB_RUN_ID}"
   nightly-java8: # Compile with JDK 11. Run tests with JDK 8.
-    needs: filter
-    if: ${{ needs.filter.outputs.library == 'true' }}
     strategy:
       fail-fast: false
       matrix:
@@ -66,9 +55,12 @@ jobs:
           distribution: temurin
           cache: maven
       - run: mvn -version
-      - name: Install with Java 11
-        run: mvn install --errors --batch-mode --no-transfer-progress -Dcheckstyle.skip -DskipTests -Dfmt.skip
-
+      - name: Install all modules first
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          BUILD_SUBDIR: sdk-platform-java
+          JOB_TYPE: install
       - uses: actions/setup-java@v5
         with:
           java-version: 8
@@ -77,6 +69,7 @@ jobs:
       - name: Test with Java 8
         # Direct goal invocation ("surefire:test") prevents recompiling tests
         run: mvn surefire:test --errors --batch-mode --no-transfer-progress -Dfmt.skip
+        working-directory: sdk-platform-java
       - name: Create issue if previous step fails
         if: ${{ failure() }}
         env:

.kokoro/common.sh

@@ -33,6 +33,7 @@ excluded_modules=(
   'java-spanner-jdbc'
   'google-auth-library-java'
   'java-storage'
+  'java-storage-nio'
 )
 
 function retry_with_backoff {
@@ -422,6 +423,16 @@ function install_modules() {
       'sdk-platform-java/java-shared-dependencies/first-party-dependencies'
       'sdk-platform-java/java-shared-dependencies/third-party-dependencies'
       'sdk-platform-java/gapic-generator-java-bom'
+      'sdk-platform-java/java-iam/grpc-google-iam-v1'
+      'sdk-platform-java/java-iam/grpc-google-iam-v2'
+      'sdk-platform-java/java-iam/grpc-google-iam-v2beta'
+      'sdk-platform-java/java-iam/grpc-google-iam-v3'
+      'sdk-platform-java/java-iam/grpc-google-iam-v3beta'
+      'sdk-platform-java/java-iam/proto-google-iam-v1'
+      'sdk-platform-java/java-iam/proto-google-iam-v2'
+      'sdk-platform-java/java-iam/proto-google-iam-v2beta'
+      'sdk-platform-java/java-iam/proto-google-iam-v3'
+      'sdk-platform-java/java-iam/proto-google-iam-v3beta'
       'sdk-platform-java/java-core/google-cloud-core-bom'
       'sdk-platform-java/java-core/google-cloud-core'
       'sdk-platform-java/java-core/google-cloud-core-grpc'