build: use matrix for split unit tests (googleapis#12127)

Uses a GitHub actions matrix and the dorny/paths-filter action (already
in use) to conditionally run split unit tests for a select number of
modules.

When running the bulk unit test suite, we enable use the `bulkTests`
profile. This profile is configured to skip the tests for the ones that
have split tests.

Also fixes the showcase-clirr check by installing necessary dependencies
without running the enforcer check.

Fixes googleapis#12185

Author: chingor13

.github/workflows/ci.yaml

@@ -53,6 +53,85 @@ jobs:
       id: date
       run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
     - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        java-version: 8
+        distribution: temurin
+    - name: "Set jvm system property environment variable for surefire plugin (unit tests)"
+      # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests.
+      # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
+      run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV
+      shell: bash    
+    - uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: temurin
+        cache: maven
+    - uses: actions/cache@v4
+      id: mvn-cache
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
+    - run: .kokoro/build.sh
+      shell: bash      
+      env:
+        JOB_TYPE: test
+        JOB_NAME: units-8-runtime-${{matrix.java}}
+  # detect which libraries have changed
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      packages: ${{ steps.filter.outputs.changes }}
+    steps:
+    - uses: dorny/paths-filter@v4
+      id: filter
+      with:
+        filters: |
+          java-bigquery: java-bigquery/**
+          java-bigquerystorage: java-bigquerystorage/**
+          java-datastore: java-datastore/**
+          java-logging: java-logging/**
+          java-logging-logback: java-logging-logback/**
+          java-spanner: java-spanner/**
+          sdk-platform-java: sdk-platform-java/**
+  split-units:
+    runs-on: ubuntu-latest
+    needs: changes
+    strategy:
+      fail-fast: false
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+        java: [11, 17, 21, 25]
+    steps:
+    - name: Get current week within the year
+      id: date
+      run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{matrix.java}}
+    - run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
+        JOB_TYPE: test
+        JOB_NAME: units-${{matrix.package}}-${{matrix.java}}
+  split-units-8:
+    runs-on: ubuntu-latest
+    name: "split-units"
+    needs: changes
+    strategy:
+      fail-fast: false
+      matrix:
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+        java: [8]
+    steps:
+    - name: Get current week within the year
+      id: date
+      run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
+    - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
       with:
         java-version: 11
@@ -66,15 +145,16 @@ jobs:
         key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
     - name: Install all modules using Java 11
       shell: bash
-      run: |
-        mvn -V -B -ntp clean install -DskipTests -Dfmt.skip -Dclirr.skip -Denforcer.skip -Dcheckstyle.skip -Dflatten.skip -Danimal-sniffer.skip -Djacoco.skip
-      working-directory: sdk-platform-java
+      run: .kokoro/build.sh
+      env:
+        BUILD_SUBDIR: ${{matrix.package}}
+        JOB_TYPE: install
     - uses: actions/setup-java@v4
       with:
-        java-version: 8
+        java-version: ${{matrix.java}}
         distribution: temurin
     - run: java -version
-    - name: Run tests in Java 8 with the source compiled in Java 11
+    - name: Run tests in Java ${{matrix.java}} with the source compiled in Java 11
       run: |
         mvn test \
           -B -ntp \
@@ -86,11 +166,13 @@ jobs:
           -Dflatten.skip=true \
           -Danimal.sniffer.skip=true \
           -Dmaven.wagon.http.retryHandler.count=5 \
+          --also-make \
           -T 1C
       env:
+        BUILD_SUBDIR: ${{matrix.package}}
         JOB_TYPE: test
         JOB_NAME: units-8-runtime-${{matrix.java}}
-      working-directory: sdk-platform-java
+      working-directory: ${{matrix.package}}
   windows:
     runs-on: windows-latest
     steps:

.github/workflows/java-bigquery-ci.yaml

@@ -430,8 +430,11 @@ jobs:
       - name: Checkout sdk-platform-java @ PR merge commit
         uses: actions/checkout@v3
       - name: Install sdk-platform-java @ PR merge commit
-        run: mvn install -B -ntp -T 1C -DskipTests
-        working-directory: sdk-platform-java
+        shell: bash
+        run: .kokoro/build.sh
+        env:
+          JOB_TYPE: install
+          BUILD_SUBDIR: sdk-platform-java
         # Showcase golden test ensures that src changes are already reflected in the PR.
       - name: Clirr check
         working-directory: sdk-platform-java/java-showcase

.github/workflows/sdk-platform-java-ci.yaml

@@ -43,7 +43,9 @@ case ${JOB_TYPE} in
       install_modules "${BUILD_SUBDIR}"
       echo "Running in subdir: ${BUILD_SUBDIR}"
       pushd "${BUILD_SUBDIR}"
+      EXTRA_PROFILE_OPTS=()
     else
+      EXTRA_PROFILE_OPTS=("-PbulkTests")
       install_modules "sdk-platform-java"
     fi
     echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}"
@@ -59,7 +61,7 @@ case ${JOB_TYPE} in
         -Danimal.sniffer.skip=true \
         -Dmaven.wagon.http.retryHandler.count=5 \
         --also-make \
-         ${SUREFIRE_JVM_OPT}
+        ${SUREFIRE_JVM_OPT} "${EXTRA_PROFILE_OPTS[@]}"
     RETURN_CODE=$?
 
     if [[ -n "${BUILD_SUBDIR}" ]]
@@ -69,6 +71,28 @@ case ${JOB_TYPE} in
     fi
     echo "Finished running unit tests"
     ;;
+  install)
+    if [[ -n "${BUILD_SUBDIR}" ]]
+    then
+      echo "Compiling and building all modules for ${BUILD_SUBDIR}"
+      install_modules "${BUILD_SUBDIR}"
+    else
+      install_modules "sdk-platform-java"
+      mvn install \
+        -B -ntp \
+        -Dorg.slf4j.simpleLogger.showDateTime=true \
+        -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+        -Dclirr.skip=true \
+        -Denforcer.skip=true \
+        -Dcheckstyle.skip=true \
+        -Dflatten.skip=true \
+        -Danimal.sniffer.skip=true \
+        -Dmaven.wagon.http.retryHandler.count=5 \
+        -DskipTests=true \
+        --also-make \
+        -T 1C
+    fi
+    ;;
   integration)
     generate_modified_modules_list
     if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then

.kokoro/build.sh

@@ -162,5 +162,12 @@
         <module>benchmark</module>
       </modules>
     </profile>
+    <!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
+    <profile>
+      <id>bulkTests</id>
+      <properties>
+        <skipTests>true</skipTests>
+      </properties>
+    </profile>
   </profiles>
 </project>

java-bigquery/pom.xml

@@ -202,4 +202,14 @@
     <module>google-cloud-bigquerystorage-bom</module>
   </modules>
 
-  </project>
+  <profiles>
+    <!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
+    <profile>
+      <id>bulkTests</id>
+      <properties>
+        <skipTests>true</skipTests>
+      </properties>
+    </profile>
+  </profiles>
+
+</project>

java-bigquerystorage/pom.xml

@@ -232,4 +232,13 @@
     <module>google-cloud-datastore-utils</module>
   </modules>
 
-  </project>
+  <profiles>
+    <!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
+    <profile>
+      <id>bulkTests</id>
+      <properties>
+        <skipTests>true</skipTests>
+      </properties>
+    </profile>
+  </profiles>
+</project>

java-datastore/pom.xml

@@ -168,5 +168,12 @@
         <test>!LoggingAppenderTest</test>
       </properties>
     </profile>
+    <!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
+    <profile>
+      <id>bulkTests</id>
+      <properties>
+        <skipTests>true</skipTests>
+      </properties>
+    </profile>
   </profiles>
 </project>