From 4e8db207fdb2e809bc817f1bbf7baaae28264dda Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 26 Feb 2026 00:38:41 -0500 Subject: [PATCH 1/9] ci: Run fmt plugin only for the relevant modules --- .kokoro/build.sh | 62 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 7afa8f3b6f26..695b85587223 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -194,12 +194,15 @@ case ${JOB_TYPE} in echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" fi + + MODULE_FILTER="" + if [ -n "${BASE_SHA}" ] && [ -n "${HEAD_SHA}" ]; then changed_file_list=$(git diff --name-only "${BASE_SHA}" "${HEAD_SHA}") echo "${changed_file_list}" - + has_code_change="false" - + while IFS= read -r changed_file; do # Checks if the line is not empty AND if it matches a .java file if [ -n "${changed_file}" ] && [[ "${changed_file}" == *.java ]]; then @@ -208,19 +211,66 @@ case ${JOB_TYPE} in break fi done <<< "${changed_file_list}" - + if [ "${has_code_change}" == "false" ]; then echo "No java modules affected. Skipping linter check." exit 0 fi + + # Compute list of changed Maven modules from changed Java files + # Walk each changed .java file up to its nearest pom.xml to find the module + changed_modules=() + while IFS= read -r changed_file; do + if [ -n "${changed_file}" ] && [[ "${changed_file}" == *.java ]]; then + dir=$(dirname "${changed_file}") + while [ "${dir}" != "." ] && [ ! -f "${dir}/pom.xml" ]; do + dir=$(dirname "${dir}") + done + if [ -f "${dir}/pom.xml" ] && [ "${dir}" != "." ]; then + changed_modules+=("${dir}") + fi + fi + done <<< "${changed_file_list}" + + # Deduplicate + if [ ${#changed_modules[@]} -gt 0 ]; then + unique_modules=$(printf '%s\n' "${changed_modules[@]}" | sort -u | paste -sd ',' -) + MODULE_FILTER="-pl ${unique_modules}" + echo "Formatting only changed modules: ${unique_modules}" + fi else echo "BASE_SHA or HEAD_SHA is empty. Skipping file difference check." + + # For non-PR runs (no diff available), exclude generated modules + # (proto-google-*, grpc-google-*, *-bom) to skip formatting generated code + exclusions="" + # Find generated sub-modules and parent/aggregator POMs to exclude + # - proto-google-*, grpc-google-* : generated protobuf/gRPC stubs + # - *-bom : BOM modules with no source code + # - java-* at depth 1 : per-library aggregator parent POMs + # - google-cloud-pom-parent, google-cloud-jar-parent, gapic-libraries-bom : repo-level parents + for dir in $(find . -maxdepth 1 -type d -name "java-*" | sort; \ + find . -maxdepth 2 -type d \( -name "proto-google-*" -o -name "grpc-google-*" -o -name "*-bom" -o -name "google-cloud-pom-parent" -o -name "google-cloud-jar-parent" \) | sort); do + # Strip leading ./ + dir="${dir#./}" + if [ -n "${exclusions}" ]; then + exclusions="${exclusions},!${dir}" + else + exclusions="!${dir}" + fi + done + + if [ -n "${exclusions}" ]; then + MODULE_FILTER="-pl ${exclusions}" + echo "Excluding generated modules from formatting" + fi fi - + mvn -B -ntp \ - -T 1.5C \ + -T 1C \ + ${MODULE_FILTER} \ com.spotify.fmt:fmt-maven-plugin:check - mvn -B -ntp checkstyle:check@checkstyle + mvn -B -ntp ${MODULE_FILTER} checkstyle:check@checkstyle if [[ -n "${BUILD_SUBDIR}" ]] then From 12008542b4c5352f21bddd9535330590c678e1e3 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 26 Feb 2026 01:07:12 -0500 Subject: [PATCH 2/9] ci: Optimize integration tests to run only for impacted modules --- .kokoro/build.sh | 19 ++++++++++++------- .kokoro/common.sh | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 695b85587223..8e6b934c43b6 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -71,13 +71,18 @@ case ${JOB_TYPE} in if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then echo "Skipping integration tests as this is Release Please SNAPSHOT pull request." elif [[ ${#modified_module_list[@]} -gt 0 ]]; then - module_list=$( - IFS=, - echo "${modified_module_list[*]}" - ) - setup_cloud "$module_list" - install_modules "$module_list" - run_integration_tests "$module_list" + filter_modules_with_integration_tests + if [[ ${#filtered_it_module_list[@]} -eq 0 ]]; then + echo "No modified modules contain integration tests. Skipping." + else + module_list=$( + IFS=, + echo "${filtered_it_module_list[*]}" + ) + setup_cloud "$module_list" + install_modules "$module_list" + run_integration_tests "$module_list" + fi else echo "No Integration Tests to run" fi diff --git a/.kokoro/common.sh b/.kokoro/common.sh index ba6ca87a3fee..17ac17052378 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -66,16 +66,20 @@ function retry_with_backoff { # comma-delimited list of /. function parse_submodules() { submodules_array=() - mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules -pl "$1") - if mvn_submodules=$(grep '<.*>.*' <<< "$mvn_submodules"); then - mvn_submodules=$(sed -e 's/<.*>\(.*\)<\/.*>/\1/g' <<< "$mvn_submodules") - for submodule in $mvn_submodules; do - # Each entry = / - submodules_array+=("$1/${submodule}"); - done + if [ -f "$1/pom.xml" ]; then + local modules + modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/') + if [ -n "$modules" ]; then + for submodule in $modules; do + # Each entry = / + submodules_array+=("$1/${submodule}") + done + else + # If this module contains no submodules, select the module itself. + submodules_array+=("$1") + fi else - # If this module contains no submodules, select the module itself. - submodules_array+=("$1"); + submodules_array+=("$1") fi # Convert from array to comma-delimited string @@ -216,6 +220,21 @@ function generate_modified_modules_list() { fi } +# Filters the modified_module_list to only include modules that contain +# integration test files (matching IT*.java or *IT.java in src/test/java). +# Sets filtered_it_module_list as a bash array. +function filter_modules_with_integration_tests() { + filtered_it_module_list=() + for module in "${modified_module_list[@]}"; do + if find "$module" -path '*/src/test/java/*' \( -name 'IT*.java' -o -name '*IT.java' \) -print -quit 2>/dev/null | grep -q .; then + filtered_it_module_list+=("$module") + fi + done + printf "Modules with integration tests:\n" + printf " %s\n" "${filtered_it_module_list[@]}" + echo "Found ${#filtered_it_module_list[@]} modules with integration tests (out of ${#modified_module_list[@]} modified modules)" +} + function run_integration_tests() { printf "Running integration tests for modules:\n%s\n" "$1" parse_all_submodules "$1" From 69a0348ce9361af0d01f2de238c653415c7b2e6a Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 14:22:14 -0500 Subject: [PATCH 3/9] chore: Add comments to explain the logic --- .kokoro/common.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 17ac17052378..4538faee33b3 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -68,6 +68,9 @@ function parse_submodules() { submodules_array=() if [ -f "$1/pom.xml" ]; then local modules + + # Use grep to find the modules in the aggregator pom file + # Faster than invoking mvn help:evaluate to list all the project modules modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/') if [ -n "$modules" ]; then for submodule in $modules; do @@ -222,10 +225,14 @@ function generate_modified_modules_list() { # Filters the modified_module_list to only include modules that contain # integration test files (matching IT*.java or *IT.java in src/test/java). -# Sets filtered_it_module_list as a bash array. +# Not all modules will have ITs written and there is not need to test +# modules without ITs. function filter_modules_with_integration_tests() { filtered_it_module_list=() for module in "${modified_module_list[@]}"; do + # 1. Search for files in the Java test directory (*/src/test/java/*) + # 2. Filter for ITs that match the typical file name (IT prefix or suffix) + # 3. Stop searching when a single file match has been found if find "$module" -path '*/src/test/java/*' \( -name 'IT*.java' -o -name '*IT.java' \) -print -quit 2>/dev/null | grep -q .; then filtered_it_module_list+=("$module") fi From e1481d0e07710b1284929ffdf5e5ebd22c43670c Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 14:41:34 -0500 Subject: [PATCH 4/9] chore: Trigger all ITs to run --- google-cloud-pom-parent/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-pom-parent/pom.xml b/google-cloud-pom-parent/pom.xml index cf9c14884cbd..a02d903670df 100644 --- a/google-cloud-pom-parent/pom.xml +++ b/google-cloud-pom-parent/pom.xml @@ -11,6 +11,7 @@ https://github.com/googleapis/google-cloud-java The top-level parent for all modules in the repository. + com.google.cloud From a1616a7b0788965506597c70d363e80b70a136d7 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 14:41:34 -0500 Subject: [PATCH 5/9] chore: Trigger all ITs to run --- .kokoro/common.sh | 2 +- google-cloud-pom-parent/pom.xml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 4538faee33b3..29bbb5cf126c 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -71,7 +71,7 @@ function parse_submodules() { # Use grep to find the modules in the aggregator pom file # Faster than invoking mvn help:evaluate to list all the project modules - modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/') + modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/' || true) if [ -n "$modules" ]; then for submodule in $modules; do # Each entry = / diff --git a/google-cloud-pom-parent/pom.xml b/google-cloud-pom-parent/pom.xml index cf9c14884cbd..a02d903670df 100644 --- a/google-cloud-pom-parent/pom.xml +++ b/google-cloud-pom-parent/pom.xml @@ -11,6 +11,7 @@ https://github.com/googleapis/google-cloud-java The top-level parent for all modules in the repository. + com.google.cloud From ebec6ef4a62c0a9b5034cad8012c5494ffb94121 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 15:43:59 -0500 Subject: [PATCH 6/9] chore: Add comment for cases where there is no sub-module --- .kokoro/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 29bbb5cf126c..b957af71fa9a 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -71,6 +71,7 @@ function parse_submodules() { # Use grep to find the modules in the aggregator pom file # Faster than invoking mvn help:evaluate to list all the project modules + # Not all modules will have sub-modules (use `|| true`) to prevent the script from failing. modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/' || true) if [ -n "$modules" ]; then for submodule in $modules; do From a1a3a6fe31d30deac7f556a141b5803c48bf43c5 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 15:58:58 -0500 Subject: [PATCH 7/9] chore: Remove unused samples profile in BQStorage --- java-bigquerystorage/pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/java-bigquerystorage/pom.xml b/java-bigquerystorage/pom.xml index 735302d3649a..b01d048284a2 100644 --- a/java-bigquerystorage/pom.xml +++ b/java-bigquerystorage/pom.xml @@ -202,14 +202,4 @@ google-cloud-bigquerystorage-bom - - - include-samples - - samples - tutorials - - - - From e9278ada8581f0a537cf168aacd5f727612b48d0 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 15:58:58 -0500 Subject: [PATCH 8/9] chore: Remove unused samples profile in BQStorage --- .kokoro/common.sh | 51 +++++++++++++++++++++++++++++---- google-cloud-pom-parent/pom.xml | 1 - java-bigquerystorage/pom.xml | 10 ------- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index b957af71fa9a..28d4f2e3a762 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -62,6 +62,46 @@ function retry_with_backoff { return $exit_code } +# Helper function to reliably extract the text between tags strictly +# within the default block, natively ignoring . +# Uses a pure Bash loop to avoid spawning slower external processes like awk or sed, +# and naturally survives single-module components without throwing exit signals. +function extract_pom_modules() { + local pom_file="$1" + local modules_list="" + local in_profiles=false + local in_modules=false + + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" == *""* ]]; then + in_profiles=true + elif [[ "$line" == *""* ]]; then + in_profiles=false + elif [[ "$line" == *""* ]] && [ "$in_profiles" = false ]; then + in_modules=true + elif [[ "$line" == *""* ]] && [ "$in_profiles" = false ]; then + in_modules=false + break + elif [ "$in_modules" = true ] && [[ "$line" == *""* ]]; then + # Extract text between tags + local module="${line#*}" + module="${module%*}" + + # Trim whitespace natively + module="${module#"${module%%[![:space:]]*}"}" + module="${module%"${module##*[![:space:]]}"}" + + if [ -z "$modules_list" ]; then + modules_list="$module" + else + modules_list="${modules_list} ${module}" + fi + fi + done < "$pom_file" + + echo "$modules_list" +} + # Given a folder containing a maven multi-module, assign the variable 'submodules' to a # comma-delimited list of /. function parse_submodules() { @@ -69,10 +109,10 @@ function parse_submodules() { if [ -f "$1/pom.xml" ]; then local modules - # Use grep to find the modules in the aggregator pom file - # Faster than invoking mvn help:evaluate to list all the project modules - # Not all modules will have sub-modules (use `|| true`) to prevent the script from failing. - modules=$(grep '' "$1/pom.xml" | sed 's/.*\(.*\)<\/module>.*/\1/' || true) + # Use pure Bash extraction to find the modules in the aggregator pom file. + # Faster than invoking mvn help:evaluate to list all the project modules, + # cleanly ignores optional , and gracefully skips flat POMs. + modules=$(extract_pom_modules "$1/pom.xml") if [ -n "$modules" ]; then for submodule in $modules; do # Each entry = / @@ -83,7 +123,8 @@ function parse_submodules() { submodules_array+=("$1") fi else - submodules_array+=("$1") + echo "Module does not have a pom.xml file: $1" + exit 1 fi # Convert from array to comma-delimited string diff --git a/google-cloud-pom-parent/pom.xml b/google-cloud-pom-parent/pom.xml index a02d903670df..cf9c14884cbd 100644 --- a/google-cloud-pom-parent/pom.xml +++ b/google-cloud-pom-parent/pom.xml @@ -11,7 +11,6 @@ https://github.com/googleapis/google-cloud-java The top-level parent for all modules in the repository. - com.google.cloud diff --git a/java-bigquerystorage/pom.xml b/java-bigquerystorage/pom.xml index 735302d3649a..b01d048284a2 100644 --- a/java-bigquerystorage/pom.xml +++ b/java-bigquerystorage/pom.xml @@ -202,14 +202,4 @@ google-cloud-bigquerystorage-bom - - - include-samples - - samples - tutorials - - - - From c53923405c4bf399eb455a2457b93dbe8b8d1673 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 27 Feb 2026 19:39:32 -0500 Subject: [PATCH 9/9] chore: Test optimized integration CI --- google-cloud-pom-parent/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-pom-parent/pom.xml b/google-cloud-pom-parent/pom.xml index cf9c14884cbd..a02d903670df 100644 --- a/google-cloud-pom-parent/pom.xml +++ b/google-cloud-pom-parent/pom.xml @@ -11,6 +11,7 @@ https://github.com/googleapis/google-cloud-java The top-level parent for all modules in the repository. + com.google.cloud