From ea6076572a3d0dc4b0a4a24579afdf46758160e5 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 17:04:54 -0500 Subject: [PATCH 1/4] ci: Speed up Source Compatibility checks by running specific modules --- .../downstream-protobuf-source-compatibility.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index c7caeb724c..f48e6d771d 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -36,11 +36,26 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn clean test -B -V -ntp \ + if [ "${repo}" == "google-cloud-java" ]; then + # For google-cloud-java, only test specific handwritten libraries + mvn clean test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ + # -am command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + # Specify the nested `google-cloud-*` path (except for grafeas as it doesn't have one) because maven + # will only build the parent folder and ignore all the sub-modules inside + -pl java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager -am \ -T 1C + else + mvn clean test -B -V -ntp \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ + -T 1C + fi + popd done \ No newline at end of file From 807f6b9758e90655b587c66fbd15f6adc32698d7 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 17:16:11 -0500 Subject: [PATCH 2/4] chore: Move comment above command --- .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index f48e6d771d..495ade573d 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -38,14 +38,14 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Run unit tests to help check for any behavior differences (dependant on coverage) if [ "${repo}" == "google-cloud-java" ]; then # For google-cloud-java, only test specific handwritten libraries + # -am command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + # Specify the nested `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl + # will only build the specified folder (i.e. parent folder) and ignore all the sub-modules inside mvn clean test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ - # -am command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) - # Specify the nested `google-cloud-*` path (except for grafeas as it doesn't have one) because maven - # will only build the parent folder and ignore all the sub-modules inside -pl java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager -am \ -T 1C else From c3cdf5ce442d2ed57330ee0aaaed659f942cf986 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 13 Mar 2025 17:44:11 -0400 Subject: [PATCH 3/4] ci: Fix the mvn command --- .kokoro/nightly/common.sh | 6 ++++++ ...ownstream-protobuf-binary-compatibility.sh | 12 +++++++++-- ...ownstream-protobuf-source-compatibility.sh | 21 +++++++++---------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.kokoro/nightly/common.sh b/.kokoro/nightly/common.sh index ee30ec3c74..589ee00bfb 100644 --- a/.kokoro/nightly/common.sh +++ b/.kokoro/nightly/common.sh @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# For google-cloud-java, only test specific handwritten libraries included in the monorepo. This is to +# help speed up the execution as building the entire repo is an expensive operation. Specify the nested +# `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl will only build the +# specified folder (i.e. parent folder) and ignore all the related sub-modules inside +google_cloud_java_handwritten_maven_args="java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager,java-translate/google-cloud-translate" + # Checks that the protobuf compatibility scripts provide non-empty input function validate_protobuf_compatibility_script_inputs { # Comma-delimited list of repos to test diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 46dd4e967c..85231390d5 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -76,8 +76,16 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform testing on main (with latest changes). Shallow copy as history is not important git clone "https://github.com/googleapis/${repo}.git" --depth=1 pushd "${repo}" - # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) - mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + + if [ "${repo}" == "google-cloud-java" ]; then + mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip \ + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + -pl "${google_cloud_java_handwritten_maven_args}" -am + else + # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip + fi + linkage_checker_arguments="" build_program_arguments "${repo}" diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index dde11d05e6..12cff404ca 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -31,22 +31,21 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) if [ "${repo}" == "google-cloud-java" ]; then - # For google-cloud-java, only test specific handwritten libraries - # -am command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) - # Specify the nested `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl - # will only build the specified folder (i.e. parent folder) and ignore all the sub-modules inside mvn clean test -B -V -ntp \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ + -Dclirr.skip \ + -Denforcer.skip \ + -Dmaven.javadoc.skip \ + -Denforcer.skip \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ - -pl java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager -am \ + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + -pl "${google_cloud_java_handwritten_maven_args}" -am \ -T 1C else mvn clean test -B -V -ntp \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ + -Dclirr.skip \ + -Denforcer.skip \ + -Dmaven.javadoc.skip \ + -Denforcer.skip \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ -T 1C fi From 2432e3e9004e4eaf9428ecefbad3b8f5b36e3c08 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 13 Mar 2025 17:57:47 -0400 Subject: [PATCH 4/4] ci: Fix the mvn command --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 2 +- .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 85231390d5..55f75496cc 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -78,8 +78,8 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma pushd "${repo}" if [ "${repo}" == "google-cloud-java" ]; then + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip \ - # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) -pl "${google_cloud_java_handwritten_maven_args}" -am else # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 12cff404ca..ce377feedd 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -31,13 +31,13 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) if [ "${repo}" == "google-cloud-java" ]; then + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) mvn clean test -B -V -ntp \ -Dclirr.skip \ -Denforcer.skip \ -Dmaven.javadoc.skip \ -Denforcer.skip \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ - # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) -pl "${google_cloud_java_handwritten_maven_args}" -am \ -T 1C else