From 4f00dae56bbd500c8239449e1e693684348fd183 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Fri, 12 Jun 2026 08:11:50 +0000 Subject: [PATCH 1/3] chore(ci): optimize kokoro system tests with concurrency --- .kokoro/system.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 469d0e81c7fa..491767c7dfe2 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -140,10 +140,26 @@ for path in `find 'packages' \ set -e if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? + PACKAGES_TO_TEST="${PACKAGES_TO_TEST} ${package_name}" else echo "No changes in ${package_name} and not a continuous build, skipping." fi done + +if [ -n "$PACKAGES_TO_TEST" ]; then + mkdir -p .logs + export -f run_package_test + export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR + + echo "$PACKAGES_TO_TEST" | xargs -n 1 -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"' + + for failed in .logs/*.failed; do + if [ -f "$failed" ]; then + echo "--- FAILED: ${failed%.failed} ---" + cat "${failed%.failed}.log" + RETVAL=1 + fi + done +fi + exit ${RETVAL} From c20eedc652937a4d33bf6f3864982d089209b488 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Fri, 12 Jun 2026 08:28:50 +0000 Subject: [PATCH 2/3] fix filename issue --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 491767c7dfe2..9cd4f5f45298 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -151,7 +151,7 @@ if [ -n "$PACKAGES_TO_TEST" ]; then export -f run_package_test export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR - echo "$PACKAGES_TO_TEST" | xargs -n 1 -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"' + echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"' for failed in .logs/*.failed; do if [ -f "$failed" ]; then From 5ffa37d61ccaae2f6c511b5a9afa2b99aa45b362 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 16 Jul 2026 13:20:26 -0400 Subject: [PATCH 3/3] chore(ci): optimize parallel system test execution and fix race conditions --- .kokoro/system.sh | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 9cd4f5f45298..aeb290159d6e 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -35,10 +35,24 @@ RETVAL=0 pwd +reap_parallel_results() { + local retval=0 + for failed in .logs/*.failed; do + if [ -f "$failed" ]; then + local pkg=$(basename "$failed" .failed) + echo "--- FAILED: $pkg ---" + cat ".logs/$pkg.log" + retval=1 + fi + done + return $retval +} + + run_package_test() { local package_name=$1 local package_path="packages/${package_name}" - + # Declare local overrides to prevent bleeding into the next loop iteration local PROJECT_ID local GOOGLE_APPLICATION_CREDENTIALS @@ -77,7 +91,8 @@ run_package_test() { export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" - gcloud config set project "$PROJECT_ID" + export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}" + # Run the actual test pushd "${package_path}" > /dev/null @@ -86,7 +101,7 @@ run_package_test() { local res=$? set -e popd > /dev/null - + return $res } @@ -150,16 +165,12 @@ if [ -n "$PACKAGES_TO_TEST" ]; then mkdir -p .logs export -f run_package_test export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR - - echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"' - - for failed in .logs/*.failed; do - if [ -f "$failed" ]; then - echo "--- FAILED: ${failed%.failed} ---" - cat "${failed%.failed}.log" - RETVAL=1 - fi - done + + CONCURRENCY=${MAX_PARALLEL_TESTS:-8} + + printf '%s\n' $PACKAGES_TO_TEST | xargs -P "$CONCURRENCY" -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"' + + reap_parallel_results || RETVAL=1 fi exit ${RETVAL}