Skip to content

chore(ci): optimize kokoro system tests with concurrency #17444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ohmayr wants to merge 3 commits into
base: main
Choose a base branch
from
Open

chore(ci): optimize kokoro system tests with concurrency #17444

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f00dae
chore(ci): optimize kokoro system tests with concurrency
ohmayr Jun 12, 2026
c20eedc
fix filename issue
ohmayr Jun 12, 2026
5ffa37d
chore(ci): optimize parallel system test execution and fix race condi…
chalmerlowe Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions .kokoro/system.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,24 @@ RETVAL=0

pwd

reap_parallel_results() {
local retval=0
for failed in .logs/*.failed; do
if [ -f "$failed" ]; then
local pkg=$(basename "$failed" .failed)
echo "--- FAILED: $pkg ---"
cat ".logs/$pkg.log"
retval=1
fi
done
return $retval
}


run_package_test() {
local package_name=$1
local package_path="packages/${package_name}"

# Declare local overrides to prevent bleeding into the next loop iteration
local PROJECT_ID
local GOOGLE_APPLICATION_CREDENTIALS
Expand Down Expand Up @@ -77,7 +91,8 @@ run_package_test() {
export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"

gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
gcloud config set project "$PROJECT_ID"
export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}"


# Run the actual test
pushd "${package_path}" > /dev/null
Expand All @@ -86,7 +101,7 @@ run_package_test() {
local res=$?
set -e
popd > /dev/null

return $res
}

Expand Down Expand Up @@ -140,10 +155,22 @@ for path in `find 'packages' \
set -e

if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
# Call the function - its internal exports won't affect the next loop
run_package_test "$package_name" || RETVAL=$?
PACKAGES_TO_TEST="${PACKAGES_TO_TEST} ${package_name}"
else
echo "No changes in ${package_name} and not a continuous build, skipping."
fi
done

if [ -n "$PACKAGES_TO_TEST" ]; then
mkdir -p .logs
export -f run_package_test
export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR

CONCURRENCY=${MAX_PARALLEL_TESTS:-8}

printf '%s\n' $PACKAGES_TO_TEST | xargs -P "$CONCURRENCY" -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'

reap_parallel_results || RETVAL=1
fi

exit ${RETVAL}
Loading