chore(ci): optimize parallel system test execution and fix race conditions

Author: chalmerlowe

.kokoro/system.sh

@@ -35,10 +35,24 @@ RETVAL=0
 
 pwd
 
+reap_parallel_results() {
+  local retval=0
+  for failed in .logs/*.failed; do
+    if [ -f "$failed" ]; then
+      local pkg=$(basename "$failed" .failed)
+      echo "--- FAILED: $pkg ---"
+      cat ".logs/$pkg.log"
+      retval=1
+    fi
+  done
+  return $retval
+}
+
+
 run_package_test() {
   local package_name=$1
   local package_path="packages/${package_name}"
-  
+
   # Declare local overrides to prevent bleeding into the next loop iteration
   local PROJECT_ID
   local GOOGLE_APPLICATION_CREDENTIALS
@@ -77,7 +91,8 @@ run_package_test() {
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
 
   gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
-  gcloud config set project "$PROJECT_ID"
+  export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}"
+
 
   # Run the actual test
   pushd "${package_path}" > /dev/null
@@ -86,7 +101,7 @@ run_package_test() {
   local res=$?
   set -e
   popd > /dev/null
-  
+
   return $res
 }
 
@@ -150,16 +165,12 @@ if [ -n "$PACKAGES_TO_TEST" ]; then
   mkdir -p .logs
   export -f run_package_test
   export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
-  
-  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
-  
-  for failed in .logs/*.failed; do
-    if [ -f "$failed" ]; then
-      echo "--- FAILED: ${failed%.failed} ---"
-      cat "${failed%.failed}.log"
-      RETVAL=1
-    fi
-  done
+
+  CONCURRENCY=${MAX_PARALLEL_TESTS:-8}
+
+  printf '%s\n' $PACKAGES_TO_TEST | xargs -P "$CONCURRENCY" -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
+
+  reap_parallel_results || RETVAL=1
 fi
 
 exit ${RETVAL}