feat: Use bulk instance creation to find available zone (GoogleCloudPlatform#5000)

Author: simrankaurb

tools/cloud-build/find_available_zone.sh

@@ -17,6 +17,7 @@ BUILD_ID_SHORT="${BUILD_ID:0:6}"
 PROVISIONING_MODEL="SPOT"
 TERMINATION_ACTION="DELETE"
 FULL_INSTANCE_PREFIX="${INSTANCE_PREFIX}-${BUILD_ID_SHORT}-"
+MIN_NODES=2 # Define minimum number of nodes required
 
 generate_instance_names() {
 	local prefix=$1
@@ -71,28 +72,55 @@ SUCCESS=false
 # Loop through all zones to find capacity
 for ZONE in "${ZONES_ARRAY[@]}"; do
 	readarray -t INSTANCE_NAMES_ARRAY < <(generate_instance_names "${FULL_INSTANCE_PREFIX}" "${NUM_NODES}")
+	# Join instance names with commas for the --predefined-names flag
+	instance_names_str=$(
+		IFS=,
+		echo "${INSTANCE_NAMES_ARRAY[*]}"
+	)
 
 	declare -a GCLOUD_CMD
 	GCLOUD_CMD=(
-		gcloud compute instances create "${INSTANCE_NAMES_ARRAY[@]}"
+		gcloud compute instances bulk create
+		--predefined-names="${instance_names_str}"
 		--project="${PROJECT_ID}"
 		--zone="${ZONE}"
 		--machine-type="${MACHINE_TYPE}"
 		--provisioning-model="${PROVISIONING_MODEL}"
 		--instance-termination-action="${TERMINATION_ACTION}"
 		--no-address
 		--quiet
+		--min-count="${MIN_NODES}"
 	)
 	if CREATE_OUTPUT=$("${GCLOUD_CMD[@]}" 2>&1); then
+		instance_list_output=$(gcloud compute instances list --project="${PROJECT_ID}" --zones="${ZONE}" \
+			--filter="name ~ ^${FULL_INSTANCE_PREFIX}" --format='value(name)')
+
+		if [[ $? -eq 0 && -n "${instance_list_output}" ]]; then
+			readarray -t created_instances < <(echo "${instance_list_output}")
+			NUM_CREATED=$((${#created_instances[@]}))
+			echo "${ZONE} ${NUM_CREATED}"
+			cleanup_instances "${PROJECT_ID}" "${ZONE}" "${FULL_INSTANCE_PREFIX}"
+
+			if [[ "${NUM_CREATED}" -ge "${MIN_NODES}" ]]; then
+				SELECTED_ZONE="${ZONE}"
+				SUCCESS=true
+				break
+			else
+				echo "ERROR: Bulk create succeeded but only ${NUM_CREATED} instances found, less than min_count ${MIN_NODES} in ${ZONE}." >&2
+			fi
+		else
+			echo "ERROR: Bulk create command succeeded in ${ZONE}, but failed to list the created instances or none found." >&2
+			cleanup_instances "${PROJECT_ID}" "${ZONE}" "${FULL_INSTANCE_PREFIX}"
+		fi
+	else
+		# Command failed
+		if [[ "${CREATE_OUTPUT}" != *"INSUFFICIENT_CAPACITY"* &&
+			"${CREATE_OUTPUT}" != *"ZONE_RESOURCE_POOL_EXHAUSTED"* ]]; then
+			echo "ERROR: Unexpected error during bulk create in ${ZONE}: ${CREATE_OUTPUT}" >&2
+		fi
+		# bulk create with --min-count should handle rollback on failure, but cleanup just in case.
 		cleanup_instances "${PROJECT_ID}" "${ZONE}" "${FULL_INSTANCE_PREFIX}"
-		SELECTED_ZONE="${ZONE}"
-		SUCCESS=true
-		break
-	elif [[ "${CREATE_OUTPUT}" != *"INSUFFICIENT_CAPACITY"* &&
-		"${CREATE_OUTPUT}" != *"ZONE_RESOURCE_POOL_EXHAUSTED"* ]]; then
-		echo "ERROR: Unexpected error ${CREATE_OUTPUT}" >&2
 	fi
-	cleanup_instances "${PROJECT_ID}" "${ZONE}" "${FULL_INSTANCE_PREFIX}"
 done
 
 if [[ "${SUCCESS}" == "true" ]]; then