chore(storage): optimize zonal system tests CloudBuild and make concurrency-safe (googleapis#17171)

This PR optimizes the zonal system tests CloudBuild configuration
(`zb-system-tests-cloudbuild.yaml`) and its execution script
(`run_zonal_tests.sh`) to make the test execution robust, fast, and
concurrency-safe.

### Proposed Changes

- **Direct Workspace Packaging & SCP Transfer**: Replaces git-cloning
and fetching inside the GCE VM with packaging the local
`/workspace/packages/google-cloud-storage` directory on the Cloud Build
runner and copying the tarball via `scp` to the VM. This eliminates Git
and repo-cloning requirements on the GCE VM, improving speed and
reliability.
- **OS Login TTL Keys for Concurrency Safety**: Replaces the aggressive,
concurrency-breaking `cleanup-old-keys` step (which deleted all OS Login
SSH keys for the project and interfered with concurrent builds) by
registering the generated SSH key with OS Login using a 1-hour
Time-To-Live (TTL). GCP automatically expires old keys, preventing key
accumulation without affecting other concurrent builds.
- **Robust Variable Handling & Safety**: 
- Safely quotes all instances of `"${_VM_NAME}"` in `gcloud compute`
ssh, scp, and delete commands.
- Adds default empty substitutions (`_PR_NUMBER`,
`_CROSS_REGION_BUCKET`, `_ZONAL_BUCKET`, `_ZONAL_VM_SERVICE_ACCOUNT`) to
support manual builds.
- Exports `CROSS_REGION_BUCKET` with a fallback default in the test
runner script.
- Enables `dynamicSubstitutions: true` under CloudBuild options to
support active evaluation of build variables.

### Verification Results

Both verification scenarios were executed and confirmed:
1. **Full Zonal Tests Run**: CloudBuild successfully created the GCE VM,
SCP'd the packaged code, executed the zonal system tests inside the VM,
and cleanly deleted the VM upon completion.
2. **SSH Key Lifecycle**: Verified the generated SSH key is registered
with a 1-hour TTL and successfully cleaned up, avoiding OS Login profile
accumulation.



fixes - **b/514186407**

TAG=agy
CONV=16d817fd-2422-432e-816b-bf159b381df2

Author: chandra-siri

packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh

@@ -1,16 +1,9 @@
 
 set -euxo pipefail
-echo '--- Installing git and cloning repository on VM ---'
-sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv
-
-# Clone the repository and checkout the specific commit from the build trigger.
-git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/google-cloud-python.git
-cd google-cloud-python
-git sparse-checkout set packages/google-cloud-storage
-git fetch origin "refs/pull/${_PR_NUMBER}/head"
-git checkout ${COMMIT_SHA}
-cd packages/google-cloud-storage
-
+echo '--- Extracting source code tarball on VM ---'
+sudo apt-get update && sudo apt-get install -y python3-pip python3-venv
+tar -xzf google-cloud-storage.tar.gz
+cd google-cloud-storage
 
 echo '--- Installing Python and dependencies on VM ---'
 python3 -m venv env
@@ -24,6 +17,7 @@ pip install -e .
 
 echo '--- Setting up environment variables on VM ---'
 export ZONAL_BUCKET=${_ZONAL_BUCKET}
+export CROSS_REGION_BUCKET=${CROSS_REGION_BUCKET:-}
 export RUN_ZONAL_SYSTEM_TESTS=True
 export GCE_METADATA_MTLS_MODE=None
 CURRENT_ULIMIT=$(ulimit -n)

packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml

@@ -1,9 +1,11 @@
 substitutions:
-  _REGION: "us-central1"
   _ZONE: "us-central1-a"
   _SHORT_BUILD_ID: ${BUILD_ID:0:8}
   _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}"
   _ULIMIT: "10000" # 10k, for gRPC bidi streams
+  _CROSS_REGION_BUCKET: ""
+  _ZONAL_BUCKET: ""
+  _ZONAL_VM_SERVICE_ACCOUNT: ""
 
 
 
@@ -21,44 +23,23 @@ steps:
         ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb
         # Save the public key content to a file for the cleanup step
         cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub
+        # Register the SSH key with OS Login with a 1 hour TTL to prevent accumulation
+        gcloud compute os-login ssh-keys add \
+          --key-file=/workspace/.ssh/google_compute_engine.pub \
+          --ttl=1h
     waitFor: ["-"]
 
+  # Step 1: Package google-cloud-storage directory for direct transfer to VM
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "cleanup-old-keys"
+    id: "package-code"
     entrypoint: "bash"
     args:
       - "-c"
       - |
-        #!/bin/bash
-        set -e
-
-        echo "Fetching OS Login SSH keys..."
-        echo "Removing all keys."
-        echo "---------------------------------------------------------------------"
-
-        FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \
-          --format="value(fingerprint)")
-
-        echo "Keys to delete: $$FINGERPRINTS_TO_DELETE"
-
-        if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then
-          echo "No keys found to delete. Nothing to do."
-          exit 0
-        fi
-
-        while IFS= read -r FINGERPRINT; do
-          if [ -n "$$FINGERPRINT" ]; then
-            echo "Deleting key with fingerprint: $$FINGERPRINT"
-            gcloud compute os-login ssh-keys remove \
-              --key="$$FINGERPRINT" \
-              --quiet || true
-          fi
-        done <<< "$$FINGERPRINTS_TO_DELETE"
-
-        echo "---------------------------------------------------------------------"
-        echo "Cleanup complete."
+        tar --exclude='.nox' --exclude='venv_314' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage
+    waitFor: ["-"]
 
-  # Step 1 Create a GCE VM to run the tests.
+  # Step 2 Create a GCE VM to run the tests.
   # The VM is created in the same zone as the buckets to test rapid storage features.
   # It's given the 'cloud-platform' scope to allow it to access GCS and other services.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
@@ -79,7 +60,7 @@ steps:
       - "--metadata=enable-oslogin=TRUE"
     waitFor: ["-"]
 
-  # Step 2: Run the integration tests inside the newly created VM and cleanup.
+  # Step 3: Run the integration tests inside the newly created VM and cleanup.
   # This step uses 'gcloud compute ssh' to execute a remote script.
   # The VM is deleted after tests are run, regardless of success.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
@@ -91,19 +72,19 @@ steps:
         set -e
         # Wait for the VM to be fully initialized and SSH to be ready.
         for i in {1..10}; do
-          if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
+          if gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
             break
           fi
           echo "Waiting for VM to become available... (attempt $i/10)"
           sleep 15
         done
-        # copy the script to the VM
-        gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
+        # copy the script and the package tarball to the VM
+        gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh /workspace/google-cloud-storage.tar.gz "${_VM_NAME}":~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
 
         # Execute the script on the VM via SSH.
         # Capture the exit code to ensure cleanup happens before the build fails.
         set +e
-        gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh"
+        gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} bash run_zonal_tests.sh"
         EXIT_CODE=$?
         set -e
 
@@ -115,7 +96,7 @@ steps:
     waitFor:
       - "create-vm"
       - "generate-ssh-key"
-      - "cleanup-old-keys"
+      - "package-code"
 
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-ssh-key"
@@ -133,5 +114,6 @@ timeout: "3600s" # 60 minutes
 
 options:
   logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true
   pool:
     name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"