Fix G4 GKE Rapid Version & Add Onboard On-Spot Test Support For Same (#5591)

Co-authored-by: laveen <laveenekka@google.com>

Author: LAVEEN

examples/gke-g4/gke-g4.yaml

@@ -143,7 +143,7 @@ deployment_groups:
         ))
       # Cluster versions cannot be updated through the toolkit after creation
       # Please manage cluster version from the Google Cloud Console directly
-      version_prefix: "1.35.2-gke.1485000"
+      version_prefix: "1.35.3-gke.1522000"
       release_channel: RAPID
       maintenance_exclusions:
       - name: no-minor-or-node-upgrades-indefinite

tools/cloud-build/daily-tests/builds/gke-g4-onspot.yaml

@@ -0,0 +1,109 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+tags:
+- gke
+- m.gke-cluster
+- m.gke-node-pool
+- m.service-account
+- m.vpc
+- m.kubectl-apply
+- m.gke-job-template
+
+substitutions:
+  _TEST_PREFIX: "" # Default to no prefix
+
+timeout: 14400s  # 4hr
+steps:
+# While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
+- id: check_for_running_build
+  name: gcr.io/cloud-builders/gcloud
+  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-g4-onspot.yaml"
+
+- id: gke-g4-onspot
+  name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
+  entrypoint: /bin/bash
+  env:
+  - "ANSIBLE_HOST_KEY_CHECKING=false"
+  - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
+  - "PROJECT_ID=$PROJECT_ID"
+  - "NUM_NODES=2"
+  - "MACHINE_TYPE=g4-standard-48"
+  - "INSTANCE_PREFIX=g4spgke"
+  - "BUILD_ID=$BUILD_ID"
+  - "OPTIONS_GCS_PATH=gs://hpc-ctk1357/g4options.txt"
+  - "ENABLE_SPOT_FALLBACK=true"
+  args:
+  - -c
+  - |
+    set -e -u -o pipefail
+    echo "Sourcing find_available_zone.sh to determine zone."
+    source /workspace/tools/cloud-build/find_available_zone.sh
+    if [ -z "$${ZONE:-}" ]; then
+      echo "ERROR: ZONE not found" >&2
+      exit 1
+    fi
+    set -x -e
+    cd /workspace
+    if [ "${_TEST_PREFIX}" == "daily-" ]; then
+        gsutil cp gs://$${GCLUSTER_GCS_PATH}/latest/gcluster-bundle.zip .
+        unzip -o gcluster-bundle.zip
+        # Grant execution permissions to the binary
+        chmod +x gcluster
+    else
+        make
+    fi
+    REGION="$${ZONE%-*}"
+    BUILD_ID_FULL=$BUILD_ID
+    BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6}
+    EXAMPLE_BP=examples/gke-g4/gke-g4.yaml
+    # adding vm to act as remote node
+    echo '  - id: remote-node'                           >> $${EXAMPLE_BP}
+    echo '    source: modules/compute/vm-instance'       >> $${EXAMPLE_BP}
+    echo '    use: [gke-g4-net-0]'                        >> $${EXAMPLE_BP}
+    echo '    settings:'                                 >> $${EXAMPLE_BP}
+    echo '      machine_type: e2-standard-2'             >> $${EXAMPLE_BP}
+    echo '      name_prefix: remote-node'                >> $${EXAMPLE_BP}
+    echo '      add_deployment_name_before_prefix: true' >> $${EXAMPLE_BP}
+
+    ENABLE_SPOT="true"
+    GKE_VARS_FILE="tools/cloud-build/daily-tests/tests/gke-g4-onspot.yml"
+    if [[ "$$PROVISIONING_MODEL" == "STANDARD" ]]; then
+      ENABLE_SPOT="false"
+      sed -i '/instance_labels:/,+1d' "$${GKE_VARS_FILE}"
+      sed -i '/enable_spot: true/d' "$${GKE_VARS_FILE}"
+    else
+      echo "INFO: Using $${GKE_VARS_FILE} as it is for SPOT provisioning."
+    fi
+
+    # Inject spot setting after auto_upgrade: true
+    sed -i 's/auto_upgrade: true/auto_upgrade: true\n      spot: '"$$ENABLE_SPOT"'/' $${EXAMPLE_BP}
+    sed -i 's/name: run-nvidia-smi/name: my-job/' $${EXAMPLE_BP}
+
+    bash tools/add_ttl_label.sh "$${EXAMPLE_BP}"
+    cat $${EXAMPLE_BP}
+
+    ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \
+      --extra-vars="test_prefix=${_TEST_PREFIX}" \
+        --user=sa_106486320838376751393 \
+        --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
+        --extra-vars="region=$${REGION} zone=$${ZONE}" \
+        --extra-vars="enable_spot=$${ENABLE_SPOT}" \
+        --extra-vars="@$${GKE_VARS_FILE}"
+  secretEnv: ['GCLUSTER_GCS_PATH']
+availableSecrets:
+  secretManager:
+  - versionName: projects/${PROJECT_ID}/secrets/gcluster-develop-release-bucket/versions/latest
+    env: 'GCLUSTER_GCS_PATH'

tools/cloud-build/daily-tests/tests/gke-g4-onspot.yml

@@ -0,0 +1,38 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+# region, zone must be defined in build file with --extra-vars flag!
+test_name: gke-g4-spot
+deployment_name: gke-g4-spot-{{ build }}
+workspace: /workspace
+blueprint_yaml: "{{ workspace }}/examples/gke-g4/gke-g4.yaml"
+network: "{{ deployment_name }}-net-0"
+remote_node: "{{ deployment_name }}-remote-node-0"
+static_node_count: 2
+cli_deployment_vars:
+  region: "{{ region }}"
+  zone: "{{ zone }}"
+  static_node_count: "{{ static_node_count }}"
+  authorized_cidr: "{{ build_ip.stdout }}/32"
+  machine_type: g4-standard-48
+  num_gpus: 1
+custom_vars:
+  project: "{{ project }}"
+  instance_labels:
+    g4_onspot: true
+  enable_spot: true
+post_deploy_tests:
+- test-validation/test-gke-job.yml