diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54251c25e7..7f01ac5b83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,15 +74,6 @@ repos: files: 'tools/cloud-build/daily-tests/builds/.*\.yaml' pass_filenames: false require_serial: true - - id: validate-daily-test-builds - name: validate-daily-test-builds - entry: tools/cloud-build/daily-tests/validate_daily_test_builds.py - language: python - language_version: python3 - additional_dependencies: ['pyyaml'] - files: 'tools/cloud-build/daily-tests/builds/.*\.(yaml|yml)' - pass_filenames: true - require_serial: true - id: pytest-check name: pytest-check entry: pytest diff --git a/tools/cloud-build/check_retriable_error.sh b/tools/cloud-build/check_retriable_error.sh new file mode 100644 index 0000000000..2c11951e11 --- /dev/null +++ b/tools/cloud-build/check_retriable_error.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Returns 0 if a retriable error (e.g. Terraform capacity error) is found in the log, 1 otherwise. +# This script is called by Kueue pipelines to determine if the job should be requeued. +LOG_FILE=$1 +if [ -z "$LOG_FILE" ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +if [ ! -f "$LOG_FILE" ]; then + echo "Error: Log file '$LOG_FILE' does not exist or is not a regular file." >&2 + exit 2 +fi + +# Define all retriable errors here. +# Note: "Couldn't find a zone to deploy" and "ERROR: ZONE not found" are not included here +# because find_available_zone.sh now internally loops and waits for zone capacity to maintain Kueue locks. +RETRIABLE_ERRORS="ZONE_RESOURCE_POOL_EXHAUSTED|does not have enough resources available|not enough resources available|stockout" + +if grep -q -i -E "$RETRIABLE_ERRORS" "$LOG_FILE"; then + exit 0 +else + exit 1 +fi diff --git a/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8.yaml b/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8.yaml index 2f798058d7..0435c96ed7 100644 --- a/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8.yaml +++ b/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8.yaml @@ -26,42 +26,238 @@ tags: substitutions: _TEST_PREFIX: "" # Default to no prefix -timeout: 14400s # 4hr +timeout: 86400s +queueTtl: 86400s # 24hr + +options: + defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET + steps: -# While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests) -- id: check_for_running_build +# 1. Build and push workspace container containing the current code layer +- id: build-workspace-image + name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + args: + - -c + - | + docker build -t us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID -f- . < job.yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: slurm-gcp-v6-rocky8-$$BUILD_ID_SHORT + namespace: default + labels: + kueue.x-k8s.io/queue-name: local-queue-test-locks + build-id: "$BUILD_ID" + spec: + suspend: true # Required for Kueue: the job must be created in suspended state for Kueue to manage it + backoffLimit: 0 + template: + spec: + $$PRIORITY_LINE + terminationGracePeriodSeconds: 1800 # Grants 30 minutes for terraform destroy to finish gracefully if cancelled + serviceAccountName: test-kueue-cluster-runner-ksa + restartPolicy: Never + containers: + - name: runner + image: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID + env: + - name: ANSIBLE_HOST_KEY_CHECKING + value: "false" + - name: ANSIBLE_CONFIG + value: "/workspace/tools/cloud-build/ansible.cfg" + - name: _TEST_PREFIX + value: "${_TEST_PREFIX}" + - name: TRIAGE_GCS_BUCKET + value: "$$TRIAGE_GCS_BUCKET" + - name: TRIAGE_PROJECT_NUMBER + value: "$$TRIAGE_PROJECT_NUMBER" + - name: TRIAGE_INVOKER_SA + value: "$$TRIAGE_INVOKER_SA" + - name: TRIAGE_CLOUD_RUN_URL + value: "$$TRIAGE_CLOUD_RUN_URL" + command: ["/bin/bash", "-c"] + args: + - | + set -x -e + cd /workspace + + DEPLOYMENT_NAME="rock-8-$$BUILD_ID_SHORT" + + RUN_CLEANUP=true + # Trap function that runs the rescue playbook (terraform destroy) + # if the pod is terminated by Kueue, fails, or is cancelled. + cleanup_pod() { + local exit_code=\$$? + trap - EXIT SIGTERM SIGINT ERR + set +e + + if [ "\$$RUN_CLEANUP" = "false" ]; then + exit 0 + fi + if [ \$$exit_code -eq 0 ]; then exit_code=1; fi + + echo "" + echo "==========================================================================" + echo "CAUGHT SIGTERM OR SCRIPT ERROR!" + echo "Halting primary Ansible execution..." + echo "==========================================================================" + if [ -n "\$${ANSIBLE_PID:-}" ]; then + kill -TERM \$$ANSIBLE_PID 2>/dev/null || true + wait \$$ANSIBLE_PID 2>/dev/null || true + fi + + echo "" + echo "INITIATING RESCUE PLAYBOOK: Destroying leaked infrastructure for \$$DEPLOYMENT_NAME..." + echo "- hosts: localhost" > /workspace/cleanup-playbook.yml + echo " tasks:" >> /workspace/cleanup-playbook.yml + echo " - ansible.builtin.include_tasks:" >> /workspace/cleanup-playbook.yml + echo " file: tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml" >> /workspace/cleanup-playbook.yml + + ansible-playbook /workspace/cleanup-playbook.yml -e deployment_name="\$$DEPLOYMENT_NAME" -e workspace="/workspace" || true -- id: slurm-gcp-v6-rocky8 - name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner + echo "Graceful cleanup finished." + exit \$$exit_code + } + trap cleanup_pod EXIT SIGTERM SIGINT + + if [ "${_TEST_PREFIX}" == "daily-" ]; then + gsutil cp gs://$$GCLUSTER_GCS_PATH/latest/gcluster-bundle.zip . + unzip -o gcluster-bundle.zip + chmod +x gcluster + else + make + fi + BLUEPRINT="examples/hpc-slurm.yaml" + bash tools/add_ttl_label.sh \$$BLUEPRINT + python3 tools/fix_vpc_name.py \$$BLUEPRINT "${_TEST_PREFIX}" + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --extra-vars="test_prefix=${_TEST_PREFIX} use_fixed_vpc=true" \ + --user="$$OSLOGIN_USER" --extra-vars="project=$PROJECT_ID build=$$BUILD_ID_SHORT" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-v6-rocky8.yml" \ + --extra-vars="triage_gcs_bucket_override=\$$TRIAGE_GCS_BUCKET" \ + --extra-vars="triage_project_number_override=\$$TRIAGE_PROJECT_NUMBER" \ + --extra-vars="triage_invoker_sa_override=\$$TRIAGE_INVOKER_SA" \ + --extra-vars="triage_cloud_run_url_override=\$$TRIAGE_CLOUD_RUN_URL" & + ANSIBLE_PID=\$$! + + wait \$$ANSIBLE_PID + RUN_CLEANUP=false + resources: + requests: + cpu: 200m + memory: "2Gi" + test-locks/slurm: 1 + test-locks/slurm-gcp-v6-rocky8: 1 + limits: + cpu: 1 + memory: "2Gi" + test-locks/slurm: 1 + test-locks/slurm-gcp-v6-rocky8: 1 + EOF + +# 3. Submit and Monitor GKE Kueue Job with Retry +- id: submit-and-monitor-gke-job + name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + args: + - -c + - | + gcloud container clusters get-credentials test-kueue-cluster --region=us-central1 + BUILD_ID_SHORT=$$(echo "$BUILD_ID" | cut -c1-6) + JOB_NAME="slurm-gcp-v6-rocky8-$$BUILD_ID_SHORT" + + # Cloud Build trap: If the Cloud Build step itself receives a cancellation signal, + # delete the GKE job so the pod receives a SIGTERM and performs infrastructure cleanup. + cleanup_cb() { + echo "" + echo "==========================================================================" + echo "PIPELINE CANCELLED: The Cloud Build step received a termination signal." + echo "Deleting GKE Kueue Job ($$JOB_NAME) to force the Pod to clean itself up!" + echo "==========================================================================" + kubectl delete job "$$JOB_NAME" -n default || true + exit 1 + } + trap cleanup_cb SIGTERM SIGINT + + MAX_RETRIES=280 + RETRY_DELAY=300 + ATTEMPT=1 + + while true; do + echo "=== ATTEMPT $$ATTEMPT: Submitting Kueue Job ===" + kubectl apply -f /workspace/job.yaml + + set +e + ( bash tools/cloud-build/monitor_kueue_job.sh \ + test-kueue-cluster \ + us-central1 \ + "$$JOB_NAME" \ + default | tee /workspace/job_logs.txt; exit $${PIPESTATUS[0]} ) & + MONITOR_PID=$$! + wait $$MONITOR_PID + EXIT_CODE=$$? + set -e + + if [ $$EXIT_CODE -eq 0 ]; then + echo "Job succeeded!" + break + fi + + # Check if the failure was specifically due to a lack of GCP zone capacity. + # If so, retry. If it's a real error (like a terraform syntax error), fail immediately. + if bash tools/cloud-build/check_retriable_error.sh /workspace/job_logs.txt; then + echo "WARNING: Retriable error detected. Kueue Job has already been deleted. Retrying in $$RETRY_DELAY seconds..." + else + echo "ERROR: Test failed due to an actual error (not zone capacity). Failing pipeline." >&2 + exit 1 + fi + + if [ $$ATTEMPT -ge $$MAX_RETRIES ]; then + echo "ERROR: Job failed to find zone capacity after $$MAX_RETRIES attempts." >&2 + exit 1 + fi + + sleep $$RETRY_DELAY + ATTEMPT=$$((ATTEMPT + 1)) + done + +# 4. Cleanup the built image from Registry to save costs +- id: cleanup-image + name: gcr.io/cloud-builders/gcloud entrypoint: /bin/bash - env: - - "BUILD_ID=$BUILD_ID" - - "ANSIBLE_HOST_KEY_CHECKING=false" - - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" args: - -c - | - set -x -e - cd /workspace - bash tools/get_binary.sh "${_TEST_PREFIX}" - BUILD_ID_FULL=$BUILD_ID - BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} - BLUEPRINT="examples/hpc-slurm.yaml" - bash tools/add_ttl_label.sh $${BLUEPRINT} - python3 tools/fix_vpc_name.py $${BLUEPRINT} "${_TEST_PREFIX}" - - ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ - --extra-vars="test_prefix=${_TEST_PREFIX}" \ - --extra-vars="use_fixed_vpc=true" \ - --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT} full_build_id=$BUILD_ID" \ - --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-v6-rocky8.yml" \ - --extra-vars="triage_gcs_bucket_override=$$TRIAGE_GCS_BUCKET" \ - --extra-vars="triage_project_number_override=$$TRIAGE_PROJECT_NUMBER" \ - --extra-vars="triage_invoker_sa_override=$$TRIAGE_INVOKER_SA" \ - --extra-vars="triage_cloud_run_url_override=$$TRIAGE_CLOUD_RUN_URL" - secretEnv: ['GCLUSTER_GCS_PATH', 'TRIAGE_GCS_BUCKET', 'TRIAGE_PROJECT_NUMBER', 'TRIAGE_INVOKER_SA', 'TRIAGE_CLOUD_RUN_URL'] + gcloud artifacts docker images delete us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID --quiet availableSecrets: secretManager: - versionName: projects/${PROJECT_ID}/secrets/gcluster-develop-release-bucket/versions/latest @@ -74,3 +270,5 @@ availableSecrets: env: 'TRIAGE_INVOKER_SA' - versionName: projects/${PROJECT_ID}/secrets/triage-cloud-run-url/versions/latest env: 'TRIAGE_CLOUD_RUN_URL' + - versionName: projects/${PROJECT_ID}/secrets/sa-email/versions/latest + env: 'SA_EMAIL' diff --git a/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8_old.yaml b/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8_old.yaml new file mode 100644 index 0000000000..2f798058d7 --- /dev/null +++ b/tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8_old.yaml @@ -0,0 +1,76 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +tags: +- slurm6 +- m.filestore +- m.private-service-access +- m.schedmd-slurm-gcp-v6-controller +- m.schedmd-slurm-gcp-v6-login +- m.schedmd-slurm-gcp-v6-nodeset +- m.schedmd-slurm-gcp-v6-partition +- m.vpc + +substitutions: + _TEST_PREFIX: "" # Default to no prefix + +timeout: 14400s # 4hr +steps: +# While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests) +- id: check_for_running_build + name: gcr.io/cloud-builders/gcloud + script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/slurm-gcp-v6-rocky8.yaml" + +- id: slurm-gcp-v6-rocky8 + name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner + entrypoint: /bin/bash + env: + - "BUILD_ID=$BUILD_ID" + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + cd /workspace + bash tools/get_binary.sh "${_TEST_PREFIX}" + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + BLUEPRINT="examples/hpc-slurm.yaml" + bash tools/add_ttl_label.sh $${BLUEPRINT} + python3 tools/fix_vpc_name.py $${BLUEPRINT} "${_TEST_PREFIX}" + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --extra-vars="test_prefix=${_TEST_PREFIX}" \ + --extra-vars="use_fixed_vpc=true" \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT} full_build_id=$BUILD_ID" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-v6-rocky8.yml" \ + --extra-vars="triage_gcs_bucket_override=$$TRIAGE_GCS_BUCKET" \ + --extra-vars="triage_project_number_override=$$TRIAGE_PROJECT_NUMBER" \ + --extra-vars="triage_invoker_sa_override=$$TRIAGE_INVOKER_SA" \ + --extra-vars="triage_cloud_run_url_override=$$TRIAGE_CLOUD_RUN_URL" + secretEnv: ['GCLUSTER_GCS_PATH', 'TRIAGE_GCS_BUCKET', 'TRIAGE_PROJECT_NUMBER', 'TRIAGE_INVOKER_SA', 'TRIAGE_CLOUD_RUN_URL'] +availableSecrets: + secretManager: + - versionName: projects/${PROJECT_ID}/secrets/gcluster-develop-release-bucket/versions/latest + env: 'GCLUSTER_GCS_PATH' + - versionName: projects/${PROJECT_ID}/secrets/triage-gcs-bucket/versions/latest + env: 'TRIAGE_GCS_BUCKET' + - versionName: projects/${PROJECT_ID}/secrets/triage-project-number/versions/latest + env: 'TRIAGE_PROJECT_NUMBER' + - versionName: projects/${PROJECT_ID}/secrets/triage-invoker-sa/versions/latest + env: 'TRIAGE_INVOKER_SA' + - versionName: projects/${PROJECT_ID}/secrets/triage-cloud-run-url/versions/latest + env: 'TRIAGE_CLOUD_RUN_URL' diff --git a/tools/cloud-build/daily-tests/validate_daily_test_builds.py b/tools/cloud-build/daily-tests/validate_daily_test_builds.py deleted file mode 100755 index 94b6be3774..0000000000 --- a/tools/cloud-build/daily-tests/validate_daily_test_builds.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import sys -from pathlib import Path - -import yaml - -def validate_build_file(file_path: Path) -> bool: - """Validates a daily test build file. - - Args: - file_path: The path to the build file. - - Returns: - True if the file is valid, False otherwise. - """ - try: - with open(file_path, "r") as f: - data = yaml.safe_load(f) - except yaml.YAMLError as e: - print(f"Error parsing YAML file {file_path}: {e}", file=sys.stderr) - return False - - if not isinstance(data, dict): - print(f"Error: {file_path} is not a valid YAML dictionary.", file=sys.stderr) - return False - - steps = data.get("steps") - if not isinstance(steps, list): - print(f"Error: 'steps' not found or not a list in {file_path}", file=sys.stderr) - return False - - for step in steps: - if isinstance(step, dict) and step.get("id") == "check_for_running_build": - script = step.get("script") - if not script: - print( - f"Error: 'script' not found in 'check_for_running_build' step in {file_path}", - file=sys.stderr, - ) - return False - expected_script = f"tools/cloud-build/check_running_build.sh {file_path}" - if script != expected_script: - print( - f"Error: Invalid 'script' in 'check_for_running_build' step in {file_path}", - file=sys.stderr, - ) - print(f" Expected: {expected_script}", file=sys.stderr) - print(f" Got: {script}", file=sys.stderr) - return False - return True - - print( - f"Error: 'check_for_running_build' step not found in {file_path}", - file=sys.stderr, - ) - return False - - -def main(): - parser = argparse.ArgumentParser( - description="Validates daily test build files." - ) - parser.add_argument( - "filenames", - nargs="*", - help="The files to validate.", - ) - args = parser.parse_args() - - results = [validate_build_file(Path(filename)) for filename in args.filenames] - if not all(results): - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/cloud-build/monitor_kueue_job.sh b/tools/cloud-build/monitor_kueue_job.sh new file mode 100644 index 0000000000..5864eb81eb --- /dev/null +++ b/tools/cloud-build/monitor_kueue_job.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + exit 1 +fi + +CLUSTER_NAME="$1" +REGION="$2" +JOB_NAME="$3" +NAMESPACE="$4" + +echo "Connecting to GKE cluster $CLUSTER_NAME in $REGION..." +gcloud container clusters get-credentials "$CLUSTER_NAME" --region="$REGION" + +echo "Waiting for Job $JOB_NAME to be admitted by Kueue..." +while true; do + if ! SUSPENDED=$(kubectl get job "$JOB_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.suspend}' 2>/dev/null); then + echo "Error: Job $JOB_NAME not found yet or kubectl failed. Retrying..." + sleep 5 + continue + fi + if [ "$SUSPENDED" = "false" ]; then + echo "Job admitted by Kueue!" + break + fi + echo "Job is currently suspended/queued. Waiting..." + sleep 10 +done + +echo "Waiting for Pod to be initialized..." +while true; do + POD_NAME=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$JOB_NAME" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + if [ -n "$POD_NAME" ]; then + POD_PHASE=$(kubectl get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null) + if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ] || [ "$POD_PHASE" = "Failed" ]; then + echo "Pod $POD_NAME is $POD_PHASE. Ready to stream logs." + break + fi + echo "Pod $POD_NAME detected but is in phase: $POD_PHASE. Waiting for it to start running..." + fi + sleep 5 +done + +echo "Streaming logs from pod $POD_NAME..." +while true; do + # Stream logs + kubectl logs -f "$POD_NAME" -n "$NAMESPACE" -c runner + # Allow the Kubernetes controller a few seconds to update the Job status + # after the pod terminates. This prevents a race condition where we check + # the status before it's updated and accidentally restart the log stream. + sleep 5 + + # Check if job is completed + SUCCEEDED=$(kubectl get job "$JOB_NAME" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' 2>/dev/null) + FAILED=$(kubectl get job "$JOB_NAME" -n "$NAMESPACE" -o jsonpath='{.status.failed}' 2>/dev/null) + + if [ "$SUCCEEDED" = "1" ] || [ "$FAILED" = "1" ]; then + break + fi + echo "Logs disconnected but job is not finished. Reconnecting..." + # If it actually disconnected mid-job, sleep a moment before reconnecting + sleep 2 +done + +echo "Job finished execution. Fetching final status..." +SUCCEEDED=$(kubectl get job "$JOB_NAME" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' 2>/dev/null) +FAILED=$(kubectl get job "$JOB_NAME" -n "$NAMESPACE" -o jsonpath='{.status.failed}' 2>/dev/null) + +echo "Job status: Succeeded=$SUCCEEDED, Failed=$FAILED" + +echo "Cleaning up GKE Job resource..." +kubectl delete job "$JOB_NAME" -n "$NAMESPACE" + +if [ "$SUCCEEDED" = "1" ]; then + echo "GKE Kueue Job completed successfully." + exit 0 +else + echo "GKE Kueue Job failed (or was aborted)." + exit 1 +fi diff --git a/tools/cloud-build/provision/README.md b/tools/cloud-build/provision/README.md index bd763c7f5d..ba6aaae630 100644 --- a/tools/cloud-build/provision/README.md +++ b/tools/cloud-build/provision/README.md @@ -39,6 +39,7 @@ When prompted for project, use integration test project. | [daily\_project\_cleanup\_schedule](#module\_daily\_project\_cleanup\_schedule) | ./trigger-schedule | n/a | | [daily\_project\_cleanup\_slurm\_schedule](#module\_daily\_project\_cleanup\_slurm\_schedule) | ./trigger-schedule | n/a | | [daily\_test\_schedule](#module\_daily\_test\_schedule) | ./trigger-schedule | n/a | +| [daily\_test\_schedule\_migrated](#module\_daily\_test\_schedule\_migrated) | ./trigger-schedule | n/a | | [weekly\_build\_dependency\_check\_schedule](#module\_weekly\_build\_dependency\_check\_schedule) | ./trigger-schedule | n/a | ## Resources @@ -49,6 +50,7 @@ When prompted for project, use integration test project. | [google_cloudbuild_trigger.daily_project_cleanup_filestore](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | | [google_cloudbuild_trigger.daily_project_cleanup_slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | | [google_cloudbuild_trigger.daily_test](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | +| [google_cloudbuild_trigger.daily_test_migrated](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | | [google_cloudbuild_trigger.image_build_test_runner](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | | [google_cloudbuild_trigger.pr_go_build_test](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | | [google_cloudbuild_trigger.pr_ofe_test](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | @@ -66,6 +68,8 @@ When prompted for project, use integration test project. | Name | Description | Type | Default | Required | | ---- | ----------- | ---- | ------- | :------: | +| [daily\_tests\_project\_id](#input\_daily\_tests\_project\_id) | The GCP project for daily tests | `string` | `"hpc-toolkit-dev-2"` | no | +| [kueue\_migrated\_tests](#input\_kueue\_migrated\_tests) | List of tests migrated to Kueue | `list(string)` | 
[
  "slurm-gcp-v6-rocky8"
]
| no | | [project\_id](#input\_project\_id) | GCP project ID | `string` | `"hpc-toolkit-dev"` | no | | [region](#input\_region) | GCP region | `string` | `"us-central1"` | no | | [repo\_uri](#input\_repo\_uri) | URI of GitHub repo | `string` | `"https://github.com/GoogleCloudPlatform/cluster-toolkit"` | no | diff --git a/tools/cloud-build/provision/daily-tests.tf b/tools/cloud-build/provision/daily-tests.tf index 5c22eb450c..6ed46e682d 100644 --- a/tools/cloud-build/provision/daily-tests.tf +++ b/tools/cloud-build/provision/daily-tests.tf @@ -45,3 +45,37 @@ module "daily_test_schedule" { trigger = google_cloudbuild_trigger.daily_test[each.key] schedule = each.value } + +resource "google_cloudbuild_trigger" "daily_test_migrated" { + for_each = toset(var.kueue_migrated_tests) + name = "DAILY-test-${each.key}" + project = var.daily_tests_project_id + tags = [local.notify_chat_tag] + + git_file_source { + path = "tools/cloud-build/daily-tests/builds/${each.key}.yaml" + revision = local.ref_develop + uri = var.repo_uri + repo_type = "GITHUB" + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_develop + repo_type = "GITHUB" + } + # Following fields will be auto-set by CloudBuild after creation + # Specify it explicitly to reduce discreppancy. + ignored_files = [] + included_files = [] + substitutions = { + _TEST_PREFIX = "daily-" + } +} + +module "daily_test_schedule_migrated" { + source = "./trigger-schedule" + for_each = toset(var.kueue_migrated_tests) + trigger = google_cloudbuild_trigger.daily_test_migrated[each.key] + schedule = data.external.list_tests_midnight.result[each.key] +} diff --git a/tools/cloud-build/provision/trigger-schedule/main.tf b/tools/cloud-build/provision/trigger-schedule/main.tf index 123bd13a20..5621185615 100644 --- a/tools/cloud-build/provision/trigger-schedule/main.tf +++ b/tools/cloud-build/provision/trigger-schedule/main.tf @@ -14,6 +14,7 @@ resource "google_cloud_scheduler_job" "schedule" { name = "${var.trigger.name}-schedule" + project = var.trigger.project schedule = var.schedule time_zone = var.time_zone diff --git a/tools/cloud-build/provision/variables.tf b/tools/cloud-build/provision/variables.tf index cf0dcbe8cd..b71392a52a 100644 --- a/tools/cloud-build/provision/variables.tf +++ b/tools/cloud-build/provision/variables.tf @@ -37,3 +37,17 @@ variable "repo_uri" { type = string default = "https://github.com/GoogleCloudPlatform/cluster-toolkit" } + +variable "daily_tests_project_id" { + description = "The GCP project for daily tests" + type = string + default = "hpc-toolkit-dev-2" +} + +variable "kueue_migrated_tests" { + description = "List of tests migrated to Kueue" + type = list(string) + default = [ + "slurm-gcp-v6-rocky8" + ] +} diff --git a/tools/cloud-build/wait_for_available_zone.sh b/tools/cloud-build/wait_for_available_zone.sh new file mode 100644 index 0000000000..adfd10e8dd --- /dev/null +++ b/tools/cloud-build/wait_for_available_zone.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper around find_available_zone.sh that loops instead of exiting when out of capacity. +# Since find_available_zone.sh uses 'exit 1', we must run it in a subshell +# to prevent it from killing the pod. +while true; do + # Run the script in a subshell, streaming stdout and stderr to the console and a log file. + # To extract the exported variables, we have the subshell write them to a file. + set +e + ( + set -e + source /workspace/tools/cloud-build/find_available_zone.sh + # If it succeeds, these lines will execute and save the exports. + echo "export ZONE=\"${ZONE}\"" >/tmp/zone_export.sh + echo "export PROVISIONING_MODEL=\"${PROVISIONING_MODEL}\"" >>/tmp/zone_export.sh + ) 2>&1 | tee /tmp/zone_output.log + + EXIT_CODE=${PIPESTATUS[0]} + set -e + + if [ "$EXIT_CODE" -eq 0 ]; then + source /tmp/zone_export.sh + break + else + # Check if the failure was specifically due to zone capacity + if grep -q "Couldn't find a zone to deploy" /tmp/zone_output.log; then + echo "--- RETRYING in 5 minutes... ---" >&2 + sleep 300 + else + echo "--- FATAL ERROR: find_available_zone.sh failed due to a configuration or system error. Exiting. ---" >&2 + exit 1 + fi + fi +done