From f7a6985c412bdf52436acfdd0a959efd12a51e53 Mon Sep 17 00:00:00 2001 From: Siddhartha Tamatam Date: Tue, 7 Jul 2026 09:52:31 +0000 Subject: [PATCH 1/2] Increase version to 1.97.0 --- community/modules/compute/gke-nodeset/versions.tf | 2 +- community/modules/compute/gke-partition/versions.tf | 2 +- community/modules/compute/htcondor-execute-point/versions.tf | 2 +- community/modules/compute/mig/versions.tf | 2 +- .../compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf | 2 +- .../compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf | 2 +- .../modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf | 2 +- .../compute/schedmd-slurm-gcp-v6-partition/versions.tf | 2 +- .../modules/database/slurm-cloudsql-federation/versions.tf | 4 ++-- community/modules/file-system/nfs-server/versions.tf | 2 +- community/modules/files/fsi-montecarlo-on-batch/versions.tf | 4 ++-- community/modules/internal/slurm-gcp/login/versions.tf | 2 +- community/modules/project/service-enablement/versions.tf | 2 +- community/modules/pubsub/bigquery-sub/versions.tf | 4 ++-- community/modules/pubsub/topic/versions.tf | 2 +- community/modules/scheduler/htcondor-access-point/versions.tf | 2 +- .../modules/scheduler/htcondor-central-manager/versions.tf | 2 +- community/modules/scheduler/htcondor-pool-secrets/versions.tf | 2 +- .../scheduler/schedmd-slurm-gcp-v6-controller/versions.tf | 2 +- .../modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf | 2 +- community/modules/scripts/wait-for-startup/versions.tf | 2 +- community/modules/scripts/windows-startup-script/versions.tf | 2 +- modules/compute/gke-node-pool/versions.tf | 4 ++-- modules/compute/resource-policy/versions.tf | 2 +- modules/compute/vm-instance/versions.tf | 4 ++-- modules/file-system/cloud-storage-bucket/versions.tf | 4 ++-- modules/file-system/filestore/versions.tf | 4 ++-- modules/file-system/gke-persistent-volume/versions.tf | 2 +- modules/file-system/gke-storage/versions.tf | 2 +- modules/file-system/managed-lustre/versions.tf | 4 ++-- modules/file-system/netapp-storage-pool/versions.tf | 4 ++-- modules/file-system/netapp-volume/versions.tf | 4 ++-- modules/management/kubectl-apply/versions.tf | 2 +- modules/monitoring/dashboard/versions.tf | 2 +- modules/network/firewall-rules/versions.tf | 2 +- modules/network/pre-existing-subnetwork/versions.tf | 2 +- modules/network/pre-existing-vpc/versions.tf | 2 +- modules/network/private-service-access/versions.tf | 4 ++-- modules/scheduler/batch-login-node/versions.tf | 2 +- modules/scheduler/gke-cluster/versions.tf | 4 ++-- modules/scheduler/pre-existing-gke-cluster/versions.tf | 2 +- modules/scripts/startup-script/versions.tf | 2 +- pkg/config/config.go | 2 +- 43 files changed, 55 insertions(+), 55 deletions(-) diff --git a/community/modules/compute/gke-nodeset/versions.tf b/community/modules/compute/gke-nodeset/versions.tf index 19fa997279..abeea79eb2 100644 --- a/community/modules/compute/gke-nodeset/versions.tf +++ b/community/modules/compute/gke-nodeset/versions.tf @@ -22,6 +22,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.97.0" } } diff --git a/community/modules/compute/gke-partition/versions.tf b/community/modules/compute/gke-partition/versions.tf index bc66df4322..43cbcd9434 100644 --- a/community/modules/compute/gke-partition/versions.tf +++ b/community/modules/compute/gke-partition/versions.tf @@ -22,6 +22,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.97.0" } } diff --git a/community/modules/compute/htcondor-execute-point/versions.tf b/community/modules/compute/htcondor-execute-point/versions.tf index fb1d38473e..25c33159f9 100644 --- a/community/modules/compute/htcondor-execute-point/versions.tf +++ b/community/modules/compute/htcondor-execute-point/versions.tf @@ -29,6 +29,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.97.0" } } diff --git a/community/modules/compute/mig/versions.tf b/community/modules/compute/mig/versions.tf index abf9500257..cd62feb95f 100644 --- a/community/modules/compute/mig/versions.tf +++ b/community/modules/compute/mig/versions.tf @@ -22,6 +22,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.97.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf index fafb758827..d45a509f55 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf @@ -17,6 +17,6 @@ terraform { required_version = ">= 1.12.2" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.97.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf index ca7ee7f981..124a2884d2 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 1.12.2" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.97.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf index 5707c11add..22437f6f80 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.97.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf index f291e96cb2..53e1cebfdf 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 1.12.2" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.97.0" } } diff --git a/community/modules/database/slurm-cloudsql-federation/versions.tf b/community/modules/database/slurm-cloudsql-federation/versions.tf index c64ef95022..30609be422 100644 --- a/community/modules/database/slurm-cloudsql-federation/versions.tf +++ b/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/file-system/nfs-server/versions.tf b/community/modules/file-system/nfs-server/versions.tf index 115401ea95..1a3279bbff 100644 --- a/community/modules/file-system/nfs-server/versions.tf +++ b/community/modules/file-system/nfs-server/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/community/modules/files/fsi-montecarlo-on-batch/versions.tf index 355fa8e4bd..9739cb432a 100644 --- a/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ b/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -35,9 +35,9 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.97.0" } } diff --git a/community/modules/internal/slurm-gcp/login/versions.tf b/community/modules/internal/slurm-gcp/login/versions.tf index 30786ca4ad..4e6ca71ea3 100644 --- a/community/modules/internal/slurm-gcp/login/versions.tf +++ b/community/modules/internal/slurm-gcp/login/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.97.0" } } diff --git a/community/modules/project/service-enablement/versions.tf b/community/modules/project/service-enablement/versions.tf index 15b2c7b3a4..d23035dc5d 100644 --- a/community/modules/project/service-enablement/versions.tf +++ b/community/modules/project/service-enablement/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/pubsub/bigquery-sub/versions.tf b/community/modules/pubsub/bigquery-sub/versions.tf index f65cb02ad7..47af8eff0f 100644 --- a/community/modules/pubsub/bigquery-sub/versions.tf +++ b/community/modules/pubsub/bigquery-sub/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.97.0" } required_version = ">= 1.12.2" } diff --git a/community/modules/pubsub/topic/versions.tf b/community/modules/pubsub/topic/versions.tf index 2f5c8069dd..833ef16a3c 100644 --- a/community/modules/pubsub/topic/versions.tf +++ b/community/modules/pubsub/topic/versions.tf @@ -27,6 +27,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:topic/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:topic/v1.97.0" } } diff --git a/community/modules/scheduler/htcondor-access-point/versions.tf b/community/modules/scheduler/htcondor-access-point/versions.tf index d4e50822da..ec72d32204 100644 --- a/community/modules/scheduler/htcondor-access-point/versions.tf +++ b/community/modules/scheduler/htcondor-access-point/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/scheduler/htcondor-central-manager/versions.tf b/community/modules/scheduler/htcondor-central-manager/versions.tf index 0ac73ee216..ecd45f5b8c 100644 --- a/community/modules/scheduler/htcondor-central-manager/versions.tf +++ b/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/community/modules/scheduler/htcondor-pool-secrets/versions.tf index 6115c9d0a5..1924698e7d 100644 --- a/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ b/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf index af0913fd4e..419af34a4e 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.97.0" } } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf index a4a4585ad6..1e167cc8e5 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 1.12.2" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.97.0" } } diff --git a/community/modules/scripts/wait-for-startup/versions.tf b/community/modules/scripts/wait-for-startup/versions.tf index 9b67b09ecf..1c940e8816 100644 --- a/community/modules/scripts/wait-for-startup/versions.tf +++ b/community/modules/scripts/wait-for-startup/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.97.0" } required_version = ">= 1.12.2" diff --git a/community/modules/scripts/windows-startup-script/versions.tf b/community/modules/scripts/windows-startup-script/versions.tf index cb110cae3a..e9f2675421 100644 --- a/community/modules/scripts/windows-startup-script/versions.tf +++ b/community/modules/scripts/windows-startup-script/versions.tf @@ -16,7 +16,7 @@ terraform { provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/compute/gke-node-pool/versions.tf b/modules/compute/gke-node-pool/versions.tf index c495462ff6..3c2959a8e4 100644 --- a/modules/compute/gke-node-pool/versions.tf +++ b/modules/compute/gke-node-pool/versions.tf @@ -30,9 +30,9 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.97.0" } } diff --git a/modules/compute/resource-policy/versions.tf b/modules/compute/resource-policy/versions.tf index 0c0c1220e4..acd8ef863f 100644 --- a/modules/compute/resource-policy/versions.tf +++ b/modules/compute/resource-policy/versions.tf @@ -27,7 +27,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/compute/vm-instance/versions.tf b/modules/compute/vm-instance/versions.tf index 5859dc418a..64b50fdc03 100644 --- a/modules/compute/vm-instance/versions.tf +++ b/modules/compute/vm-instance/versions.tf @@ -31,10 +31,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/file-system/cloud-storage-bucket/versions.tf b/modules/file-system/cloud-storage-bucket/versions.tf index 39c9484fa1..9f23ff6957 100644 --- a/modules/file-system/cloud-storage-bucket/versions.tf +++ b/modules/file-system/cloud-storage-bucket/versions.tf @@ -30,10 +30,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.97.0" } required_version = ">= 1.12.2" } diff --git a/modules/file-system/filestore/versions.tf b/modules/file-system/filestore/versions.tf index 9cb09e219d..4222ceb58d 100644 --- a/modules/file-system/filestore/versions.tf +++ b/modules/file-system/filestore/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/file-system/gke-persistent-volume/versions.tf b/modules/file-system/gke-persistent-volume/versions.tf index 86d5dedd8f..84f4122051 100644 --- a/modules/file-system/gke-persistent-volume/versions.tf +++ b/modules/file-system/gke-persistent-volume/versions.tf @@ -25,6 +25,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.97.0" } } diff --git a/modules/file-system/gke-storage/versions.tf b/modules/file-system/gke-storage/versions.tf index 05482428b2..fff1763641 100644 --- a/modules/file-system/gke-storage/versions.tf +++ b/modules/file-system/gke-storage/versions.tf @@ -16,6 +16,6 @@ terraform { required_version = ">= 1.12.2" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.97.0" } } diff --git a/modules/file-system/managed-lustre/versions.tf b/modules/file-system/managed-lustre/versions.tf index ad0635c88c..dc56c3cdd5 100644 --- a/modules/file-system/managed-lustre/versions.tf +++ b/modules/file-system/managed-lustre/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/file-system/netapp-storage-pool/versions.tf b/modules/file-system/netapp-storage-pool/versions.tf index 399215640e..68665afa91 100644 --- a/modules/file-system/netapp-storage-pool/versions.tf +++ b/modules/file-system/netapp-storage-pool/versions.tf @@ -27,10 +27,10 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/file-system/netapp-volume/versions.tf b/modules/file-system/netapp-volume/versions.tf index 96a169c8dd..299432f351 100644 --- a/modules/file-system/netapp-volume/versions.tf +++ b/modules/file-system/netapp-volume/versions.tf @@ -22,10 +22,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/management/kubectl-apply/versions.tf b/modules/management/kubectl-apply/versions.tf index e104f16af7..c383240acd 100644 --- a/modules/management/kubectl-apply/versions.tf +++ b/modules/management/kubectl-apply/versions.tf @@ -39,7 +39,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:kubectl-apply/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:kubectl-apply/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/monitoring/dashboard/versions.tf b/modules/monitoring/dashboard/versions.tf index 4e5f788224..0ec7caa3e3 100644 --- a/modules/monitoring/dashboard/versions.tf +++ b/modules/monitoring/dashboard/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/network/firewall-rules/versions.tf b/modules/network/firewall-rules/versions.tf index f8a79270e8..5b004a7bec 100644 --- a/modules/network/firewall-rules/versions.tf +++ b/modules/network/firewall-rules/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/network/pre-existing-subnetwork/versions.tf b/modules/network/pre-existing-subnetwork/versions.tf index 8c49479338..c39d171c02 100644 --- a/modules/network/pre-existing-subnetwork/versions.tf +++ b/modules/network/pre-existing-subnetwork/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/network/pre-existing-vpc/versions.tf b/modules/network/pre-existing-vpc/versions.tf index 3151ae83c5..1e01c3fbcd 100644 --- a/modules/network/pre-existing-vpc/versions.tf +++ b/modules/network/pre-existing-vpc/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/network/private-service-access/versions.tf b/modules/network/private-service-access/versions.tf index 19641b7b5b..37d656deef 100644 --- a/modules/network/private-service-access/versions.tf +++ b/modules/network/private-service-access/versions.tf @@ -26,11 +26,11 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/scheduler/batch-login-node/versions.tf b/modules/scheduler/batch-login-node/versions.tf index 510b674c32..180c597f9d 100644 --- a/modules/scheduler/batch-login-node/versions.tf +++ b/modules/scheduler/batch-login-node/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/scheduler/gke-cluster/versions.tf b/modules/scheduler/gke-cluster/versions.tf index b1d97cbc08..dc06db5ad4 100644 --- a/modules/scheduler/gke-cluster/versions.tf +++ b/modules/scheduler/gke-cluster/versions.tf @@ -30,10 +30,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.97.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.97.0" } } diff --git a/modules/scheduler/pre-existing-gke-cluster/versions.tf b/modules/scheduler/pre-existing-gke-cluster/versions.tf index 6db0c9b0f8..cdd907e0a7 100644 --- a/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ b/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -23,7 +23,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.97.0" } required_version = ">= 1.12.2" diff --git a/modules/scripts/startup-script/versions.tf b/modules/scripts/startup-script/versions.tf index c1b555a7f1..2f39c83592 100644 --- a/modules/scripts/startup-script/versions.tf +++ b/modules/scripts/startup-script/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.96.0" + module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.97.0" } required_version = ">= 1.12.2" diff --git a/pkg/config/config.go b/pkg/config/config.go index 30eac1b099..131e222e23 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,7 +44,7 @@ import ( const ( maxHintDist int = 3 // Maximum Levenshtein distance where we suggest a hint - latestToolkitVersion = "v1.96.0" + latestToolkitVersion = "v1.97.0" // SharedModulesDirName is the name of the shared directory for embedded modules SharedModulesDirName = "_modules" ) From 229e988150542a8b277d58ed778c803181978065 Mon Sep 17 00:00:00 2001 From: Leiyi Zhang Date: Fri, 10 Jul 2026 12:40:03 -0700 Subject: [PATCH 2/2] integrate cluster health check into toolkit. (#5917) Co-authored-by: Neelabh94 --- .pre-commit-config.yaml | 22 + community/cluster-health-check/.gitignore | 1 + community/cluster-health-check/Dockerfile | 83 ++ community/cluster-health-check/README.md | 117 +++ .../build-and-push-cluster-health-check.sh | 109 +++ .../cmd/dcgm-healthcheck/main.go | 743 ++++++++++++++++++ .../cmd/dcgm-healthcheck/main_test.go | 157 ++++ .../deployment/configmap.yaml | 21 + .../deployment/dcgm-healthcheck.yaml | 139 ++++ community/cluster-health-check/go.mod | 50 ++ community/cluster-health-check/go.sum | 155 ++++ 11 files changed, 1597 insertions(+) create mode 100644 community/cluster-health-check/.gitignore create mode 100644 community/cluster-health-check/Dockerfile create mode 100644 community/cluster-health-check/README.md create mode 100755 community/cluster-health-check/build-and-push-cluster-health-check.sh create mode 100644 community/cluster-health-check/cmd/dcgm-healthcheck/main.go create mode 100644 community/cluster-health-check/cmd/dcgm-healthcheck/main_test.go create mode 100644 community/cluster-health-check/deployment/configmap.yaml create mode 100644 community/cluster-health-check/deployment/dcgm-healthcheck.yaml create mode 100644 community/cluster-health-check/go.mod create mode 100644 community/cluster-health-check/go.sum diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a036cde02e..54251c25e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,11 +127,33 @@ repos: types: [python] pass_filenames: false + - id: go-vet-community-health-check + name: go-vet-community-health-check + entry: bash -c 'cd community/cluster-health-check && go vet ./...' + language: system + files: ^community/cluster-health-check/.*\.go$ + pass_filenames: false + + - id: go-unit-tests-community-health-check + name: go-unit-tests-community-health-check + entry: bash -c 'cd community/cluster-health-check && go test -timeout 30s -short -v ./...' + language: system + files: ^community/cluster-health-check/.*\.go$ + pass_filenames: false + + - id: go-mod-tidy-community-health-check + name: go-mod-tidy-community-health-check + entry: bash -c 'cd community/cluster-health-check && go mod tidy && git diff --exit-code go.mod go.sum' + language: system + files: ^community/cluster-health-check/(go\.mod|go\.sum|.*\.go)$ + pass_filenames: false + - repo: https://github.com/dnephin/pre-commit-golang rev: v0.5.1 hooks: - id: go-fmt - id: go-vet + exclude: ^community/cluster-health-check/ - id: go-imports - id: go-cyclo args: [-over=13] diff --git a/community/cluster-health-check/.gitignore b/community/cluster-health-check/.gitignore new file mode 100644 index 0000000000..48b8bf9072 --- /dev/null +++ b/community/cluster-health-check/.gitignore @@ -0,0 +1 @@ +vendor/ diff --git a/community/cluster-health-check/Dockerfile b/community/cluster-health-check/Dockerfile new file mode 100644 index 0000000000..76efec0301 --- /dev/null +++ b/community/cluster-health-check/Dockerfile @@ -0,0 +1,83 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=${BUILDPLATFORM} golang:1.25.10 AS build +ARG TARGETARCH +ARG BUILDARCH +ARG TARGETOS + +WORKDIR /build +RUN apt-get update && apt-get install -yq --no-install-recommends \ + gcc libc6-dev \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross +COPY . . + +RUN CGO_ENABLED=0 \ + GOARCH=${TARGETARCH} \ + GOOS=${TARGETOS} \ + GOTOOLCHAIN=local \ + go build -mod=vendor -o dcgm-healthcheck ./cmd/dcgm-healthcheck + +# ----------------------------------------------------------------------------- +# Base Debian Libraries +# ----------------------------------------------------------------------------- +FROM gke.gcr.io/debian-base:bookworm-v1.0.2-gke.0 AS base +RUN apt-get update && apt-get install -y --no-install-recommends infiniband-diags + +# ----------------------------------------------------------------------------- +# Part 1: Base Application Image (Without DCGM dependencies) +# ----------------------------------------------------------------------------- +FROM gke.gcr.io/gke-distroless/libc:gke_distroless_20240307.00_p0 AS dcgm-healthcheck-base + +# Required for GPU metrics +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,compat32 +ENV NVIDIA_VISIBLE_DEVICES=all + +# COPY over system library files from debian-base. +# Empirically, without these files, distroless causes a panic. +# We still avoid exposing any shell capability. +COPY --from=base /lib/* /lib/ +COPY --from=base /usr/lib/*-linux-gnu/libtinfo.so.6 /lib/ +COPY --from=base /bin/dmesg /bin/dmesg +COPY --from=base /usr/sbin/ibstat /usr/bin/ibstat +COPY --from=base /usr/lib/*-linux-gnu/libibumad.so* /usr/lib/ +COPY --from=base /usr/lib/*-linux-gnu/libibmad.so* /usr/lib/ + +# Bring in dcgm-healthcheck. +COPY --from=build /build/dcgm-healthcheck /bin/dcgm-healthcheck +ENTRYPOINT ["/bin/dcgm-healthcheck"] + +# ----------------------------------------------------------------------------- +# Part 2: NVIDIA DCGM Source +# Pulls the official NVIDIA packages to build a functional image in 1 command +# ----------------------------------------------------------------------------- +FROM --platform=linux/${TARGETARCH} nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 AS dcgm-nvidia +ARG TARGETARCH +RUN apt-get update && apt-get install -y --no-install-recommends datacenter-gpu-manager-4-cuda12 + +# ----------------------------------------------------------------------------- +# Final Standalone Image (For local testing / standalone deployment) +# ----------------------------------------------------------------------------- +FROM dcgm-healthcheck-base AS dcgm-healthcheck + +# Copy DCGM binaries +COPY --from=dcgm-nvidia /usr/bin/nv-hostengine /usr/bin/nv-hostengine +COPY --from=dcgm-nvidia /usr/bin/dcgmi /usr/bin/dcgmi + +# Copy DCGM libraries. Because the source path uses a wildcard (*-linux-gnu), +# Docker flattens the output into the destination directory (/usr/lib/). +COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgm.so* /usr/lib/ +COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulenvswitch.so* /usr/lib/ +COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulehealth.so* /usr/lib/ +COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulesysmon.so* /usr/lib/ diff --git a/community/cluster-health-check/README.md b/community/cluster-health-check/README.md new file mode 100644 index 0000000000..6e34b601b7 --- /dev/null +++ b/community/cluster-health-check/README.md @@ -0,0 +1,117 @@ +# DCGM Health Check + +![Platform: GKE](https://img.shields.io/badge/Platform-GKE-green.svg) + +This tool runs as a Kubernetes DaemonSet to perform passive health checks on NVIDIA GPUs within a GKE cluster using DCGM (Data Center GPU Manager) and NVML. + +It continuously monitors GPU nodes for hardware errors, XID errors, and InfiniBand/network issues. When an issue is detected, the agent patches the Kubernetes Node object to report the failure, applying the appropriate severity label (`Warning`, `Failure` or `Fatal`). + +-------------------------------------------------------------------------------- + +## ✨ Key Features + +- **Passive Health Checking**: Connects to the local DCGM daemon to watch for hardware errors without disrupting running workloads. +- **Failure Reporting**: Automatically adds conditions to the Kubernetes Node (e.g., `GPUUnhealthy`) and sets the `cloud.google.com/health-check-status` label to the severity of the issue (e.g., `Fatal`, `Warning`) when issues occur. The label is cleared when the node is healthy. +- **Configurable XID Severities**: Any detected XID error marks the node as unhealthy with a `Warning` severity. You can define specific XID errors to escalate to `Fatal` severity via an optional ConfigMap. + +## 🚀 Quick Start + +The health check agent is deployed as a DaemonSet to ensure it runs on every node with an NVIDIA GPU. + +### Deploying to a GKE cluster + +1. **Build and push the image**: + Use the provided script to build a multi-architecture Docker image and push it to your Artifact Registry: + + ```bash + ./build-and-push-cluster-health-check.sh -p -r -i cluster-health-check + ``` + +2. **Update the image reference**: + Edit `deployment/dcgm-healthcheck.yaml` to replace the `//cluster-health-check:latest` image string with your remote destination image created in the previous step. + +3. **Deploy the DaemonSet**: + Apply the Kubernetes manifest in the `deployment/` directory: + + ```bash + kubectl apply -f deployment/dcgm-healthcheck.yaml + ``` + +### 📝 Example Output + +Once deployed, the agent will continuously monitor the node's health. When a failure is detected, the agent patches the node's status conditions and metadata labels. You can view the health check results directly on the nodes: + +```bash +$ kubectl get nodes -o custom-columns=NAME:.metadata.name,HEALTH:.metadata.labels.cloud\.google\.com/health-check-status +NAME HEALTH +gke-sa-gke-a4x-a4x-highgpu-4g-a4x-poo-a482f777-078q +gke-sa-gke-a4x-a4x-highgpu-4g-a4x-poo-a482f777-1rk7 warning +``` + +You can view the specific failure message in the node conditions using `kubectl describe`: + +```bash +$ kubectl get node -o json | jq -r '["Type", "Status", "LastTransitionTime", "Reason", "Message"], (.status.conditions[] | select(.type == "GPUUnhealthy") | [.type, .status, .lastTransitionTime, .reason, .message]) | @tsv' | column -t + +Type Status LastTransitionTime Reason Message +GPUUnhealthy True 2026-07-08T20:57:02Z HealthCheckFailed +``` + +## ⚙️ Configuration (Optional) + +The `fatal-xids-config` ConfigMap is **optional**. It controls which NVIDIA XID errors escalate the node's issue severity from `Warning` to `Fatal`. + +By default, any XID error will cause the node to be marked as unhealthy (`GPUUnhealthy=True`) with a `Warning` severity. If you apply this ConfigMap and an error matches the `fatal-xids` list, the `cloud.google.com/health-check-status` label will instead be set to `Fatal`. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fatal-xids-config + namespace: default +data: + fatal-xids: "79, 119" # Comma-separated list of fatal XIDs +``` + +If you wish to configure fatal XIDs, apply the ConfigMap: + +```bash +kubectl apply -f deployment/configmap.yaml +``` + +You can update this ConfigMap at any time and the agent will automatically reload the configuration. + +## 🛠️ Developer Guide: Modifying and Releasing Your Own Image + +This section is for developers who wish to customize the script's behavior or release their own version of the container image to a private Google Artifact Registry. + +### 1. Modifying the Code + +- The core logic for generating the health check is located in the `cmd/` directory, with the main entry point being `cmd/dcgm-healthcheck/main.go`. +- The image and all relevant dependencies are defined in the `Dockerfile`. + +### 2. Building and Pushing to Artifact Registry + +We provide a convenient shell script to build and push your customized image to your own Artifact Registry. + +You can do so by invoking the `build-and-push-cluster-health-check.sh` script with the following parameters: + +| Flag | Description | Required | +| :--- | :-------------------------------------------------------------- | :------- | +| `-p` | Your Google Cloud Project ID. | **Yes** | +| `-r` | The name of your Artifact Registry repository. | **Yes** | +| `-i` | The name for your image. | **Yes** | +| `-l` | The region of your Artifact Registry. Defaults to `us-central1` | No | +| `-v` | Version tag for the image. Defaults to `YYYY-MM-DD`. | No | +| `-h` | Display the help message. | No | + +Sample command to build and push a new image: + +```bash +bash build-and-push-cluster-health-check.sh \ + -p ${PROJECT?} \ + -r ${ARTIFACT_REPO?} \ + -i "cluster-health-check" \ + -l "us-east1" \ + -v "0.0.3" +``` diff --git a/community/cluster-health-check/build-and-push-cluster-health-check.sh b/community/cluster-health-check/build-and-push-cluster-health-check.sh new file mode 100755 index 0000000000..1a1fe0137e --- /dev/null +++ b/community/cluster-health-check/build-and-push-cluster-health-check.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Copyright 2026 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -u -o pipefail + +usage() { + echo >&2 "Usage: bash build-and-push-cluster-health-check.sh -p -r -i [-l ] [-v ]" + echo >&2 "This script builds a Docker image and pushes it to Google Artifact Registry." + echo >&2 "" + echo >&2 "Options:" + echo >&2 " -p Your Google Cloud Project ID." + echo >&2 " -r The Artifact Registry repository name." + echo >&2 " -i The name for the Docker image." + echo >&2 " -l (Optional) The Artifact Registry region. Defaults to 'us-central1'." + echo >&2 " -v (Optional) The version tag for the image. Defaults to YYYY-MM-DD." + echo >&2 " -h Display this help message." + echo >&2 "" + echo >&2 "Example (default region):" + echo >&2 " bash build-and-push-cluster-health-check.sh -p gpu-test-project -r cluster-health-check-repo -i cluster-health-check" + echo >&2 "" + echo >&2 "Example (specific region):" + echo >&2 " bash build-and-push-cluster-health-check.sh -p gpu-test-project -r cluster-health-check-repo -i cluster-health-check -l us-east4" + exit 1 +} + +PROJECT="" +REPO="" +IMAGE="" +REGION="us-central1" +VERSION="" + +while getopts ":p:r:i:l:v:h" opt; do + case ${opt} in + p) + PROJECT=$OPTARG + ;; + r) + REPO=$OPTARG + ;; + i) + IMAGE=$OPTARG + ;; + l) + REGION=$OPTARG + ;; + v) + VERSION=$OPTARG + ;; + h) + usage + ;; + \?) + echo "Invalid Option: -$OPTARG" >&2 + usage + ;; + :) + echo "Invalid Option: -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [[ -z "${PROJECT}" ]] || [[ -z "${REPO}" ]] || [[ -z "${IMAGE}" ]]; then + echo "Error: Missing required arguments." >&2 + usage +fi + +if [[ -z "${VERSION}" ]]; then + VERSION=$(date +%Y-%m-%d) +fi +REMOTE_DESTINATION="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${IMAGE}" + +echo "=================================================" +echo "Configuration received:" +echo " Project ID: ${PROJECT}" +echo " Repository Name: ${REPO}" +echo " Image Name: ${IMAGE}" +echo " Generated Version: ${VERSION}" +echo " Remote Destination: ${REMOTE_DESTINATION}" +echo "=================================================" + +set -x + +echo "Building image ${IMAGE}:${VERSION}" + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +echo "Vendoring Go dependencies..." +(cd "${SCRIPT_DIR}" && go mod vendor) + +docker buildx build --platform linux/amd64,linux/arm64 \ + -t "${REMOTE_DESTINATION}:${VERSION}" \ + -t "${REMOTE_DESTINATION}:latest" \ + --target dcgm-healthcheck \ + --push \ + -f "${SCRIPT_DIR}/Dockerfile" \ + "${SCRIPT_DIR}" diff --git a/community/cluster-health-check/cmd/dcgm-healthcheck/main.go b/community/cluster-health-check/cmd/dcgm-healthcheck/main.go new file mode 100644 index 0000000000..e3677c458f --- /dev/null +++ b/community/cluster-health-check/cmd/dcgm-healthcheck/main.go @@ -0,0 +1,743 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" +) + +var lastRxPackets = make(map[string]int64) + +type GPUUnhealthyReason string + +const ( + ConditionTypeGPUUnhealthy = "GPUUnhealthy" + + ReasonHealthCheckFailed GPUUnhealthyReason = "HealthCheckFailed" + ReasonActiveTestFailed GPUUnhealthyReason = "ActiveTestFailed" + + LabelHealthStatus = "cloud.google.com/health-check-status" + + SeverityWarning = "warning" + SeverityFailure = "failure" + SeverityFatal = "fatal" + SeverityInfo = "info" +) + +func main() { + startTime := time.Now() + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + log.Fatal("NODE_NAME environment variable is required") + } + + checkInterval := flag.Duration("check-interval", 5*time.Minute, "Interval between health checks") + flag.Parse() + + // Wire SIGINT/SIGTERM to ctx so shutdown is cooperative — the informer + // stops, subprocesses started with CommandContext get killed, and the + // main select returns cleanly. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + log.Println("Starting nv-hostengine...") + hostEngine := exec.CommandContext(ctx, "nv-hostengine", "-n") + if err := hostEngine.Start(); err != nil { + log.Fatalf("Failed to start nv-hostengine: %v", err) + } + + go func() { + err := hostEngine.Wait() + // a non-nil ctx.Err() means ctx received a shutdown signal, so nv-hostengine + // was killed as part of that shutdown instead of errored out. In that case + // we don't Fatalf + if ctx.Err() != nil { + log.Printf("nv-hostengine exited during container shutdown: %v", err) + return + } + log.Fatalf("nv-hostengine exited unexpectedly: %v", err) + }() + + if err := waitForHostEngine(ctx, 30*time.Second); err != nil { + log.Fatalf("nv-hostengine not ready: %v", err) + } + + log.Println("Setting all health watches via dcgmi...") + setupCtx, setupCancel := context.WithTimeout(ctx, 30*time.Second) + err := exec.CommandContext(setupCtx, "dcgmi", "health", "-s", "a").Run() + setupCancel() + if err != nil { + log.Fatalf("Failed to set health watches: %v", err) + } + + config, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Failed to create in-cluster config: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create kubernetes clientset: %v", err) + } + + tweakListOptions := func(options *metav1.ListOptions) { + options.FieldSelector = fmt.Sprintf("metadata.name=%s", nodeName) + } + factory := informers.NewSharedInformerFactoryWithOptions(clientset, 5*time.Minute, informers.WithTweakListOptions(tweakListOptions)) + nodeInformer := factory.Core().V1().Nodes() + informer := nodeInformer.Informer() + + log.Println("Starting Node informer cache...") + factory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + log.Fatalf("Timed out waiting for caches to sync") + } + + ticker := time.NewTicker(*checkInterval) + defer ticker.Stop() + + log.Printf("Starting health check loop for node %s", nodeName) + runHealthCheck(ctx, clientset, nodeInformer.Lister(), nodeName, startTime) + for { + select { + case <-ticker.C: + runHealthCheck(ctx, clientset, nodeInformer.Lister(), nodeName, startTime) + case <-ctx.Done(): + log.Println("Shutting down...") + return + } + } +} + +// waitForHostEngine polls dcgmi discovery until nv-hostengine responds, or +// timeout elapses. This replaces a blind time.Sleep — slow-boot nodes fail +// on 5s but usually succeed within 10-15s. +func waitForHostEngine(ctx context.Context, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + probeCtx, probeCancel := context.WithTimeout(ctx, 5*time.Second) + err := exec.CommandContext(probeCtx, "dcgmi", "discovery", "-l").Run() + probeCancel() + if err == nil { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for nv-hostengine: %v", err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } +} + +func getFatalXids() map[int]struct{} { + data, err := os.ReadFile("/etc/dcgm-healthcheck/fatal-xids") + if err != nil { + return make(map[int]struct{}) + } + return parseFatalXids(data) +} + +func parseFatalXids(data []byte) map[int]struct{} { + fatalXids := make(map[int]struct{}) + for _, xidStr := range strings.Split(string(data), ",") { + xidStr = strings.TrimSpace(xidStr) + if xidStr == "" { + continue + } + if xid, err := strconv.Atoi(xidStr); err == nil { + fatalXids[xid] = struct{}{} + } + } + return fatalXids +} + +func runHealthCheck(mainCtx context.Context, clientset *kubernetes.Clientset, nodeLister corev1listers.NodeLister, nodeName string, startTime time.Time) { + ctx, cancel := context.WithTimeout(mainCtx, 5*time.Minute) + defer cancel() + + var errorMessages []string + // SeverityWarning is the baseline: when no check fails, errorMessages is + // empty and no NodeCondition/label is written, so this default is never + // surfaced. It only takes effect after a check returns an error without + // its own severity — escalating severity can happen but never lowering it. + highestSeverity := SeverityWarning + + fatalXids := getFatalXids() + + passiveChecks := []struct { + name string + fn func(context.Context) (string, error) + }{ + {"dcgmi health", runDcgmiHealth}, + // {"NIC heartbeat", checkNICHeartbeat}, + {"XID/SXID monitoring", func(c context.Context) (string, error) { return checkKernelLogsForXidSxid(c, fatalXids, startTime) }}, + {"ECC errors", checkECCErrors}, + {"PCIe link health", checkPCIe}, + {"InfiniBand links", checkIB}, + {"GPU temperature", checkTemperature}, + // {"HCA Firmware", checkHCAFW}, + // {"GPU Power draw", checkPower}, + } + for _, chk := range passiveChecks { + sev, err := chk.fn(ctx) + if err == nil { + continue + } + highestSeverity = mergeStatus(highestSeverity, sev) + errorMessages = append(errorMessages, fmt.Sprintf("%s: %v", chk.name, err)) + } + + node, err := nodeLister.Get(nodeName) + if err != nil { + log.Printf("Failed to get node %s from cache: %v", nodeName, err) + return + } + + hasIssue := false + conditionExists := false + var currentReason string + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeConditionType(ConditionTypeGPUUnhealthy) { + conditionExists = true + if condition.Status == corev1.ConditionTrue { + hasIssue = true + } + currentReason = string(condition.Reason) + break + } + } + + _, hasStatus := node.Labels[LabelHealthStatus] + + if len(errorMessages) == 0 { + if currentReason != string(ReasonActiveTestFailed) && (conditionExists || hasStatus) { + log.Println("Node is now healthy from passive checks. Ensuring condition/labels are cleared.") + clearNodeHealth(ctx, clientset, nodeName, conditionExists, hasStatus) + } + return + } + + desiredMessage := strings.Join(errorMessages, " | ") + log.Printf("Passive checks failed: %s", desiredMessage) + + // When the node already carries an active-test-failed reason, keep + // that reason (which has precedence) but still surface the passive + // severity/message via mergeMessage inside updateNodeHealth. + reason := ReasonHealthCheckFailed + if hasIssue && currentReason == string(ReasonActiveTestFailed) { + reason = ReasonActiveTestFailed + } + + updateNodeHealth(ctx, clientset, node, reason, highestSeverity, desiredMessage) +} + +func extractOverflowMessages(v interface{}) []string { + var messages []string + switch val := v.(type) { + case map[string]interface{}: + // Sort keys so identical dcgmi output produces an identical joined + // message across ticks — otherwise randomized map iteration order + // causes constant Node PATCH churn. + keys := make([]string, 0, len(val)) + for k := range val { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + child := val[k] + if k == "overflow" { + if arr, ok := child.([]interface{}); ok { + var parts []string + for _, p := range arr { + if s, ok := p.(string); ok { + parts = append(parts, strings.TrimSpace(s)) + } + } + messages = append(messages, strings.Join(parts, " ")) + } + } else { + messages = append(messages, extractOverflowMessages(child)...) + } + } + case []interface{}: + for _, child := range val { + messages = append(messages, extractOverflowMessages(child)...) + } + } + return messages +} + +func runDcgmiHealth(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "dcgmi", "health", "-c", "-j").CombinedOutput() + if err != nil { + return SeverityWarning, fmt.Errorf("%v: %s", err, string(out)) + } + + var parsed struct { + Body map[string]interface{} `json:"body"` + } + if err := json.Unmarshal(out, &parsed); err != nil { + return SeverityWarning, fmt.Errorf("failed to parse dcgmi health output: %v", err) + } + + overallHealthVal := "" + if overall, ok := parsed.Body["Overall Health"].(map[string]interface{}); ok { + if val, ok := overall["value"].(string); ok { + overallHealthVal = val + } + } + + if overallHealthVal != "Healthy" { + severity := SeverityWarning + if strings.ToLower(overallHealthVal) == SeverityFailure { + severity = SeverityFailure + } + + detailsStr := "" + delete(parsed.Body, "Overall Health") + + messages := extractOverflowMessages(parsed.Body) + if len(messages) > 0 { + detailsStr = fmt.Sprintf(" | Details: %s", strings.Join(messages, "; ")) + } + + return severity, fmt.Errorf("health issues detected (Overall Health: %s)%s", overallHealthVal, detailsStr) + } + + return "", nil +} + +func checkNICHeartbeat(ctx context.Context) (string, error) { + sysClassNet := os.Getenv("SYS_CLASS_NET") + if sysClassNet == "" { + sysClassNet = "/sys/class/net" + } + matches, err := filepath.Glob(filepath.Join(sysClassNet, "gpu*rdma*/statistics/rx_packets")) + if err != nil || len(matches) == 0 { + return SeverityWarning, fmt.Errorf("no gpu rdma net interfaces found in %s", sysClassNet) + } + allZeroDiff := true + for _, match := range matches { + data, err := os.ReadFile(match) + if err != nil { + continue + } + val, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + continue + } + + if lastVal, ok := lastRxPackets[match]; ok { + if val-lastVal > 0 { + allZeroDiff = false + } + } else { + // First check, don't fail immediately + allZeroDiff = false + } + lastRxPackets[match] = val + } + if allZeroDiff { + return SeverityWarning, fmt.Errorf("rx_packets difference is 0 on all GPU RDMA interfaces") + } + return "", nil +} + +func checkHCAFW(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "dmesg").CombinedOutput() + if err != nil { + log.Printf("Cannot run dmesg for HCA FW monitoring: %v", err) + return "", nil + } + lines := strings.Split(string(out), "\n") + // Only check the last 1000 lines for simplicity + start := len(lines) - 1000 + if start < 0 { + start = 0 + } + for i := start; i < len(lines); i++ { + if strings.Contains(lines[i], "Health issue observed, firmware internal error") { + return SeverityWarning, fmt.Errorf("firmware internal error found in dmesg: %s", lines[i]) + } + } + return "", nil +} + +func getSeverity(s string) int { + switch strings.ToLower(s) { + case SeverityFatal: + return 4 + case SeverityFailure: + return 3 + case SeverityWarning: + return 2 + case SeverityInfo: + return 1 + default: + return 0 + } +} + +func mergeStatus(oldStatus, newStatus string) string { + if getSeverity(oldStatus) > getSeverity(newStatus) { + return oldStatus + } + return newStatus +} + +func mergeMessage(oldMsg, newMsg string) string { + if oldMsg == "" { + return newMsg + } + if newMsg == "" { + return oldMsg + } + + var segments []string + seen := make(map[string]bool) + + for _, seg := range strings.Split(oldMsg, "|") { + seg = strings.TrimSpace(seg) + if seg != "" && !seen[seg] { + segments = append(segments, seg) + seen[seg] = true + } + } + for _, seg := range strings.Split(newMsg, "|") { + seg = strings.TrimSpace(seg) + if seg != "" && !seen[seg] { + segments = append(segments, seg) + seen[seg] = true + } + } + + return strings.Join(segments, " | ") +} + +// The patch shapes are declared as typed structs and marshaled with +// json.Marshal so that user-derived strings (dmesg lines, dcgmi output) are +// escaped by the standard JSON encoder rather than fmt %q — Go's %q emits +// invalid JSON escapes (\a, \v, \xHH) for control bytes. + +type nodeStatusPatch struct { + Status struct { + Conditions []corev1.NodeCondition `json:"conditions"` + } `json:"status"` +} + +type nodeLabelsPatch struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` +} + +type nodeLabelsClearPatch struct { + Metadata struct { + Labels map[string]interface{} `json:"labels"` + } `json:"metadata"` +} + +type nodeConditionDeletePatch struct { + Status struct { + Conditions []map[string]interface{} `json:"conditions"` + } `json:"status"` +} + +func updateNodeHealth(ctx context.Context, clientset *kubernetes.Clientset, node *corev1.Node, reason GPUUnhealthyReason, status, details string) { + nodeName := node.Name + + oldStatus := "" + oldMsg := "" + oldReason := "" + oldCondStatus := corev1.ConditionUnknown + var oldTransitionTime time.Time + if val, ok := node.Labels[LabelHealthStatus]; ok { + oldStatus = val + } + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeConditionType(ConditionTypeGPUUnhealthy) { + oldMsg = cond.Message + oldReason = string(cond.Reason) + oldCondStatus = cond.Status + oldTransitionTime = cond.LastTransitionTime.Time + break + } + } + + finalStatus := mergeStatus(oldStatus, status) + + // Replace the message only when the current condition is a + // same-reason passive re-report (rotating passive failures should + // not accumulate). Otherwise merge — either the reason is transitioning + // (preserve prior signal) or we're accumulating active-test context. + var finalMsg string + if oldReason != "" && !(oldReason == string(reason) && reason == ReasonHealthCheckFailed) { + finalMsg = mergeMessage(oldMsg, details) + } else { + finalMsg = details + } + + // No-op if the node already reflects the intended state — otherwise we + // would rewrite lastTransitionTime every tick and thrash the API server. + if oldCondStatus == corev1.ConditionTrue && + oldReason == string(reason) && + oldMsg == finalMsg && + oldStatus == finalStatus { + return + } + + // Only advance lastTransitionTime when the Status field is actually transitioning + transitionTime := time.Now() + if oldCondStatus == corev1.ConditionTrue { + transitionTime = oldTransitionTime + } + + var condPatch nodeStatusPatch + condPatch.Status.Conditions = []corev1.NodeCondition{{ + Type: corev1.NodeConditionType(ConditionTypeGPUUnhealthy), + Status: corev1.ConditionTrue, + Reason: string(reason), + Message: finalMsg, + LastTransitionTime: metav1.Time{Time: transitionTime}, + }} + + var labelsPatch nodeLabelsPatch + labelsPatch.Metadata.Labels = map[string]string{LabelHealthStatus: finalStatus} + + applyPatches(ctx, clientset, nodeName, condPatch, labelsPatch, finalStatus) +} + +func applyPatches(ctx context.Context, clientset *kubernetes.Clientset, nodeName string, condPatch nodeStatusPatch, labelsPatch nodeLabelsPatch, finalStatus string) { + statusPatchBytes, err := json.Marshal(condPatch) + if err != nil { + log.Printf("Error building status patch for %s: %v", nodeName, err) + return + } + if _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, statusPatchBytes, metav1.PatchOptions{}, "status"); err != nil { + log.Printf("Error patching node status %s: %v", nodeName, err) + } + + labelsPatchBytes, err := json.Marshal(labelsPatch) + if err != nil { + log.Printf("Error building metadata patch for %s: %v", nodeName, err) + return + } + if _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, labelsPatchBytes, metav1.PatchOptions{}); err != nil { + log.Printf("Error patching node metadata %s: %v", nodeName, err) + } else { + log.Printf("Successfully updated node %s with status %s", nodeName, finalStatus) + } +} + +var xidRegex = regexp.MustCompile(`(?i)(?:xid|sxid)(?:\s*\(.*?\))?:\s*(\d+)`) + +func checkKernelLogsForXidSxid(ctx context.Context, fatalXids map[int]struct{}, startTime time.Time) (string, error) { + timeStr := startTime.Format("2006-01-02 15:04:05") + out, err := exec.CommandContext(ctx, "dmesg", "--since", timeStr).CombinedOutput() + if err != nil { + log.Printf("Cannot run dmesg for kernel log monitoring: %v", err) + return "", nil + } + lines := strings.Split(string(out), "\n") + + latestByCode := make(map[int]string) + severity := SeverityWarning + + for _, line := range lines { + lowerLine := strings.ToLower(line) + if strings.Contains(lowerLine, "xid") || strings.Contains(lowerLine, "sxid") { + matches := xidRegex.FindStringSubmatch(line) + if len(matches) > 1 { + if xid, err := strconv.Atoi(matches[1]); err == nil { + latestByCode[xid] = line + if _, ok := fatalXids[xid]; ok { + severity = SeverityFatal + } + } + } + } + } + + var codes []int + for code := range latestByCode { + codes = append(codes, code) + } + // sort the codes to ensure the generated string is deterministic. + // Otherwise, random map iteration order will cause constant K8s API patches. + sort.Ints(codes) + + var foundErrors []string + for _, code := range codes { + foundErrors = append(foundErrors, latestByCode[code]) + } + + if len(foundErrors) > 0 { + return severity, fmt.Errorf("XID/SXID error(s) found in dmesg: %s", strings.Join(foundErrors, " | ")) + } + return "", nil +} + +func nvidiaSmiPath() string { + if _, err := os.Stat("/usr/bin/nvidia-smi"); err == nil { + return "/usr/bin/nvidia-smi" + } + return "/usr/local/nvidia/bin/nvidia-smi" +} + +func checkECCErrors(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, nvidiaSmiPath(), "--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits").CombinedOutput() + if err != nil { + return SeverityWarning, fmt.Errorf("nvidia-smi ecc query failed: %v: %s", err, strings.TrimSpace(string(out))) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for i, line := range lines { + parts := strings.Split(line, ",") + if len(parts) >= 2 { + uncorr, _ := strconv.Atoi(strings.TrimSpace(parts[1])) + if uncorr > 0 { + return SeverityWarning, fmt.Errorf("found uncorrected ECC errors on GPU %d: %d", i, uncorr) + } + } + } + return "", nil +} + +func checkPCIe(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, nvidiaSmiPath(), "--query-gpu=pcie.link.width.current,pcie.link.width.max", "--format=csv,noheader,nounits").CombinedOutput() + if err != nil { + return SeverityWarning, fmt.Errorf("nvidia-smi pcie query failed: %v: %s", err, strings.TrimSpace(string(out))) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for i, line := range lines { + parts := strings.Split(line, ",") + if len(parts) >= 2 { + width, _ := strconv.Atoi(strings.TrimSpace(parts[0])) + maxWidth, errMax := strconv.Atoi(strings.TrimSpace(parts[1])) + if errMax != nil || maxWidth <= 0 { + maxWidth = 16 // fallback + } + if width > 0 && width < maxWidth { + return SeverityWarning, fmt.Errorf("PCIe link width degraded on GPU %d: %d (expected %d)", i, width, maxWidth) + } + } + } + return "", nil +} + +func checkIB(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "ibstat").CombinedOutput() + if err != nil { + return "", nil + } + if strings.Contains(string(out), "Down") { + return SeverityWarning, fmt.Errorf("ibstat reported Down") + } + return "", nil +} + +func checkTemperature(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, nvidiaSmiPath(), "--query-gpu=temperature.gpu,temperature.gpu.tlimit", "--format=csv,noheader,nounits").CombinedOutput() + if err != nil { + return SeverityWarning, fmt.Errorf("nvidia-smi temperature query failed: %v: %s", err, strings.TrimSpace(string(out))) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for i, line := range lines { + parts := strings.Split(line, ",") + if len(parts) >= 2 { + tempStr := strings.TrimSpace(parts[0]) + limitStr := strings.TrimSpace(parts[1]) + + countdown, err := strconv.Atoi(limitStr) + if err == nil { + if countdown <= 0 { + return SeverityWarning, fmt.Errorf("GPU %d temperature too high: current is %s degree which is %d degrees over target temperature", i, tempStr, -countdown) + } + } else { + // Fallback to absolute temperature check if countdown is not supported + temp, errTemp := strconv.Atoi(tempStr) + if errTemp == nil && temp >= 90 { + return SeverityWarning, fmt.Errorf("GPU %d temperature too high: %d C >= 90 C", i, temp) + } + } + } + } + return "", nil +} + +func checkPower(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, nvidiaSmiPath(), "--query-gpu=power.draw", "--format=csv,noheader,nounits").CombinedOutput() + if err != nil { + return SeverityWarning, fmt.Errorf("nvidia-smi power query failed: %v: %s", err, strings.TrimSpace(string(out))) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for i, line := range lines { + val, err := strconv.ParseFloat(strings.TrimSpace(line), 64) + if err == nil && val > 700.0 { + return SeverityWarning, fmt.Errorf("GPU %d power draw too high: %.1f W", i, val) + } + } + return "", nil +} + +func clearNodeHealth(ctx context.Context, clientset *kubernetes.Clientset, nodeName string, clearCondition, clearLabel bool) { + if clearCondition { + var condPatch nodeConditionDeletePatch + condPatch.Status.Conditions = []map[string]interface{}{ + {"type": ConditionTypeGPUUnhealthy, "$patch": "delete"}, + } + patchBytes, err := json.Marshal(condPatch) + if err != nil { + log.Printf("Error building status delete patch for %s: %v", nodeName, err) + } else if _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status"); err != nil { + log.Printf("Error deleting node status condition %s: %v", nodeName, err) + } + } + + if clearLabel { + var labelsPatch nodeLabelsClearPatch + labelsPatch.Metadata.Labels = map[string]interface{}{LabelHealthStatus: nil} + patchBytes, err := json.Marshal(labelsPatch) + if err != nil { + log.Printf("Error building metadata clear patch for %s: %v", nodeName, err) + } else if _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil { + log.Printf("Info: Attempted to clear health labels on node %s: %v", nodeName, err) + } + } +} diff --git a/community/cluster-health-check/cmd/dcgm-healthcheck/main_test.go b/community/cluster-health-check/cmd/dcgm-healthcheck/main_test.go new file mode 100644 index 0000000000..b301c1306a --- /dev/null +++ b/community/cluster-health-check/cmd/dcgm-healthcheck/main_test.go @@ -0,0 +1,157 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" +) + +func TestGetSeverity(t *testing.T) { + tests := []struct { + input string + expected int + }{ + {SeverityFatal, 4}, + {SeverityFailure, 3}, + {SeverityWarning, 2}, + {SeverityInfo, 1}, + {"unknown", 0}, + {"", 0}, + } + + for _, test := range tests { + if got := getSeverity(test.input); got != test.expected { + t.Errorf("getSeverity(%q) = %d, expected %d", test.input, got, test.expected) + } + } +} + +func TestMergeStatus(t *testing.T) { + tests := []struct { + oldStatus string + newStatus string + expected string + }{ + {SeverityWarning, SeverityFatal, SeverityFatal}, + {SeverityFatal, SeverityWarning, SeverityFatal}, + {SeverityWarning, SeverityFailure, SeverityFailure}, + {SeverityFailure, SeverityWarning, SeverityFailure}, + {SeverityInfo, SeverityWarning, SeverityWarning}, + {SeverityWarning, SeverityInfo, SeverityWarning}, + {"unknown", SeverityInfo, SeverityInfo}, + {SeverityInfo, "unknown", SeverityInfo}, + {"", "", ""}, + } + + for _, test := range tests { + if got := mergeStatus(test.oldStatus, test.newStatus); got != test.expected { + t.Errorf("mergeStatus(%q, %q) = %q, expected %q", test.oldStatus, test.newStatus, got, test.expected) + } + } +} + +func TestMergeMessage(t *testing.T) { + tests := []struct { + oldMsg string + newMsg string + expected string + }{ + {"", "new message", "new message"}, + {"old message", "", "old message"}, + {"a | b", "b | c", "a | b | c"}, + } + + for _, test := range tests { + if got := mergeMessage(test.oldMsg, test.newMsg); got != test.expected { + t.Errorf("mergeMessage(%q, %q) = %q, expected %q", test.oldMsg, test.newMsg, got, test.expected) + } + } +} + +func TestExtractOverflowMessages(t *testing.T) { + tests := []struct { + desc string + input interface{} + expected []string + }{ + { + desc: "nested overflow maps", + input: map[string]interface{}{ + "a": map[string]interface{}{ + "overflow": []interface{}{"msg1", "msg2"}, + }, + "b": map[string]interface{}{ + "c": map[string]interface{}{ + "overflow": []interface{}{"msg3"}, + }, + }, + }, + expected: []string{"msg1 msg2", "msg3"}, + }, + } + + for _, test := range tests { + msgs := extractOverflowMessages(test.input) + if !reflect.DeepEqual(msgs, test.expected) { + t.Errorf("extractOverflowMessages(%s) = %v, expected %v", test.desc, msgs, test.expected) + } + } +} + +func TestXidRegex(t *testing.T) { + tests := []struct { + input string + xid string + }{ + {"NVRM: Xid (PCI:0000:01:00): 31, pid=1234, name=python", "31"}, + {"NVRM: sxid (PCI:0000:02:00): 10045, info", "10045"}, + {"xid: 13", "13"}, + } + + for _, test := range tests { + matches := xidRegex.FindStringSubmatch(test.input) + if len(matches) < 2 { + t.Errorf("Expected match for %q", test.input) + } else if matches[1] != test.xid { + t.Errorf("Expected %q, got %q", test.xid, matches[1]) + } + } +} + +func TestParseFatalXids(t *testing.T) { + tests := []struct { + desc string + input []byte + expected map[int]struct{} + }{ + { + desc: "valid and invalid xids", + input: []byte(" 31 , 12, 13, invalid, 14 "), + expected: map[int]struct{}{ + 31: {}, + 12: {}, + 13: {}, + 14: {}, + }, + }, + } + + for _, test := range tests { + if got := parseFatalXids(test.input); !reflect.DeepEqual(got, test.expected) { + t.Errorf("parseFatalXids(%s) = %v, expected %v", test.desc, got, test.expected) + } + } +} diff --git a/community/cluster-health-check/deployment/configmap.yaml b/community/cluster-health-check/deployment/configmap.yaml new file mode 100644 index 0000000000..e08778e4dd --- /dev/null +++ b/community/cluster-health-check/deployment/configmap.yaml @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fatal-xids-config + namespace: kube-system +data: + fatal-xids: "79" diff --git a/community/cluster-health-check/deployment/dcgm-healthcheck.yaml b/community/cluster-health-check/deployment/dcgm-healthcheck.yaml new file mode 100644 index 0000000000..85ceb6806e --- /dev/null +++ b/community/cluster-health-check/deployment/dcgm-healthcheck.yaml @@ -0,0 +1,139 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dcgm-healthcheck + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dcgm-healthcheck-node-patcher +rules: +- apiGroups: [""] + resources: ["nodes", "nodes/status"] + verbs: ["get", "list", "watch", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dcgm-healthcheck-node-patcher-binding +subjects: +- kind: ServiceAccount + name: dcgm-healthcheck + namespace: kube-system +roleRef: + kind: ClusterRole + name: dcgm-healthcheck-node-patcher + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-healthcheck + namespace: kube-system + labels: + app: dcgm-healthcheck +spec: + selector: + matchLabels: + app: dcgm-healthcheck + template: + metadata: + labels: + app: dcgm-healthcheck + spec: + serviceAccountName: dcgm-healthcheck + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-accelerator + operator: Exists + initContainers: + - name: nvml-wait + image: gke.gcr.io/gke-distroless/bash:gke_distroless_20240307.00_p0 + command: + - bash + - -c + - | + echo "Waiting for NVML drivers..." + # Bounded wait: if drivers never load, exit non-zero so the kubelet + # restarts the pod and the failure surfaces as Init:CrashLoopBackOff. + timeout=120 + elapsed=0 + while (( elapsed < timeout )); do + if [[ -f "/usr/local/nvidia/lib64/libnvidia-ml.so.1" ]]; then + echo "NVML drivers found!" + exit 0 + fi + sleep 2 + (( elapsed += 2 )) + done + echo "Timed out after ${timeout}s waiting for NVML drivers." >&2 + exit 1 + volumeMounts: + - name: nvidia-install-dir-host + mountPath: /usr/local/nvidia + readOnly: true + containers: + - name: dcgm-healthcheck + image: //cluster-health-check: + imagePullPolicy: IfNotPresent + # args: ["-check-interval=15s"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + securityContext: + privileged: true + volumeMounts: + - name: fatal-xids-config + mountPath: /etc/dcgm-healthcheck + readOnly: true + - name: nvidia-install-dir-host + mountPath: /usr/local/nvidia + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + tolerations: + - operator: Exists + volumes: + - name: fatal-xids-config + configMap: + name: fatal-xids-config + optional: true + - name: nvidia-install-dir-host + hostPath: + path: /home/kubernetes/bin/nvidia + type: Directory + - name: varlog + hostPath: + path: /var/log + type: Directory diff --git a/community/cluster-health-check/go.mod b/community/cluster-health-check/go.mod new file mode 100644 index 0000000000..d493937abf --- /dev/null +++ b/community/cluster-health-check/go.mod @@ -0,0 +1,50 @@ +module community/cluster-health-check + +go 1.25.0 + +require ( + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/client-go v0.33.3 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/term v0.43.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/community/cluster-health-check/go.sum b/community/cluster-health-check/go.sum new file mode 100644 index 0000000000..3e9ad717f1 --- /dev/null +++ b/community/cluster-health-check/go.sum @@ -0,0 +1,155 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=