Merge v1.97.0 into Develop

.pre-commit-config.yaml

@@ -127,11 +127,33 @@ repos:
     types: [python]
     pass_filenames: false
 
+  - id: go-vet-community-health-check
+    name: go-vet-community-health-check
+    entry: bash -c 'cd community/cluster-health-check && go vet ./...'
+    language: system
+    files: ^community/cluster-health-check/.*\.go$
+    pass_filenames: false
+
+  - id: go-unit-tests-community-health-check
+    name: go-unit-tests-community-health-check
+    entry: bash -c 'cd community/cluster-health-check && go test -timeout 30s -short -v ./...'
+    language: system
+    files: ^community/cluster-health-check/.*\.go$
+    pass_filenames: false
+
+  - id: go-mod-tidy-community-health-check
+    name: go-mod-tidy-community-health-check
+    entry: bash -c 'cd community/cluster-health-check && go mod tidy && git diff --exit-code go.mod go.sum'
+    language: system
+    files: ^community/cluster-health-check/(go\.mod|go\.sum|.*\.go)$
+    pass_filenames: false
+
 - repo: https://github.com/dnephin/pre-commit-golang
   rev: v0.5.1
   hooks:
   - id: go-fmt
   - id: go-vet
+    exclude: ^community/cluster-health-check/
   - id: go-imports
   - id: go-cyclo
     args: [-over=13]

community/cluster-health-check/.gitignore

@@ -0,0 +1 @@
+vendor/

community/cluster-health-check/Dockerfile

@@ -0,0 +1,83 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM --platform=${BUILDPLATFORM} golang:1.25.10 AS build
+ARG TARGETARCH
+ARG BUILDARCH
+ARG TARGETOS
+
+WORKDIR /build
+RUN apt-get update && apt-get install -yq --no-install-recommends \
+    gcc libc6-dev \
+    gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+COPY . .
+
+RUN CGO_ENABLED=0 \
+    GOARCH=${TARGETARCH} \
+    GOOS=${TARGETOS} \
+    GOTOOLCHAIN=local \
+    go build -mod=vendor -o dcgm-healthcheck ./cmd/dcgm-healthcheck
+
+# -----------------------------------------------------------------------------
+# Base Debian Libraries
+# -----------------------------------------------------------------------------
+FROM gke.gcr.io/debian-base:bookworm-v1.0.2-gke.0 AS base
+RUN apt-get update && apt-get install -y --no-install-recommends infiniband-diags
+
+# -----------------------------------------------------------------------------
+# Part 1: Base Application Image (Without DCGM dependencies)
+# -----------------------------------------------------------------------------
+FROM gke.gcr.io/gke-distroless/libc:gke_distroless_20240307.00_p0 AS dcgm-healthcheck-base
+
+# Required for GPU metrics
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,compat32
+ENV NVIDIA_VISIBLE_DEVICES=all
+
+# COPY over system library files from debian-base.
+# Empirically, without these files, distroless causes a panic.
+# We still avoid exposing any shell capability.
+COPY --from=base /lib/* /lib/
+COPY --from=base /usr/lib/*-linux-gnu/libtinfo.so.6 /lib/
+COPY --from=base /bin/dmesg /bin/dmesg
+COPY --from=base /usr/sbin/ibstat /usr/bin/ibstat
+COPY --from=base /usr/lib/*-linux-gnu/libibumad.so* /usr/lib/
+COPY --from=base /usr/lib/*-linux-gnu/libibmad.so* /usr/lib/
+
+# Bring in dcgm-healthcheck.
+COPY --from=build /build/dcgm-healthcheck /bin/dcgm-healthcheck
+ENTRYPOINT ["/bin/dcgm-healthcheck"]
+
+# -----------------------------------------------------------------------------
+# Part 2: NVIDIA DCGM Source
+# Pulls the official NVIDIA packages to build a functional image in 1 command
+# -----------------------------------------------------------------------------
+FROM --platform=linux/${TARGETARCH} nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 AS dcgm-nvidia
+ARG TARGETARCH
+RUN apt-get update && apt-get install -y --no-install-recommends datacenter-gpu-manager-4-cuda12
+
+# -----------------------------------------------------------------------------
+# Final Standalone Image (For local testing / standalone deployment)
+# -----------------------------------------------------------------------------
+FROM dcgm-healthcheck-base AS dcgm-healthcheck
+
+# Copy DCGM binaries
+COPY --from=dcgm-nvidia /usr/bin/nv-hostengine /usr/bin/nv-hostengine
+COPY --from=dcgm-nvidia /usr/bin/dcgmi /usr/bin/dcgmi
+
+# Copy DCGM libraries. Because the source path uses a wildcard (*-linux-gnu),
+# Docker flattens the output into the destination directory (/usr/lib/).
+COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgm.so* /usr/lib/
+COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulenvswitch.so* /usr/lib/
+COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulehealth.so* /usr/lib/
+COPY --from=dcgm-nvidia /usr/lib/*-linux-gnu/libdcgmmodulesysmon.so* /usr/lib/

community/cluster-health-check/README.md

@@ -0,0 +1,117 @@
+# DCGM Health Check
+
+![Platform: GKE](https://img.shields.io/badge/Platform-GKE-green.svg)
+
+This tool runs as a Kubernetes DaemonSet to perform passive health checks on NVIDIA GPUs within a GKE cluster using DCGM (Data Center GPU Manager) and NVML.
+
+It continuously monitors GPU nodes for hardware errors, XID errors, and InfiniBand/network issues. When an issue is detected, the agent patches the Kubernetes Node object to report the failure, applying the appropriate severity label (`Warning`, `Failure` or `Fatal`).
+
+--------------------------------------------------------------------------------
+
+## ✨ Key Features
+
+- **Passive Health Checking**: Connects to the local DCGM daemon to watch for hardware errors without disrupting running workloads.
+- **Failure Reporting**: Automatically adds conditions to the Kubernetes Node (e.g., `GPUUnhealthy`) and sets the `cloud.google.com/health-check-status` label to the severity of the issue (e.g., `Fatal`, `Warning`) when issues occur. The label is cleared when the node is healthy.
+- **Configurable XID Severities**: Any detected XID error marks the node as unhealthy with a `Warning` severity. You can define specific XID errors to escalate to `Fatal` severity via an optional ConfigMap.
+
+## 🚀 Quick Start
+
+The health check agent is deployed as a DaemonSet to ensure it runs on every node with an NVIDIA GPU.
+
+### Deploying to a GKE cluster
+
+1. **Build and push the image**:
+   Use the provided script to build a multi-architecture Docker image and push it to your Artifact Registry:
+
+   ```bash
+   ./build-and-push-cluster-health-check.sh -p <YOUR_PROJECT> -r <YOUR_REPO> -i cluster-health-check
+   ```
+
+2. **Update the image reference**:
+   Edit `deployment/dcgm-healthcheck.yaml` to replace the `<YOUR_REGISTRY>/<YOUR_REPO>/cluster-health-check:latest` image string with your remote destination image created in the previous step.
+
+3. **Deploy the DaemonSet**:
+   Apply the Kubernetes manifest in the `deployment/` directory:
+
+   ```bash
+   kubectl apply -f deployment/dcgm-healthcheck.yaml
+   ```
+
+### 📝 Example Output
+
+Once deployed, the agent will continuously monitor the node's health. When a failure is detected, the agent patches the node's status conditions and metadata labels. You can view the health check results directly on the nodes:
+
+```bash
+$ kubectl get nodes -o custom-columns=NAME:.metadata.name,HEALTH:.metadata.labels.cloud\.google\.com/health-check-status
+NAME                                                  HEALTH
+gke-sa-gke-a4x-a4x-highgpu-4g-a4x-poo-a482f777-078q   <none>
+gke-sa-gke-a4x-a4x-highgpu-4g-a4x-poo-a482f777-1rk7   warning
+```
+
+You can view the specific failure message in the node conditions using `kubectl describe`:
+
+```bash
+$ kubectl get node <node-name> -o json | jq -r '["Type", "Status", "LastTransitionTime", "Reason", "Message"], (.status.conditions[] | select(.type == "GPUUnhealthy") | [.type, .status, .lastTransitionTime, .reason, .message]) | @tsv' | column -t
+
+Type          Status  LastTransitionTime    Reason             Message
+GPUUnhealthy  True    2026-07-08T20:57:02Z  HealthCheckFailed  <detailed health check message>
+```
+
+## ⚙️ Configuration (Optional)
+
+The `fatal-xids-config` ConfigMap is **optional**. It controls which NVIDIA XID errors escalate the node's issue severity from `Warning` to `Fatal`.
+
+By default, any XID error will cause the node to be marked as unhealthy (`GPUUnhealthy=True`) with a `Warning` severity. If you apply this ConfigMap and an error matches the `fatal-xids` list, the `cloud.google.com/health-check-status` label will instead be set to `Fatal`.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fatal-xids-config
+  namespace: default
+data:
+  fatal-xids: "79, 119" # Comma-separated list of fatal XIDs
+```
+
+If you wish to configure fatal XIDs, apply the ConfigMap:
+
+```bash
+kubectl apply -f deployment/configmap.yaml
+```
+
+You can update this ConfigMap at any time and the agent will automatically reload the configuration.
+
+## 🛠️ Developer Guide: Modifying and Releasing Your Own Image
+
+This section is for developers who wish to customize the script's behavior or release their own version of the container image to a private Google Artifact Registry.
+
+### 1. Modifying the Code
+
+- The core logic for generating the health check is located in the `cmd/` directory, with the main entry point being `cmd/dcgm-healthcheck/main.go`.
+- The image and all relevant dependencies are defined in the `Dockerfile`.
+
+### 2. Building and Pushing to Artifact Registry
+
+We provide a convenient shell script to build and push your customized image to your own Artifact Registry.
+
+You can do so by invoking the `build-and-push-cluster-health-check.sh` script with the following parameters:
+
+| Flag | Description                                                     | Required |
+| :--- | :-------------------------------------------------------------- | :------- |
+| `-p` | Your Google Cloud Project ID.                                   | **Yes**  |
+| `-r` | The name of your Artifact Registry repository.                  | **Yes**  |
+| `-i` | The name for your image.                                        | **Yes**  |
+| `-l` | The region of your Artifact Registry. Defaults to `us-central1` | No       |
+| `-v` | Version tag for the image. Defaults to `YYYY-MM-DD`.            | No       |
+| `-h` | Display the help message.                                       | No       |
+
+Sample command to build and push a new image:
+
+```bash
+bash build-and-push-cluster-health-check.sh \
+    -p ${PROJECT?} \
+    -r ${ARTIFACT_REPO?} \
+    -i "cluster-health-check" \
+    -l "us-east1" \
+    -v "0.0.3"
+```

community/cluster-health-check/build-and-push-cluster-health-check.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+# Copyright 2026 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -u -o pipefail
+
+usage() {
+	echo >&2 "Usage: bash build-and-push-cluster-health-check.sh -p <PROJECT_ID> -r <REPO_NAME> -i <IMAGE_NAME> [-l <REGION>] [-v <VERSION>]"
+	echo >&2 "This script builds a Docker image and pushes it to Google Artifact Registry."
+	echo >&2 ""
+	echo >&2 "Options:"
+	echo >&2 "  -p    Your Google Cloud Project ID."
+	echo >&2 "  -r    The Artifact Registry repository name."
+	echo >&2 "  -i    The name for the Docker image."
+	echo >&2 "  -l    (Optional) The Artifact Registry region. Defaults to 'us-central1'."
+	echo >&2 "  -v    (Optional) The version tag for the image. Defaults to YYYY-MM-DD."
+	echo >&2 "  -h    Display this help message."
+	echo >&2 ""
+	echo >&2 "Example (default region):"
+	echo >&2 "  bash build-and-push-cluster-health-check.sh -p gpu-test-project -r cluster-health-check-repo -i cluster-health-check"
+	echo >&2 ""
+	echo >&2 "Example (specific region):"
+	echo >&2 "  bash build-and-push-cluster-health-check.sh -p gpu-test-project -r cluster-health-check-repo -i cluster-health-check -l us-east4"
+	exit 1
+}
+
+PROJECT=""
+REPO=""
+IMAGE=""
+REGION="us-central1"
+VERSION=""
+
+while getopts ":p:r:i:l:v:h" opt; do
+	case ${opt} in
+	p)
+		PROJECT=$OPTARG
+		;;
+	r)
+		REPO=$OPTARG
+		;;
+	i)
+		IMAGE=$OPTARG
+		;;
+	l)
+		REGION=$OPTARG
+		;;
+	v)
+		VERSION=$OPTARG
+		;;
+	h)
+		usage
+		;;
+	\?)
+		echo "Invalid Option: -$OPTARG" >&2
+		usage
+		;;
+	:)
+		echo "Invalid Option: -$OPTARG requires an argument." >&2
+		usage
+		;;
+	esac
+done
+
+if [[ -z "${PROJECT}" ]] || [[ -z "${REPO}" ]] || [[ -z "${IMAGE}" ]]; then
+	echo "Error: Missing required arguments." >&2
+	usage
+fi
+
+if [[ -z "${VERSION}" ]]; then
+	VERSION=$(date +%Y-%m-%d)
+fi
+REMOTE_DESTINATION="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${IMAGE}"
+
+echo "================================================="
+echo "Configuration received:"
+echo "  Project ID:       ${PROJECT}"
+echo "  Repository Name:  ${REPO}"
+echo "  Image Name:       ${IMAGE}"
+echo "  Generated Version:  ${VERSION}"
+echo "  Remote Destination: ${REMOTE_DESTINATION}"
+echo "================================================="
+
+set -x
+
+echo "Building image ${IMAGE}:${VERSION}"
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+
+echo "Vendoring Go dependencies..."
+(cd "${SCRIPT_DIR}" && go mod vendor)
+
+docker buildx build --platform linux/amd64,linux/arm64 \
+	-t "${REMOTE_DESTINATION}:${VERSION}" \
+	-t "${REMOTE_DESTINATION}:latest" \
+	--target dcgm-healthcheck \
+	--push \
+	-f "${SCRIPT_DIR}/Dockerfile" \
+	"${SCRIPT_DIR}"