Merge pull request #4669 from GoogleCloudPlatform/release-candidate

Release candidate v1.67.0

Author: Priya-Quad

cmd/root.go

@@ -53,7 +53,7 @@ HPC deployments on the Google Cloud Platform.`,
 				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.66.0",
+		Version:     "v1.67.0",
 		Annotations: annotation,
 	}
 )

community/examples/gke-tpu-v6/tpu-multislice.yaml

@@ -41,7 +41,7 @@ spec:
               cloud.google.com/gke-tpu-topology: 4x4
             containers:
             - name: jax-tpu
-              image: python:3.10
+              image: us-docker.pkg.dev/cloud-tpu-images/jax-ai-image/tpu:latest
               ports:
               - containerPort: 8471
               - containerPort: 8080
@@ -52,7 +52,6 @@ spec:
               - bash
               - -c
               - |
-                pip install "jax[tpu]" -f https://storage.googleapis.com/jax-releases/libtpu_releases.html
                 python -c 'import jax; print("Global device count:", jax.device_count())'
                 sleep 60
               resources:

community/gce-cos-nvidia-bug-report/Dockerfile

@@ -21,8 +21,8 @@ WORKDIR /app
 # Copy the local directory to the container
 COPY app/* /app/
 
-ENV PATH $PATH:/var/lib/nvidia/bin:/var/lib/nvidia/lib64:/usr/local/nvidia/bin:/usr/local/nvidia/lib64/
-ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/var/lib/nvidia/bin:/var/lib/nvidia/lib64:/usr/local/nvidia/bin:/usr/local/nvidia/lib64
+ENV PATH=$PATH:/var/lib/nvidia/bin:/var/lib/nvidia/lib64:/usr/local/nvidia/bin:/usr/local/nvidia/lib64/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/lib/nvidia/bin:/var/lib/nvidia/lib64:/usr/local/nvidia/bin:/usr/local/nvidia/lib64
 
 
 # Install system dependencies

community/gce-cos-nvidia-bug-report/README.md

@@ -166,17 +166,11 @@ docker run \
   --name gce-cos-bug-report \
   --pull=always \
   --privileged \
+  --network=host \
   --volume /etc:/etc_host \
   --volume /tmp:/tmp \
   --volume /var/lib/nvidia:/usr/local/nvidia \
-  --device /dev/nvidia0:/dev/nvidia0 \
-  --device /dev/nvidia1:/dev/nvidia1 \
-  --device /dev/nvidia2:/dev/nvidia2 \
-  --device /dev/nvidia3:/dev/nvidia3 \
-  --device /dev/nvidia4:/dev/nvidia4 \
-  --device /dev/nvidia5:/dev/nvidia5 \
-  --device /dev/nvidia6:/dev/nvidia6 \
-  --device /dev/nvidia7:/dev/nvidia7 \
+  $(find /dev -regextype posix-extended -regex '/dev/nvidia[0-9]+' | xargs -I {} echo '--device={}:{}')  \
   --device /dev/nvidia-uvm:/dev/nvidia-uvm \
   --device /dev/nvidiactl:/dev/nvidiactl \
 us-central1-docker.pkg.dev/gce-ai-infra/gce-cos-nvidia-bug-report-repo/gce-cos-nvidia-bug-report:latest \

community/gce-cos-nvidia-bug-report/app/constants.py

@@ -25,8 +25,8 @@
 NVIDIA_SMI_BINARY_NAME = "nvidia-smi"
 DUPLICATED_NVIDIA_BUG_REPORT_SCRIPT_PATH = "/app/nvidia-bug-report.sh"
 
-# Sample MFT filename for ARM: "mft-4.30.1-113-arm64-deb.tgz"
-# Sample MFT filename for x86: "mft-4.30.1-113-x86_64-deb.tgz"
+# Sample MFT filename for ARM: "mft-kernel-modules-4.32.0-120-aarch64.tgz"
+# Sample MFT filename for x86: "mft-kernel-modules-4.32.0-120-x86_64.tgz"
 MFT_DOWNLOAD_URL_PREFIX = "https://www.mellanox.com/downloads/MFT/"
 MFT_FILENAME_PREFIX = "mft"
 MFT_FILENAME_SUFFIX = "deb.tgz"

community/gce-cos-nvidia-bug-report/app/gce-cos-nvidia-bug-report.py

@@ -144,7 +144,10 @@ def GetArchitectureFromMachineType(machine_type: str) -> Architecture:
     The architecture of the machine type. This can be either X86 or ARM.
   """
   machine_type = machine_type.lower().split("-")
-  return Architecture.ARM if machine_type[0].endswith("a") else Architecture.X86
+  if machine_type[0].lower().endswith("x"):
+    return Architecture.ARM
+  else:
+    return Architecture.X86
 
 
 @retry.retry(
@@ -317,13 +320,19 @@ def UploadArtifactsToGcs(
 
   storage_client = storage.Client()
 
-  # Create the bucket if it does not exist.
-  bucket = storage_client.bucket(bucket_name)
-  try:
-    storage_client.create_bucket(bucket)
-    logging.info("Created bucket [%s] in the project.", bucket_name)
-  except google_cloud_exceptions.Conflict:
-    logging.info("Bucket [%s] already exists in the project.", bucket_name)
+  bucket = storage_client.lookup_bucket(bucket_name)
+  if bucket is None:
+    # Create the bucket if it does not exist.
+    try:
+      storage_client.create_bucket(bucket)
+      logging.info("Created bucket [%s] in the project.", bucket_name)
+    except google_cloud_exceptions.exceptions.GoogleAPIError as e:
+      logging.info(
+          "Bucket creation failed for %s: %s. Skipping GCS upload.",
+          bucket_name,
+          e,
+      )
+      return
 
   for artifact in artifacts_to_upload:
     UploadArtifactToGcs(
@@ -470,7 +479,7 @@ def DownloadFileFromUrl(url: str, destination_path: str) -> None:
 
 def InstallMftUserspaceSoftware(
     architecture: Architecture, mft_version: str, mft_build_version: str
-):
+) -> None:
   """Installs Mellanox Firmware Tool (MFT) userspace software on the instance.
 
   Args:
@@ -682,7 +691,7 @@ def InsertMftKernelModulesIfNotLoaded(
 
 def IdentifyMftKernelModules(
     gce_instance_info: GceInstanceInfo,
-) -> storage.Blob:
+):
   """Downloads the Mellanox Firmware Tool (MFT) kernel modules.
 
   Args:

community/gce-cos-nvidia-bug-report/build-and-push-gce-cos-nvidia-bug-report.sh

@@ -87,12 +87,9 @@ echo "================================================="
 set -x
 
 echo "Building image ${IMAGE}:${VERSION}"
-docker build -t "${IMAGE}:${VERSION}" .
 
-echo "Tagging image for Artifact Registry: ${REMOTE_DESTINATION}:${VERSION}"
-docker tag "${IMAGE}:${VERSION}" "${REMOTE_DESTINATION}:${VERSION}"
-docker tag "${IMAGE}:${VERSION}" "${REMOTE_DESTINATION}:latest"
-
-echo "Pushing image ${REMOTE_DESTINATION}:${VERSION}"
-docker push "${REMOTE_DESTINATION}:${VERSION}"
-docker push "${REMOTE_DESTINATION}:latest"
+docker buildx build --platform linux/amd64,linux/arm64 \
+	-t "${REMOTE_DESTINATION}:${VERSION}" \
+	-t "${REMOTE_DESTINATION}:latest" \
+	--push \
+	.

community/modules/compute/htcondor-execute-point/versions.tf

@@ -29,6 +29,6 @@ terraform {
   }
 
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.66.0"
+    module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.67.0"
   }
 }

community/modules/compute/mig/versions.tf

@@ -22,6 +22,6 @@ terraform {
     }
   }
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:mig/v1.66.0"
+    module_name = "blueprints/terraform/hpc-toolkit:mig/v1.67.0"
   }
 }

community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf

@@ -17,6 +17,6 @@
 terraform {
   required_version = ">= 1.3"
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.66.0"
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.67.0"
   }
 }