Merge pull request #4781 from GoogleCloudPlatform/release-candidate

Release candidate: v1.70.0

Author: aslam-quad

cmd/root.go

@@ -53,7 +53,7 @@ HPC deployments on the Google Cloud Platform.`,
 				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.69.0",
+		Version:     "v1.70.0",
 		Annotations: annotation,
 	}
 )

community/examples/gke-tpu-v6/README.md

@@ -90,6 +90,38 @@ This section guides you through the cluster creation process, ensuring that your
     community/examples/gke-tpu-v6/gke-tpu-v6.yaml
    ```
 
+## Advanced Blueprint: GKE TPU with GCS Integration
+
+This repository also includes an advanced blueprint, `gke-tpu-v6-gcs.yaml`, designed for production-ready workloads. It builds on the basic blueprint by adding several key features:
+* **Dedicated Service Accounts** for nodes and workloads, following security best practices.
+* **Automatic creation of two GCS buckets** for training data and checkpoints.
+* **Performance-tuned GCS FUSE mounts** pre-configured in the cluster as Persistent Volumes.
+
+### Deploying the Advanced Blueprint
+
+The process is nearly identical to the basic deployment.
+
+1. Ensure you have completed steps 1-7 from the "Create a cluster" section above. The same `gke-tpu-v6-deployment.yaml` file can be used.
+
+1. In the final deploy command, simply point to the `gke-tpu-v6-advanced.yaml` blueprint instead.
+
+    ```sh
+    cd ~/cluster-toolkit
+    ./gcluster deploy -d \
+    community/examples/gke-tpu-v6/gke-tpu-v6-deployment.yaml \
+    community/examples/gke-tpu-v6/gke-tpu-v6-advanced.yaml
+    ```
+
+1. After deployment, the blueprint will output instructions for running a fio benchmark job. This job serves as a validation test to confirm that the GCS mounts are working correctly for both reading and writing. Follow the printed instructions to run the test.
+
+### Understanding the GCS Integration
+
+The advanced blueprint provisions several key technologies to create a robust data pipeline for your TPU workloads. Here are some resources to understand how they work together:
+* [Cloud Storage Overview](https://cloud.google.com/storage/docs/introduction#quickstarts): Start here to understand what Cloud Storage buckets are and their role in storing large-scale data.
+* [Cloud TPU Storage Options](https://cloud.google.com/tpu/docs/storage-options): Learn about the recommended storage patterns for Cloud TPUs, including why GCS FUSE is a best practice for providing training data.
+* [Access GCS Buckets with the GCS FUSE CSI Driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/cloud-storage-fuse-csi-driver): This is the core technical guide explaining how GKE mounts GCS buckets into your pods, which this blueprint automates.
+* [Configure Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity): Read this to understand the secure, recommended method for GKE applications to access Google Cloud services like GCS, which this blueprint configures for you.
+
 ## Run the sample job
 
 The [tpu-available-chips.yaml](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/community/examples/gke-tpu-v6/tpu-available-chips.yaml) file creates a service and a job resource in kubernetes. It is based on https://cloud.google.com/kubernetes-engine/docs/how-to/tpus#tpu-chips-node-pool. The  workload returns the number of TPU chips across all of the nodes in a multi-host TPU slice.

community/examples/gke-tpu-v6/gke-tpu-v6-advanced.yaml

@@ -0,0 +1,287 @@
+# Copyright 2025 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+blueprint_name: gke-tpu-v6
+
+vars:
+  # The following variables should be over-written in the deployment.yaml file.
+  # Your GCP Project ID
+  project_id:
+
+  # This should be unique across all of your Cluster
+  # Toolkit Deployments.
+  deployment_name: gke-tpu-v6
+
+  # The GCP Region used for this deployment.
+  region:
+
+  # The GCP Zone used for this deployment.
+  zone:
+
+  # The number of TPU slices to create
+  num_slices:
+
+  # Machine type
+  machine_type:
+
+  # The TPU placement topology for pod slice node pool.
+  tpu_topology:
+
+  # The number of nodes to be created in each nodepool
+  static_node_count:
+
+  # Cidr block containing the IP of the machine calling terraform.
+  # To allow all (IAM restrictions still enforced), use 0.0.0.0/0
+  # To allow only your IP address, use <YOUR-IP-ADDRESS>/32
+  authorized_cidr:
+
+  # The name of the compute engine reservation of TPU v6 nodes
+  reservation:
+
+  system_node_pool_disk_size_gb: 200
+  v6e_node_pool_disk_size_gb: 100
+
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: gke-tpu-v6-net
+    source: modules/network/vpc
+    settings:
+      subnetworks:
+      - subnet_name: $(vars.deployment_name)-sub-0
+        subnet_region: $(vars.region)
+        subnet_ip: 192.168.0.0/18
+      secondary_ranges_list:
+      - subnetwork_name: $(vars.deployment_name)-sub-0
+        ranges:
+        - range_name: pods
+          ip_cidr_range: 10.4.0.0/14
+        - range_name: services
+          ip_cidr_range: 10.0.32.0/20
+      firewall_rules:
+      - name: $(vars.deployment_name)-internal-0
+        ranges: [192.168.0.0/18]
+        allow:
+        - protocol: tcp
+          ports: ["0-65535"]
+        - protocol: udp
+          ports: ["0-65535"]
+        - protocol: icmp
+
+  - id: node_pool_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-np-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectViewer
+      - artifactregistry.reader
+
+  - id: workload_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-wl-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectAdmin
+      - artifactregistry.reader
+      - container.admin
+
+  - id: training_bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings:
+      local_mount: /training-data
+      name_prefix: training-data
+      random_suffix: true
+      force_destroy: false
+      enable_hierarchical_namespace: true
+
+  - id: checkpoint_bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings:
+      local_mount: /checkpoint-data
+      name_prefix: checkpoint-data
+      random_suffix: true
+      force_destroy: false
+      enable_hierarchical_namespace: true
+
+  - id: gke-tpu-v6-cluster
+    source: modules/scheduler/gke-cluster
+    use: [gke-tpu-v6-net, workload_service_account]
+    settings:
+      system_node_pool_machine_type: "n2-standard-8"
+      system_node_pool_disk_size_gb: $(vars.system_node_pool_disk_size_gb)
+      system_node_pool_taints: []
+      enable_private_endpoint: false # Allows access from authorized public IPs
+      configure_workload_identity_sa: true
+      enable_gcsfuse_csi: true
+      master_authorized_networks:
+      - cidr_block: $(vars.authorized_cidr) # Allows your machine to run the kubectl command. Required for multi network setup.
+        display_name: "kubectl-access-network"
+      # Cluster versions cannot be updated through the toolkit after creation
+      # Please manage cluster version from the Google Cloud Console directly
+      version_prefix: "1.32."
+      release_channel: RAPID
+      maintenance_exclusions:
+      - name: no-minor-or-node-upgrades-indefinite
+        start_time: "2024-12-01T00:00:00Z"
+        end_time: "2025-12-22T00:00:00Z"
+        exclusion_scope: NO_MINOR_OR_NODE_UPGRADES
+    outputs: [instructions]
+
+  - id: gke-tpu-v6-pool
+    source: modules/compute/gke-node-pool
+    use: [gke-tpu-v6-cluster, node_pool_service_account]
+    settings:
+      num_slices: $(vars.num_slices)
+      name: gke-tpu-v6-pool
+      disk_type: hyperdisk-balanced
+      machine_type: $(vars.machine_type)
+      auto_upgrade: true
+      zones: [$(vars.zone)]
+      disk_size_gb: $(vars.v6e_node_pool_disk_size_gb)
+      static_node_count: $(vars.static_node_count)
+      reservation_affinity:
+        consume_reservation_type: SPECIFIC_RESERVATION
+        specific_reservations:
+        - name: $(vars.reservation)
+      placement_policy:
+        type: COMPACT
+        tpu_topology: $(vars.tpu_topology)
+    outputs: [instructions]
+
+  - id: workload-manager-install
+    source: modules/management/kubectl-apply
+    use: [gke-tpu-v6-cluster]
+    settings:
+      jobset:
+        install: true
+
+  - id: gcs-training
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(training_bucket.gcs_bucket_name)
+      local_mount: /training-data
+      fs_type: gcsfuse
+      mount_options: >-
+        implicit-dirs,
+        metadata-cache:ttl-secs:-1,
+        metadata-cache:stat-cache-max-size-mb:-1,
+        metadata-cache:type-cache-max-size-mb:-1,
+        file-cache:max-size-mb:-1,
+        file-cache:cache-file-for-range-read:true
+
+  - id: gcs-checkpointing
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(checkpoint_bucket.gcs_bucket_name)
+      local_mount: /checkpoint-data
+      fs_type: gcsfuse
+      mount_options: >-
+        implicit-dirs,
+        metadata-cache:ttl-secs:-1,
+        metadata-cache:stat-cache-max-size-mb:-1,
+        metadata-cache:type-cache-max-size-mb:-1,
+        file-cache:max-size-mb:-1,
+        file-cache:cache-file-for-range-read:true,
+        file-cache:enable-parallel-downloads:true,
+        rename-dir-limit=200000
+
+  # Persistent Volume for training data
+  - id: training-pv
+    source: modules/file-system/gke-persistent-volume
+    use: [gcs-training, gke-tpu-v6-cluster]
+    settings:
+      gcs_bucket_name: $(training_bucket.gcs_bucket_name)
+      capacity_gib: 1000000
+
+  # Persistent Volume for checkpoint data
+  - id: checkpointing-pv
+    source: modules/file-system/gke-persistent-volume
+    use: [gcs-checkpointing, gke-tpu-v6-cluster]
+    settings:
+      gcs_bucket_name: $(checkpoint_bucket.gcs_bucket_name)
+      capacity_gib: 1000000
+
+  # This is an example job that will install and run an `fio`
+  # benchmark against the training and checkpointing buckets.
+  - id: fio-bench-job-template
+    source: modules/compute/gke-job-template
+    use: [checkpointing-pv, training-pv, gke-tpu-v6-pool]
+    settings:
+      node_count: $(vars.static_node_count)
+      security_context:  # to make sure the job have enough access to install the fio packages
+      - key: runAsUser
+        value: 0
+      - key: runAsGroup
+        value: 100
+      - key: fsGroup
+        value: 100
+
+      k8s_service_account_name: workload-identity-k8s-sa
+      requested_cpu_per_pod: 8
+      image: ubuntu:latest
+
+      command:
+      - bash
+      - -c
+      - |
+
+        set -eux
+        export DEBIAN_FRONTEND=noninteractive
+
+        # Install fio
+        apt update -y && apt install -y fio
+
+        # Use a tag to create a unique path for tests
+        TAG=`date +%s`
+
+        # Verify mountpoints
+        df -h
+        mountpoint /checkpoint-data
+        mountpoint /training-data
+
+        # Create temporary directory for fio benchmarks
+        mkdir -p /{training,checkpoint}-data/fio-benchmarks-${TAG}
+
+        # The following will take roughly 5 minutes to complete
+
+        # Test reading from the training bucket
+        fio --ioengine=libaio --filesize=1G --ramp_time=2s --runtime=30s \
+          --numjobs=8 --create_serialize=0 --direct=1 --verify=0 \
+          --randrepeat=0 --group_reporting --directory=/training-data/fio-benchmarks-${TAG} \
+          --name=training-read --blocksize=1m --iodepth=16 --readwrite=randread
+
+        # Test writing to the checkpointing bucket
+        fio --ioengine=libaio --filesize=1G --ramp_time=2s --runtime=30s \
+          --numjobs=8 --create_serialize=0 --direct=1 --verify=0 \
+          --randrepeat=0 --group_reporting --directory=/checkpoint-data/fio-benchmarks-${TAG} \
+          --name=checkpoint-write --blocksize=10m --iodepth=16 --readwrite=write
+
+        # Perform checkpoint data reading performance test
+        fio --ioengine=libaio --filesize=1G --ramp_time=2s --runtime=30s \
+          --numjobs=8 --create_serialize=0 --direct=1 --verify=0 \
+          --randrepeat=0 --group_reporting --directory=/checkpoint-data/fio-benchmarks-${TAG} \
+          --name=checkpoint-read --blocksize=10m --iodepth=16 --readwrite=read
+
+        rm -rf /{training,checkpoint}-data/fio-benchmarks-${TAG}
+
+    outputs: [instructions]

community/examples/xpk-n2-filestore/xpk-n2-filestore.yaml

@@ -32,7 +32,7 @@ vars:
   slice_cpu_count: 64
   xpk_version: v0.8.0
   kueue_version: v0.11.1
-  jobset_version: v0.7.2
+  jobset_version: 0.10.1
   kjob_version: f775609365df47bdc3e7c10290d8efae7f512464
 
 deployment_groups:

community/modules/compute/htcondor-execute-point/versions.tf

@@ -29,6 +29,6 @@ terraform {
   }
 
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.70.0"
   }
 }

community/modules/compute/mig/versions.tf

@@ -22,6 +22,6 @@ terraform {
     }
   }
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:mig/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:mig/v1.70.0"
   }
 }

community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf

@@ -17,6 +17,6 @@
 terraform {
   required_version = ">= 1.3"
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.70.0"
   }
 }

community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf

@@ -18,6 +18,6 @@ terraform {
   required_version = ">= 1.3"
 
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.70.0"
   }
 }

community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf

@@ -24,6 +24,6 @@ terraform {
     }
   }
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.70.0"
   }
 }

community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf

@@ -18,6 +18,6 @@ terraform {
   required_version = ">= 1.3"
 
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.69.0"
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.70.0"
   }
 }