Merge pull request GoogleCloudPlatform#4441 from samskillman/gcs-fuse-optimization

samskillman · web-flow · commit 4aa15762abc3 · 2025-07-30T22:05:23.000-06:00
Add optimized gcsfuse configurations to A3 Ultra and A4 blueprints.
diff --git a/examples/machine-learning/a3-ultragpu-8g/README.md b/examples/machine-learning/a3-ultragpu-8g/README.md
@@ -6,6 +6,21 @@ see:
 
 [Create A3 Ultra Slurm Cluster](https://cloud.google.com/ai-hypercomputer/docs/create/create-slurm-cluster)
 
+### Cloud Storage FUSE
+
+This blueprint includes four Cloud Storage FUSE mounts to provide a simple and scalable way
+to manage data.
+
+1. `/gcs` is a general purpose mount that can be used for shared tools and data.
+1. `/gcs-checkpoints` is an optimized mount for writing and reading checkpoints. It
+    uses the local SSD for caching and enables parallel downloads to improve
+    performance.
+1. `/gcs-training-data` is an optimized mount for reading training data. It can
+   be further tuned if the training data fits fully within the local ssd
+   storage.
+1. `/gcs-model-serving` is an optimized mount for serving models, which
+   downloads model weights in parallel to local ssd.
+
 ## VMs without scheduler
 
 To test workloads directly on A3 Ultra VMs, you can deploy the [a3ultra-vm.yaml]:
@@ -24,8 +39,10 @@ Cluster toolkit also supports DWS Flex-Start, Spot VMs, as well as reservations
 [For more information on DWS Flex-Start in Slurm](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/docs/slurm-dws-flex.md)
 [For more information on Spot VMs](https://cloud.google.com/compute/docs/instances/spot)
 
-We provide ways to enable the alternative provisioning models in the `a3ultra-slurm-deployment.yaml` file.
+We provide ways to enable the alternative provisioning models in the
+`a3ultra-slurm-deployment.yaml` file.
 
-To make use of these other models, replace `a3u_reservation_name` in the deployment file with the variable of choice below.
+To make use of these other models, replace `a3u_reservation_name` in the
+deployment file with the variable of choice below.
 
 `a3u_enable_spot_vm: true` for spot or `a3u_dws_flex_enabled: true` for DWS Flex-Start.
diff --git a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml
@@ -246,6 +246,7 @@ deployment_groups:
       source_image_project_id: [$(vars.base_image.project)]
       image_family: $(vars.instance_image.family)
       omit_external_ip: false
+
       # Unattended upgrades are disabled in this blueprint so that software does not
       # get updated daily and lead to potential instability in the cluster environment.
       #
@@ -355,6 +356,98 @@ deployment_groups:
   #   outputs:
   #   - network_storage
 
+  # The following four modules create and mount a Cloud Storage Bucket with
+  # gcsfuse.  They are optional but recommended for many use cases.
+  # (Optional) The following creates a GCS bucket that will be mounted
+  # using gcsfuse. If you prefer to use a pre-existing bucket, use the
+  # modules/file-system/pre-existing-network-storage module.
+  - id: gcs_bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings:
+      enable_hierarchical_namespace: true
+      local_mount: /gcs
+      random_suffix: true
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for checkpoint writing/reading Uses
+  # local-ssd for and enables parallel downloads.  For more information on
+  # these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  - id: gcs_checkpoints
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-checkpoints
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      file-cache-max-size-mb=-1,\
+                      file-cache-cache-file-for-range-read=true,\
+                      file-cache-enable-parallel-downloads=true,\
+                      cache-dir=/mnt/localssd,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for reading training data.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # If your training dataset fits on localssd, then you may want to enable file
+  # cache as well, which is done by adding
+  # cache-dir=/mnt/localssd,\
+  # file-cache-max-size-mb=<DATASET_SIZE>,\
+  # file-cache-cache-file-for-range-read=true,\
+  # to the mount_options.
+  - id: gcs_training_data
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-training-data
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for model serving.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  - id: gcs_model_serving
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-model-serving
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      cache-dir=/mnt/localssd,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      file-cache-max-size-mb=-1,\
+                      file-cache-cache-file-for-range-read=true,\
+                      file-cache-enable-parallel-downloads=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
 - group: cluster
   modules:
   - id: a3ultra_startup
@@ -371,6 +464,12 @@ deployment_groups:
             "data-root": "$(vars.local_ssd_mountpoint)/docker"
           }
       runners:
+      - $(gcs_checkpoints.client_install_runner)
+      - $(gcs_checkpoints.mount_runner)
+      - $(gcs_training_data.client_install_runner)
+      - $(gcs_training_data.mount_runner)
+      - $(gcs_model_serving.client_install_runner)
+      - $(gcs_model_serving.mount_runner)
       - type: data
         destination: /etc/enroot/enroot.conf
         content: |
@@ -567,6 +666,7 @@ deployment_groups:
     - a3_ultra_partition
     - slurm_login
     - homefs
+    - gcs_bucket
     settings:
       enable_controller_public_ips: true
       disk_type: pd-extreme
diff --git a/examples/machine-learning/a4-highgpu-8g/README.md b/examples/machine-learning/a4-highgpu-8g/README.md
@@ -6,6 +6,21 @@ see:
 
 [Create an AI-optimized Slurm cluster](https://cloud.google.com/ai-hypercomputer/docs/create/create-slurm-cluster)
 
+### Cloud Storage FUSE
+
+This blueprint includes four Cloud Storage FUSE mounts to provide a simple and scalable way
+to manage data.
+
+1. `/gcs` is a general purpose mount that can be used for shared tools and data.
+1. `/gcs-checkpoints` is an optimized mount for writing and reading checkpoints. It
+    uses the local SSD for caching and enables parallel downloads to improve
+    performance.
+1. `/gcs-training-data` is an optimized mount for reading training data. It can
+   be further tuned if the training data fits fully within the local ssd
+   storage.
+1. `/gcs-model-serving` is an optimized mount for serving models, which
+   downloads model weights in parallel to local ssd.
+
 ## A4-High VMs
 
 ### Build the Cluster Toolkit gcluster binary
@@ -66,9 +81,11 @@ Cluster toolkit also supports DWS Flex-Start, Spot VMs, as well as reservations
 [For more information on DWS Flex-Start in Slurm](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/docs/slurm-dws-flex.md)
 [For more information on Spot VMs](https://cloud.google.com/compute/docs/instances/spot)
 
-We provide ways to enable the alternative provisioning models in the `a4high-slurm-deployment.yaml` file.
+We provide ways to enable the alternative provisioning models in the
+`a4high-slurm-deployment.yaml` file.
 
-To make use of these other models, replace `a4h_reservation_name` in the deployment file with the variable of choice below.
+To make use of these other models, replace `a4h_reservation_name` in the
+deployment file with the variable of choice below.
 
 `a4h_enable_spot_vm: true` for spot or `a4h_dws_flex_enabled: true` for DWS Flex-Start.
 
diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml
@@ -356,6 +356,98 @@ deployment_groups:
   #   outputs:
   #   - network_storage
 
+  # The following four modules create and mount a Cloud Storage Bucket with
+  # gcsfuse.  They are optional but recommended for many use cases.
+  # (Optional) The following creates a GCS bucket that will be mounted
+  # using gcsfuse. If you prefer to use a pre-existing bucket, use the
+  # modules/file-system/pre-existing-network-storage module.
+  - id: gcs_bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings:
+      enable_hierarchical_namespace: true
+      local_mount: /gcs
+      random_suffix: true
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for checkpoint writing/reading Uses
+  # local-ssd for and enables parallel downloads.  For more information on
+  # these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  - id: gcs_checkpoints
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-checkpoints
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      file-cache-max-size-mb=-1,\
+                      file-cache-cache-file-for-range-read=true,\
+                      file-cache-enable-parallel-downloads=true,\
+                      cache-dir=/mnt/localssd,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for reading training data.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # If your training dataset fits on localssd, then you may want to enable file
+  # cache as well, which is done by adding
+  # cache-dir=/mnt/localssd,\
+  # file-cache-max-size-mb=<DATASET_SIZE>,\
+  # file-cache-cache-file-for-range-read=true,\
+  # to the mount_options.
+  - id: gcs_training_data
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-training-data
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      enable-streaming-writes=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
+  # (Optional) Create a mount-point optimized for model serving.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  - id: gcs_model_serving
+    source: modules/file-system/pre-existing-network-storage
+    settings:
+      remote_mount: $(gcs_bucket.gcs_bucket_name)
+      local_mount: /gcs-model-serving
+      fs_type: gcsfuse
+      mount_options: "implicit-dirs,\
+                      cache-dir=/mnt/localssd,\
+                      metadata-cache-negative-ttl-secs=0,\
+                      metadata-cache-ttl-secs=-1,\
+                      stat-cache-max-size-mb=-1,\
+                      type-cache-max-size-mb=-1,\
+                      file-cache-max-size-mb=-1,\
+                      file-cache-cache-file-for-range-read=true,\
+                      file-cache-enable-parallel-downloads=true,\
+                      dir-mode=777,\
+                      file-mode=777,\
+                      allow_other"
+
 - group: cluster
   modules:
   - id: a4high_startup
@@ -372,6 +464,12 @@ deployment_groups:
             "data-root": "$(vars.local_ssd_mountpoint)/docker"
           }
       runners:
+      - $(gcs_checkpoints.client_install_runner)
+      - $(gcs_checkpoints.mount_runner)
+      - $(gcs_training_data.client_install_runner)
+      - $(gcs_training_data.mount_runner)
+      - $(gcs_model_serving.client_install_runner)
+      - $(gcs_model_serving.mount_runner)
       - type: data
         destination: /etc/enroot/enroot.conf
         content: |
@@ -573,6 +671,7 @@ deployment_groups:
     - a4high_partition
     - slurm_login
     - homefs
+    - gcs_bucket
     settings:
       enable_controller_public_ips: true
       disk_type: pd-extreme
diff --git a/tools/cloud-build/daily-tests/builds/ml-a3-ultragpu-slurm.yaml b/tools/cloud-build/daily-tests/builds/ml-a3-ultragpu-slurm.yaml
@@ -15,6 +15,8 @@
 ---
 tags:
 - m.custom-image
+- m.cloud-storage-bucket
+- m.pre-existing-network-storage
 - m.filestore
 - m.gpu-rdma-vpc
 - m.schedmd-slurm-gcp-v6-controller
diff --git a/tools/cloud-build/daily-tests/builds/ml-a4-highgpu-slurm-flex.yaml b/tools/cloud-build/daily-tests/builds/ml-a4-highgpu-slurm-flex.yaml
@@ -15,6 +15,8 @@
 ---
 tags:
 - m.custom-image
+- m.cloud-storage-bucket
+- m.pre-existing-network-storage
 - m.filestore
 - m.gpu-rdma-vpc
 - m.schedmd-slurm-gcp-v6-controller
diff --git a/tools/cloud-build/daily-tests/builds/ml-a4-highgpu-slurm.yaml b/tools/cloud-build/daily-tests/builds/ml-a4-highgpu-slurm.yaml
@@ -15,6 +15,8 @@
 ---
 tags:
 - m.custom-image
+- m.cloud-storage-bucket
+- m.pre-existing-network-storage
 - m.filestore
 - m.gpu-rdma-vpc
 - m.schedmd-slurm-gcp-v6-controller
diff --git a/tools/cloud-build/daily-tests/tests/ml-a3-ultragpu-slurm.yml b/tools/cloud-build/daily-tests/tests/ml-a3-ultragpu-slurm.yml
@@ -40,6 +40,7 @@ custom_vars:
   - a3ultra
   mounts:
   - /home
+  - /gcs
 cli_deployment_vars:
   region: "{{ region }}"
   zone: "{{ zone }}"
diff --git a/tools/cloud-build/daily-tests/tests/ml-a4-highgpu-slurm.yml b/tools/cloud-build/daily-tests/tests/ml-a4-highgpu-slurm.yml
@@ -40,6 +40,7 @@ custom_vars:
   - a4high
   mounts:
   - /home
+  - /gcs
 cli_deployment_vars:
   region: "{{ region }}"
   zone: "{{ zone }}"
