Switch to using gcsfuse profile feature in aiml gcs-bucket mounts in slurm cluster blueprints (GoogleCloudPlatform#5047)

Author: cboneti

examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml

@@ -362,44 +362,39 @@ deployment_groups:
   # (Optional) The following creates a GCS bucket that will be mounted
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
   - id: gcs_bucket
     source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
       random_suffix: true
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "implicit_dirs,\
+                      metadata_cache_negative_ttl_secs=0,\
+                      metadata_cache_ttl_secs=-1,\
+                      stat_cache_max_size_mb=-1,\
+                      type_cache_max_size_mb=-1,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for checkpoint writing/reading Uses
   # local-ssd for and enables parallel downloads.  For more information on
   # these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_checkpoints
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-checkpoints
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      cache-dir=/mnt/localssd,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-checkpointing,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for reading training data.
@@ -411,42 +406,36 @@ deployment_groups:
   # file-cache-max-size-mb=<DATASET_SIZE>,\
   # file-cache-cache-file-for-range-read=true,\
   # to the mount_options.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_training_data
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-training-data
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-training,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for model serving.
   # For more information on these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_model_serving
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-model-serving
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      cache-dir=/mnt/localssd,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-serving,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
 - group: cluster

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -367,44 +367,39 @@ deployment_groups:
   # (Optional) The following creates a GCS bucket that will be mounted
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
   - id: gcs_bucket
     source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
       random_suffix: true
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "implicit_dirs,\
+                      metadata_cache_negative_ttl_secs=0,\
+                      metadata_cache_ttl_secs=-1,\
+                      stat_cache_max_size_mb=-1,\
+                      type_cache_max_size_mb=-1,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for checkpoint writing/reading Uses
   # local-ssd for and enables parallel downloads.  For more information on
   # these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_checkpoints
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-checkpoints
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      cache-dir=/mnt/localssd,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-checkpointing,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for reading training data.
@@ -416,42 +411,36 @@ deployment_groups:
   # file-cache-max-size-mb=<DATASET_SIZE>,\
   # file-cache-cache-file-for-range-read=true,\
   # to the mount_options.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_training_data
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-training-data
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-training,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for model serving.
   # For more information on these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_model_serving
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-model-serving
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      cache-dir=/mnt/localssd,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-serving,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
 - group: cluster

examples/machine-learning/a4x-highgpu-4g/a4xhigh-slurm-blueprint.yaml

@@ -375,44 +375,39 @@ deployment_groups:
   # (Optional) The following creates a GCS bucket that will be mounted
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
   - id: gcs_bucket
     source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
       random_suffix: true
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "implicit_dirs,\
+                      metadata_cache_negative_ttl_secs=0,\
+                      metadata_cache_ttl_secs=-1,\
+                      stat_cache_max_size_mb=-1,\
+                      type_cache_max_size_mb=-1,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for checkpoint writing/reading Uses
   # local-ssd for and enables parallel downloads.  For more information on
   # these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_checkpoints
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-checkpoints
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      cache-dir=/mnt/localssd,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-checkpointing,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for reading training data.
@@ -424,42 +419,36 @@ deployment_groups:
   # file-cache-max-size-mb=<DATASET_SIZE>,\
   # file-cache-cache-file-for-range-read=true,\
   # to the mount_options.
+  # For more information on these flags, see
+  # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_training_data
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-training-data
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      enable-streaming-writes=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-training,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"
 
   # (Optional) Create a mount-point optimized for model serving.
   # For more information on these flags, see
   # https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/samples/gcsfuse_config
+  # and
+  # https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/profile-based-configurations
   - id: gcs_model_serving
     source: modules/file-system/pre-existing-network-storage
     settings:
       remote_mount: $(gcs_bucket.gcs_bucket_name)
       local_mount: /gcs-model-serving
       fs_type: gcsfuse
-      mount_options: "implicit-dirs,\
-                      cache-dir=/mnt/localssd,\
-                      metadata-cache-negative-ttl-secs=0,\
-                      metadata-cache-ttl-secs=-1,\
-                      stat-cache-max-size-mb=-1,\
-                      type-cache-max-size-mb=-1,\
-                      file-cache-max-size-mb=-1,\
-                      file-cache-cache-file-for-range-read=true,\
-                      file-cache-enable-parallel-downloads=true,\
-                      dir-mode=777,\
-                      file-mode=777,\
+      mount_options: "profile=aiml-serving,\
+                      cache_dir=/mnt/localssd,\
+                      dir_mode=777,\
+                      file_mode=777,\
                       allow_other"