Merge pull request GoogleCloudPlatform#4572 from vikramvs-gg/lustre_integration

GKE Managed Lustre integration

Author: vikramvs-gg

examples/gke-managed-lustre.yaml

@@ -0,0 +1,145 @@
+# Copyright 2025 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+blueprint_name: gke-managed-lustre
+vars:
+  project_id: # Set GCP Project ID Here
+  deployment_name: # Set deployment name
+  # Managed Lustre is only supported in specific regions and zones
+  # Please refer https://cloud.google.com/managed-lustre/docs/locations
+  # The GCP Region used for this deployment.
+  region: us-central1
+  # The GCP Zone used for this deployment.
+  zone: us-central1-c
+  # Cidr block containing the IP of the machine calling terraform.
+  # The following line must be updated for this example to work.
+  authorized_cidr: <your-ip-address>/32
+  version_prefix: "1.33."
+  base_network_name: $(vars.deployment_name)
+  lustre_instance_id: gke-lustre-instance
+  # The values of size_gib and per_unit_storage_throughput are co-related
+  # Please refer https://cloud.google.com/managed-lustre/docs/create-instance#performance-tiers
+  # Storage capacity of the lustre instance in GiB
+  size_gib: 18000
+  # Maximum throughput of the lustre instance in MBps per TiB
+  per_unit_storage_throughput: 1000
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: network
+    source: modules/network/vpc
+    settings:
+      network_name: $(vars.base_network_name)-net
+      subnetwork_name: $(vars.base_network_name)-subnet
+      secondary_ranges_list:
+      - subnetwork_name: $(vars.base_network_name)-subnet
+        ranges:
+        - range_name: pods
+          ip_cidr_range: 10.4.0.0/14
+        - range_name: services
+          ip_cidr_range: 10.0.32.0/20
+
+  # Private Service Access (PSA) requires the compute.networkAdmin role which is
+  # included in the Owner role, but not Editor.
+  # PSA is required for all Managed Lustre functionality.
+  # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions
+  - id: private_service_access
+    source: community/modules/network/private-service-access
+    use: [network]
+    settings:
+      prefix_length: 24
+
+  # Firewall to allow Managed Lustre connection
+  - id: lustre_firewall_rule
+    source: modules/network/firewall-rules
+    use: [network]
+    settings:
+      ingress_rules:
+      - name: $(vars.base_network_name)-allow-lustre-traffic
+        description: Allow Managed Lustre traffic
+        source_ranges:
+        - $(private_service_access.cidr_range)
+        allow:
+        - protocol: tcp
+        ports:
+        - "988"
+
+  - id: managed-lustre
+    source: modules/file-system/managed-lustre
+    use: [network, private_service_access]
+    settings:
+      name: $(vars.lustre_instance_id)
+      local_mount: /lustre
+      remote_mount: lustrefs
+      size_gib: $(vars.size_gib)
+      per_unit_storage_throughput: $(vars.per_unit_storage_throughput)
+
+  - id: node_pool_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-np-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectViewer
+      - artifactregistry.reader
+
+  - id: workload_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-wl-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectAdmin
+      - artifactregistry.reader
+
+  - id: gke_cluster
+    source: modules/scheduler/gke-cluster
+    use: [network, workload_service_account]
+    settings:
+      version_prefix: $(vars.version_prefix)
+      release_channel: RAPID
+      enable_managed_lustre_csi: true # Enable Managed Lustre for the cluster
+      configure_workload_identity_sa: true
+      enable_private_endpoint: false  # Allows for access from authorized public IPs
+      master_authorized_networks:
+      - display_name: kubectl-access-network
+        cidr_block: $(vars.authorized_cidr)
+      maintenance_exclusions:
+      - name: no-minor-or-node-upgrades-indefinite
+        start_time: "2025-08-01T00:00:00Z"
+        end_time: "2026-08-01T00:00:00Z"
+        exclusion_scope: NO_MINOR_OR_NODE_UPGRADES
+    outputs: [instructions]
+
+  - id: lustre-pv
+    source: modules/file-system/gke-persistent-volume
+    use: [managed-lustre, gke_cluster]
+    settings:
+      capacity_gb: $(vars.size_gib)
+
+  - id: gke-lustre-pool
+    source: modules/compute/gke-node-pool
+    use: [gke_cluster, node_pool_service_account]
+    settings:
+      name: gke-lustre-pool
+      zones: [$(vars.zone)]
+      machine_type: n2-standard-16
+      auto_upgrade: true

modules/file-system/gke-persistent-volume/README.md

@@ -157,6 +157,7 @@ No modules.
 | <a name="input_filestore_id"></a> [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no |
 | <a name="input_gcs_bucket_name"></a> [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes |
+| <a name="input_lustre_id"></a> [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no |
 | <a name="input_network_storage"></a> [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | <pre>object({<br/>    server_ip             = string,<br/>    remote_mount          = string,<br/>    local_mount           = string,<br/>    fs_type               = string,<br/>    mount_options         = string,<br/>    client_install_runner = map(string)<br/>    mount_runner          = map(string)<br/>  })</pre> | n/a | yes |
 | <a name="input_pv_name"></a> [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no |

modules/file-system/gke-persistent-volume/main.tf

@@ -20,21 +20,28 @@ locals {
 }
 
 locals {
-  is_gcs = (var.gcs_bucket_name != null)
+  is_gcs    = (var.gcs_bucket_name != null)
+  is_lustre = (var.lustre_id != null)
 
   filestore_id = (
-    !local.is_gcs ?                                  # If not using gcs
+    !local.is_gcs && !local.is_lustre ?              # If not using gcs or lustre
     var.filestore_id :                               # Then filestore_id must be provided
     "projects/empty/locations/empty/instances/empty" # Otherwise use something arbitrary as it will not be used
   )
-  location             = split("/", local.filestore_id)[3]
+
+  lustre_id = (
+    local.is_lustre ?
+    var.lustre_id :
+    "projects/empty/locations/empty/instances/empty"
+  )
+  location             = local.is_lustre ? split("/", local.lustre_id)[3] : split("/", local.filestore_id)[3]
   filestore_name       = split("/", local.filestore_id)[5]
   filestore_share_name = trimprefix(var.network_storage.remote_mount, "/")
-  base_name            = local.is_gcs ? var.gcs_bucket_name : local.filestore_name
-
-  pv_name  = var.pv_name != null ? var.pv_name : "${local.base_name}-pv"
-  pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc"
 
+  # Determine the base_name based on which storage type is used
+  base_name          = local.is_gcs ? var.gcs_bucket_name : (local.is_lustre ? split("/", local.lustre_id)[5] : local.filestore_name)
+  pv_name            = var.pv_name != null ? var.pv_name : "${local.base_name}-pv"
+  pvc_name           = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc"
   list_mount_options = split(",", var.network_storage.mount_options)
 
   filestore_pv_contents = templatefile(
@@ -87,6 +94,31 @@ locals {
     }
   )
 
+  lustre_pv_contents = templatefile(
+    "${path.module}/templates/managed-lustre-pv.yaml.tftpl",
+    {
+      pv_name         = local.pv_name
+      capacity        = "${var.capacity_gb}Gi"
+      location        = local.location
+      project         = split("/", var.cluster_id)[1]
+      labels          = local.labels
+      instance_name   = local.base_name
+      server_ip       = split("@", var.network_storage.server_ip)[0]
+      filesystem_name = var.network_storage.remote_mount
+    }
+  )
+
+  lustre_pvc_contents = templatefile(
+    "${path.module}/templates/managed-lustre-pvc.yaml.tftpl",
+    {
+      pv_name   = local.pv_name
+      pvc_name  = local.pvc_name
+      labels    = local.labels
+      capacity  = "${var.capacity_gb}Gi"
+      namespace = var.namespace
+    }
+  )
+
   cluster_name     = split("/", var.cluster_id)[5]
   cluster_location = split("/", var.cluster_id)[3]
 }
@@ -122,17 +154,17 @@ resource "kubectl_manifest" "pvc_namespace" {
 }
 
 resource "kubectl_manifest" "pv" {
-  yaml_body = local.is_gcs ? local.gcs_pv_contents : local.filestore_pv_contents
+  yaml_body = local.is_gcs ? local.gcs_pv_contents : (local.is_lustre ? local.lustre_pv_contents : local.filestore_pv_contents)
 
   lifecycle {
     precondition {
-      condition     = (var.gcs_bucket_name != null) != (var.filestore_id != null)
-      error_message = "Either gcs_bucket_name or filestore_id must be set."
+      condition     = ((var.gcs_bucket_name != null ? 1 : 0) + (var.filestore_id != null ? 1 : 0) + (var.lustre_id != null ? 1 : 0)) == 1
+      error_message = "Either gcs_bucket_name, filestore_id, or lustre_id must be set."
     }
   }
 }
 
 resource "kubectl_manifest" "pvc" {
-  yaml_body  = local.is_gcs ? local.gcs_pvc_contents : local.filestore_pvc_contents
+  yaml_body  = local.is_gcs ? local.gcs_pvc_contents : (local.is_lustre ? local.lustre_pvc_contents : local.filestore_pvc_contents)
   depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace]
 }

modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ${pv_name}
+  labels:
+  %{~ for key, val in labels ~}
+    ${key}: ${val}
+  %{~ endfor ~}
+spec:
+  storageClassName: ""
+  capacity:
+    storage: ${capacity}
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  volumeMode: Filesystem
+  csi:
+    driver: lustre.csi.storage.gke.io
+    volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container"
+    volumeAttributes:
+      ip: ${server_ip}
+      filesystem: ${filesystem_name}

modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl

@@ -0,0 +1,18 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: ${pvc_name}
+  namespace: ${namespace}
+  labels:
+  %{~ for key, val in labels ~}
+    ${key}: ${val}
+  %{~ endfor ~}
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ""
+  volumeName: ${pv_name}
+  resources:
+    requests:
+      storage: ${capacity}

modules/file-system/gke-persistent-volume/variables.tf

@@ -45,6 +45,19 @@ variable "filestore_id" {
   }
 }
 
+variable "lustre_id" {
+  description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`."
+  type        = string
+  default     = null
+  validation {
+    condition = (
+      var.lustre_id == null ||
+      try(length(split("/", var.lustre_id)), 0) == 6
+    )
+    error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'."
+  }
+}
+
 variable "gcs_bucket_name" {
   description = "The gcs bucket to be used with the persistent volume."
   type        = string

modules/file-system/managed-lustre/outputs.tf

@@ -34,7 +34,7 @@ output "install_managed_lustre_client" {
 
 output "lustre_id" {
   description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`"
-  value       = google_lustre_instance.lustre_instance.instance_id
+  value       = google_lustre_instance.lustre_instance.id
 }
 
 output "capacity_gib" {

modules/network/firewall-rules/README.md

@@ -77,6 +77,7 @@ limitations under the License.
 | Name | Version |
 |------|---------|
 | <a name="provider_google"></a> [google](#provider\_google) | >= 3.83 |
+| <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
 
 ## Modules
 
@@ -88,6 +89,7 @@ limitations under the License.
 
 | Name | Type |
 |------|------|
+| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
 | [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source |
 
 ## Inputs
@@ -96,6 +98,8 @@ limitations under the License.
 |------|-------------|------|---------|:--------:|
 | <a name="input_egress_rules"></a> [egress\_rules](#input\_egress\_rules) | List of egress rules | <pre>list(object({<br/>    name                    = string<br/>    description             = optional(string, null)<br/>    disabled                = optional(bool, null)<br/>    priority                = optional(number, null)<br/>    destination_ranges      = optional(list(string), [])<br/>    source_ranges           = optional(list(string), [])<br/>    source_tags             = optional(list(string))<br/>    source_service_accounts = optional(list(string))<br/>    target_tags             = optional(list(string))<br/>    target_service_accounts = optional(list(string))<br/><br/>    allow = optional(list(object({<br/>      protocol = string<br/>      ports    = optional(list(string))<br/>    })), [])<br/>    deny = optional(list(object({<br/>      protocol = string<br/>      ports    = optional(list(string))<br/>    })), [])<br/>    log_config = optional(object({<br/>      metadata = string<br/>    }))<br/>  }))</pre> | `[]` | no |
 | <a name="input_ingress_rules"></a> [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | <pre>list(object({<br/>    name                    = string<br/>    description             = optional(string, null)<br/>    disabled                = optional(bool, null)<br/>    priority                = optional(number, null)<br/>    destination_ranges      = optional(list(string), [])<br/>    source_ranges           = optional(list(string), [])<br/>    source_tags             = optional(list(string))<br/>    source_service_accounts = optional(list(string))<br/>    target_tags             = optional(list(string))<br/>    target_service_accounts = optional(list(string))<br/><br/>    allow = optional(list(object({<br/>      protocol = string<br/>      ports    = optional(list(string))<br/>    })), [])<br/>    deny = optional(list(object({<br/>      protocol = string<br/>      ports    = optional(list(string))<br/>    })), [])<br/>    log_config = optional(object({<br/>      metadata = string<br/>    }))<br/>  }))</pre> | `[]` | no |
+| <a name="input_network_name"></a> [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no |
 | <a name="input_subnetwork_self_link"></a> [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes |
 
 ## Outputs

modules/network/firewall-rules/main.tf

@@ -14,25 +14,46 @@
  * limitations under the License.
  */
 
+locals {
+  use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null
+}
+
 # the google_compute_network data source does not allow identification by
 # self_link, which uniquely identifies subnet, project, and network
 data "google_compute_subnetwork" "subnetwork" {
+  # Only instantiate this data source if needed
+  count     = local.use_subnetwork_data ? 1 : 0
   self_link = var.subnetwork_self_link
 }
 
-# Module-level check for Private Google Access on the subnetwork
-check "private_google_access_enabled_subnetwork" {
-  assert {
-    condition     = data.google_compute_subnetwork.subnetwork.private_ip_google_access
-    error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services."
+locals {
+  # Derived values from data source, null if data source is not used
+  derived_project_id   = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null
+  derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null
+
+  # Effective values: Use var if provided, otherwise use derived value
+  effective_project_id   = coalesce(var.project_id, local.derived_project_id)
+  effective_network_name = coalesce(var.network_name, local.derived_network_name)
+}
+
+# Module-level check for Private Google Access on the subnetwork.
+# This check is only relevant if subnetwork_self_link was provided and used.
+resource "terraform_data" "pga_check" {
+  count = local.use_subnetwork_data ? 1 : 0
+
+  lifecycle {
+    precondition {
+      condition     = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access
+      error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services."
+    }
   }
 }
 
 module "firewall_rule" {
   source       = "terraform-google-modules/network/google//modules/firewall-rules"
   version      = "~> 9.0"
-  project_id   = data.google_compute_subnetwork.subnetwork.project
-  network_name = data.google_compute_subnetwork.subnetwork.network
+  project_id   = local.effective_project_id
+  network_name = local.effective_network_name
 
   ingress_rules = var.ingress_rules
   egress_rules  = var.egress_rules