feat: add default infra to htc

2-multitenant/envs/development/README.md

@@ -8,6 +8,7 @@
 | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no |
 | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no |
 | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no |
+| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no |
 | envs | Environments | <pre>map(object({<br>    billing_account    = string<br>    folder_id          = string<br>    network_project_id = string<br>    network_self_link  = string<br>    org_id             = string<br>    subnets_self_links = list(string)<br>  }))</pre> | n/a | yes |
 | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no |
 | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no |

2-multitenant/envs/development/main.tf

@@ -35,4 +35,5 @@ module "env" {
   deletion_protection              = var.deletion_protection
   cb_private_workerpool_project_id = var.cb_private_workerpool_project_id
   cluster_release_channel          = var.cluster_release_channel
+  enable_csi_gcs_fuse              = var.enable_csi_gcs_fuse
 }

2-multitenant/envs/development/variables.tf

@@ -86,3 +86,9 @@ variable "cluster_release_channel" {
   type        = string
   default     = "REGULAR"
 }
+
+variable "enable_csi_gcs_fuse" {
+  description = "Enable the GCS Fuse CSI Driver for HTC example"
+  type        = bool
+  default     = false
+}

2-multitenant/envs/nonproduction/README.md

@@ -8,6 +8,7 @@
 | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no |
 | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no |
 | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no |
+| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no |
 | envs | Environments | <pre>map(object({<br>    billing_account    = string<br>    folder_id          = string<br>    network_project_id = string<br>    network_self_link  = string<br>    org_id             = string<br>    subnets_self_links = list(string)<br>  }))</pre> | n/a | yes |
 | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no |
 | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no |

2-multitenant/envs/nonproduction/main.tf

@@ -34,4 +34,5 @@ module "env" {
   deletion_protection              = var.deletion_protection
   cb_private_workerpool_project_id = var.cb_private_workerpool_project_id
   cluster_release_channel          = var.cluster_release_channel
+  enable_csi_gcs_fuse              = var.enable_csi_gcs_fuse
 }

2-multitenant/envs/nonproduction/variables.tf

@@ -85,3 +85,9 @@ variable "cluster_release_channel" {
   type        = string
   default     = "REGULAR"
 }
+
+variable "enable_csi_gcs_fuse" {
+  description = "Enable the GCS Fuse CSI Driver for HTC example"
+  type        = bool
+  default     = false
+}

2-multitenant/envs/production/README.md

@@ -8,6 +8,7 @@
 | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no |
 | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no |
 | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no |
+| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no |
 | envs | Environments | <pre>map(object({<br>    billing_account    = string<br>    folder_id          = string<br>    network_project_id = string<br>    network_self_link  = string<br>    org_id             = string<br>    subnets_self_links = list(string)<br>  }))</pre> | n/a | yes |
 | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no |
 | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no |

2-multitenant/envs/production/main.tf

@@ -34,4 +34,5 @@ module "env" {
   deletion_protection              = var.deletion_protection
   cb_private_workerpool_project_id = var.cb_private_workerpool_project_id
   cluster_release_channel          = var.cluster_release_channel
+  enable_csi_gcs_fuse              = var.enable_csi_gcs_fuse
 }

2-multitenant/envs/production/variables.tf

@@ -85,3 +85,9 @@ variable "cluster_release_channel" {
   type        = string
   default     = "REGULAR"
 }
+
+variable "enable_csi_gcs_fuse" {
+  description = "Enable the GCS Fuse CSI Driver for HTC example"
+  type        = bool
+  default     = false
+}

2-multitenant/modules/env_baseline/README.md

@@ -25,6 +25,7 @@ The following resources are created:
 | create\_cluster\_project | Create Cluster Project ID, otherwise the Network Project ID is used | `bool` | `true` | no |
 | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no |
 | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no |
+| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no |
 | env | The environment to prepare (ex. development) | `string` | n/a | yes |
 | folder\_id | Folder ID | `string` | n/a | yes |
 | master\_ipv4\_cidr\_blocks | List of IP ranges (One range per cluster) in CIDR notation to use for the hosted master network. This range will be used for assigning private IP addresses to the cluster master(s) and the ILB VIP. This range must not overlap with any other ranges in use within the cluster's network, and it must be a /28 subnet. | `list(string)` | <pre>[<br>  "10.11.10.0/28",<br>  "10.11.20.0/28"<br>]</pre> | no |

2-multitenant/modules/env_baseline/main.tf

@@ -321,6 +321,8 @@ module "gke-standard" {
     data.google_compute_default_service_account.compute_sa,
   ]
 
+  gcs_fuse_csi_driver = var.enable_csi_gcs_fuse
+
   // Private Cluster Configuration
   enable_private_nodes    = true
   enable_private_endpoint = true

2-multitenant/modules/env_baseline/variables.tf

@@ -127,3 +127,9 @@ variable "deletion_protection" {
   description = "Whether or not to allow Terraform to destroy the cluster."
   default     = true
 }
+
+variable "enable_csi_gcs_fuse" {
+  description = "Enable the GCS Fuse CSI Driver for HTC example"
+  type        = bool
+  default     = false
+}

4-appfactory/modules/app-group-baseline/main.tf

@@ -39,7 +39,8 @@ locals {
           "roles/resourcemanager.projectIamAdmin",
           "roles/gkehub.admin",
           "roles/modelarmor.admin", //permission to create model armor template
-          "roles/iam.serviceAccountAdmin"
+          "roles/iam.serviceAccountAdmin",
+          "roles/container.admin"
         ]
       }
     }

5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf

@@ -156,7 +156,7 @@ module "cb-gke-project-iam-bindings" {
   for_each   = local.gke_projects
   project_id = each.value
 
-  project_roles           = ["roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"]
+  project_roles           = ["roles/privateca.certificateManager", "roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"]
   prefix                  = "serviceAccount"
   service_account_address = google_service_account.cloud_build.email
 }

5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/agent_job.templ

@@ -0,0 +1,106 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${name}
+  namespace: ${namespace}
+spec:
+  parallelism: ${parallel}
+  ttlSecondsAfterFinished: 30
+  template:
+    metadata:
+      labels:
+        app: ${name}
+      annotations:
+        gke-gcsfuse/volumes: "true"
+        gke-gcsfuse/cpu-request: "50m"
+        gke-gcsfuse/memory-request: "64Mi"
+        gke-gcsfuse/cpu-limit: "1"
+        gke-gcsfuse/memory-limit: "1Gi"
+        gke-gcsfuse/ephemeral-storage-request: "1Gi"
+    spec:
+      priorityClassName: higher-priority
+      nodeSelector:
+        cloud.google.com/compute-class: spot-capacity
+      initContainers:
+      - name: workload
+        image: ${workload_image}
+        args: ${jsonencode(workload_args)}
+        workingDir: /data
+        resources:
+          requests:
+            memory: "2Gi"
+            cpu: "1"
+          limits:
+            cpu: "2"
+            memory: "4Gi"
+        volumeMounts:
+        - name: gcs-data
+          mountPath: /data
+      containers:
+      - name: agent
+        image: ${agent_image}
+        resources:
+          requests:
+            memory: "2Gi"
+            cpu: "1"
+          limits:
+            cpu: "2"
+            memory: "4Gi"
+        args: ["serve", "pubsub-pull",
+               # subscription to pull requests (tasks) from, topic to publish responses
+               "${workload_request_sub}", "${workload_response}",
+               # Log to JSON
+               "--logJSON",
+               # Log every operation
+               "--logAll",
+               # Text encode protobuf on PubSub
+               "--jsonPubSub=true",
+               # Endpoint to dispatch the work
+               "--endpoint", "${workload_endpoint}",
+               # Timeout waiting for the gRPC service to be available
+               "--timeout", "30s",
+               # Timeout when there's no more work from Pub/Sub -- stop everything.
+               "--idleTimeout", "120s",
+               # NOTE: These are important so that only a small number of messages (tasks)
+               # are pulled from Pub/Sub, as if they are chunky in size they need to be
+               # evenly distributed.
+               # Maximum number of goroutines executing
+               "--goroutines", "1",
+               # Maximum number of outstanding messages
+               "--maxoutstandingmessages", "1"]
+        env:
+        - name: PUBSUB_PROJECT_ID
+          value: "${pubsub_project_id}"
+        - name: K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      restartPolicy: Never
+      volumes:
+      - name: gcs-data
+        persistentVolumeClaim:
+          claimName: data-pvc
+          readOnly: false
+
+  backoffLimit: 2
+  podFailurePolicy:
+    rules:
+    - action: FailJob
+      onExitCodes:
+        containerName: agent
+        operator: NotIn
+        values: [0]

5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/controller_job.templ

@@ -0,0 +1,64 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${job_name}
+  namespace: ${namespace}
+spec:
+  parallelism: ${parallel}
+  ttlSecondsAfterFinished: 30
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: ${job_name}
+      annotations:
+        gke-gcsfuse/volumes: "true"
+        gke-gcsfuse/cpu-request: "50m"
+        gke-gcsfuse/memory-request: "64Mi"
+        gke-gcsfuse/cpu-limit: "1"
+        gke-gcsfuse/memory-limit: "1Gi"
+        gke-gcsfuse/ephemeral-storage-request: "1Gi"
+    spec:
+      priorityClassName: higher-priority
+      containers:
+      - name: ${container_name}
+        image: ${image}
+        workingDir: /data
+        args: ${jsonencode(args)}
+        resources:
+          requests:
+            memory: "2Gi"
+            cpu: "2"
+          limits:
+            cpu: "4"
+            memory: "4Gi"
+        volumeMounts:
+        - name: gcs-data
+          mountPath: /data
+        env:
+        - name: PUBSUB_PROJECT_ID
+          value: "${pubsub_project_id}"
+        - name: K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      restartPolicy: Never
+      volumes:
+      - name: gcs-data
+        persistentVolumeClaim:
+          claimName: data-pvc
+          readOnly: false

5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ

@@ -0,0 +1,106 @@
+# Copyright 2024 Google LLC
+# Licensed under the Apache License, Version 2.0
+# REFACTORED FOR KEDA
+
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: keda-gcp-auth-${name} # Unique name per deployment
+  namespace: ${namespace}
+spec:
+  podIdentity:
+    provider: gcp
+
+---
+# This REPLACES the standard HorizontalPodAutoscaler
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: ${name}
+  namespace: ${namespace}
+spec:
+  scaleTargetRef:
+    name: ${name} # Targets the Deployment defined below
+  minReplicaCount: 1
+  maxReplicaCount: 2
+  # Use num_undelivered_messages directly for scaling
+  triggers:
+  - type: gcp-pubsub
+    metadata:
+      subscriptionName: "projects/${pubsub_project_id}/subscriptions/${gke_hpa_request_sub}"
+      value: "1"
+    authenticationRef:
+      name: keda-gcp-auth-${name}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${name}
+  namespace: ${namespace}
+  labels:
+    app: ${name}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ${name}
+  template:
+    metadata:
+      labels:
+        app: ${name}
+      annotations:
+        gke-gcsfuse/volumes: "true"
+        gke-gcsfuse/cpu-request: "50m"
+        gke-gcsfuse/memory-request: "64Mi"
+        gke-gcsfuse/cpu-limit: "1"
+        gke-gcsfuse/memory-limit: "1Gi"
+        gke-gcsfuse/ephemeral-storage-request: "1Gi"
+    spec:
+      nodeSelector:
+        cloud.google.com/compute-class: spot-capacity
+      containers:
+      - name: workload
+        image: ${workload_image}
+        args: ${jsonencode(workload_args)}
+        workingDir: /data
+        resources:
+          requests:
+            memory: "1Gi"
+            cpu: "200m"
+          limits:
+            cpu: "250m"
+            memory: "1Gi"
+        volumeMounts:
+        - name: gcs-data
+          mountPath: /data
+      - name: agent
+        resources:
+          requests:
+            memory: "1Gi"
+            cpu: "900m"
+          limits:
+            cpu: "1000m"
+            memory: "3Gi"
+        image: ${agent_image}
+        args: ["serve", "pubsub-pull",
+               "${gke_hpa_request_sub}", "${gke_hpa_response}",
+               "--logJSON",
+               "--logAll",
+               "--jsonPubSub=true",
+               "--endpoint", "${workload_endpoint}",
+               "--timeout", "120s",
+               "--goroutines", "1",
+               "--maxoutstandingmessages", "1"]
+        env:
+        - name: PUBSUB_PROJECT_ID
+          value: "${pubsub_project_id}"
+        - name: K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      volumes:
+      - name: gcs-data
+        persistentVolumeClaim:
+          claimName: data-pvc
+          readOnly: false