Skip to content

Commit 7a84665

Browse files
ishitachailishitachail
authored
Merge pull request GoogleCloudPlatform#4293 from ishitachail/chs
created chs-cronjob manifest
2 parents 12689b6 + 85cf7e5 commit 7a84665

9 files changed

Lines changed: 197 additions & 1 deletion

File tree

‎examples/gke-a3-megagpu/chs-cronjob.yaml.tftpl‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright 2024 "Google LLC"
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: batch/v1
16+
kind: CronJob
17+
metadata:
18+
name: cluster-health-scanner-cronjob
19+
spec:
20+
schedule: "${cronjob_schedule}"
21+
concurrencyPolicy: Forbid
22+
successfulJobsHistoryLimit: 3
23+
failedJobsHistoryLimit: 1
24+
suspend: false
25+
jobTemplate:
26+
spec:
27+
template:
28+
spec:
29+
serviceAccountName: workload-identity-k8s-sa
30+
containers:
31+
- name: chs-runner
32+
image: python:3.11-slim-buster
33+
imagePullPolicy: Always
34+
command:
35+
- /bin/bash
36+
- -c
37+
- |
38+
set -ex
39+
set -x
40+
apt-get update && apt-get install -y git curl gnupg -y
41+
git clone https://github.com/GoogleCloudPlatform/cluster-health-scanner
42+
cd cluster-health-scanner
43+
apt-get install -y apt-transport-https ca-certificates
44+
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
45+
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
46+
apt-get update
47+
apt-get install -y google-cloud-cli kubectl
48+
apt-get install -y google-cloud-cli-gke-gcloud-auth-plugin
49+
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
50+
pip3 install -r cli/requirements.txt
51+
gcloud container clusters get-credentials ${deployment_name} --region ${region} --project ${project_id}
52+
OUTPUT_DIR="/mnt/output"
53+
mkdir -p $OUTPUT_DIR
54+
TIMESTAMP="`date "+%Y-%m-%d %H:%M:%S"`"
55+
OUTPUT_FILENAME="${deployment_name}_healthscan_result_$TIMESTAMP.txt"
56+
FULL_OUTPUT_PATH="$OUTPUT_DIR/$OUTPUT_FILENAME"
57+
python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c gpu --run_only_on_available_nodes
58+
python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c nccl --run_only_on_available_nodes
59+
python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c straggler --run_only_on_available_nodes
60+
python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c neper --run_only_on_available_nodes
61+
python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c tinymax --run_only_on_available_nodes
62+
#python3 cli/cluster_diag.py -o gke healthscan ${machine_type} -c status --run_only_on_available_nodes > "$FULL_OUTPUT_PATH" 2>&1
63+
kubectl get nodes -o custom-columns="NODE:.metadata.name,NCCL_MARK:.metadata.labels.aiinfra/nccl-healthcheck-test,NCCL_BANDWIDTH:.metadata.labels.aiinfra/nccl-healthcheck-bandwidth,NCCL_RESULT:.metadata.labels.aiinfra/nccl-healthcheck-result,NCCL_RUNTIME:.metadata.labels.aiinfra/nccl-healthcheck-runtime-sec,TINYMAX_MARK:.metadata.labels.aiinfra/tinymax-healthcheck-test,TINYMAX_RESULT:.metadata.labels.aiinfra/tinymax-healthcheck-result,TINYMAX_RUNTIME:.metadata.labels.aiinfra/tinymax-healthcheck-runtime-sec,GPU_MARK:.metadata.labels.aiinfra/gpu-healthcheck-test,GPU_RESULT:.metadata.labels.aiinfra/gpu-healthcheck-result,GPU_RUNTIME:.metadata.labels.aiinfra/gpu-healthcheck-runtime-sec" > "$FULL_OUTPUT_PATH" 2>&1
64+
echo "Health scan outputs saved to $OUTPUT_DIR"
65+
echo "Final output file: $OUTPUT_FILENAME"
66+
volumeMounts:
67+
- name: ${gcs_bucket}
68+
mountPath: /mnt/output
69+
volumes:
70+
- name: ${gcs_bucket}
71+
persistentVolumeClaim:
72+
claimName: ${gcs_pvc}
73+
restartPolicy: Never
74+
tolerations:
75+
- key: "nvidia.com/gpu"
76+
operator: "Exists"
77+
effect: "NoSchedule"
78+
- key: "components.gke.io/gke-managed-components"
79+
operator: "Exists"
80+
effect: "NoSchedule"
81+
backoffLimit: 0

‎examples/gke-a3-megagpu/chs-permissions.yaml.tftpl‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: workload-identity-k8s-sa
5+
namespace: default
6+
annotations:
7+
iam.gke.io/gcp-service-account: gke-wl-sa@${project_id}.iam.gserviceaccount.com
8+
---
9+
apiVersion: rbac.authorization.k8s.io/v1
10+
kind: ClusterRole
11+
metadata:
12+
name: cluster-health-scanner-job-role
13+
rules:
14+
- apiGroups: [""]
15+
resources:
16+
- "pods"
17+
- "pods/log"
18+
- "pods/exec"
19+
- "nodes"
20+
- "events"
21+
- "services"
22+
- "secrets"
23+
- "configmaps"
24+
- "serviceaccounts"
25+
verbs: ["list", "get", "create", "delete", "watch", "patch", "update"]
26+
27+
- apiGroups: ["apps"]
28+
resources:
29+
- "daemonsets"
30+
- "deployments"
31+
- "replicasets"
32+
verbs: ["list", "get", "create", "delete", "watch", "patch", "update"]
33+
34+
- apiGroups: ["batch"]
35+
resources:
36+
- "jobs"
37+
- "jobs/status"
38+
verbs: ["list", "get", "create", "delete", "watch", "patch", "update"]
39+
40+
- apiGroups: ["rbac.authorization.k8s.io"]
41+
resources:
42+
- "clusterrolebindings"
43+
- "clusterroles"
44+
- "roles"
45+
- "rolebindings"
46+
verbs: ["list", "get", "create", "delete", "watch", "patch", "update"]
47+
---
48+
apiVersion: rbac.authorization.k8s.io/v1
49+
kind: ClusterRoleBinding
50+
metadata:
51+
name: cluster-health-scanner-job-binding
52+
subjects:
53+
- kind: ServiceAccount
54+
name: workload-identity-k8s-sa
55+
namespace: default
56+
roleRef:
57+
kind: ClusterRole
58+
name: cluster-health-scanner-job-role
59+
apiGroup: rbac.authorization.k8s.io

‎examples/gke-a3-megagpu/chs-pvc.yaml.tftpl‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: PersistentVolumeClaim
3+
metadata:
4+
name: ${pvc_name}
5+
spec:
6+
accessModes:
7+
- ${access_mode}
8+
resources:
9+
requests:
10+
storage: ${capacity}
11+
storageClassName: ${storage_class_name}
12+

‎examples/gke-a3-megagpu/gke-a3-megagpu-deployment.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,6 @@ vars:
4545
# To target a BLOCK_NAME, the name of the extended reservation
4646
# can be inputted as <reservation-name>/reservationBlocks/<reservation-block-name>
4747
reservation: RESERVATION_NAME
48+
49+
enable_periodic_health_checks: false # Make this true to run CHS (healthchecks)
50+
health_check_schedule: "0 0 * * 0" # Run the health check at 12:00 AM (midnight) every Sunday

‎examples/gke-a3-megagpu/gke-a3-megagpu.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,15 @@ vars:
5050
accelerator_type: nvidia-h100-mega-80gb
5151
version_prefix: "1.32."
5252

53+
enable_periodic_health_checks: false # Make this true to run CHS (healthchecks)
54+
health_check_schedule: "0 0 * * 0" # Run the health check at 12:00 AM (midnight) every Sunday
55+
56+
permissions_file_staged_path: $(ghpc_stage("./chs-permissions.yaml.tftpl"))
57+
chs_output_bucket_name: chs-result
58+
chs_pvc_claim_name: chs-output-pvc
59+
chs_cronjob_rendered_path: $(ghpc_stage("./chs-cronjob.yaml.tftpl"))
60+
chs_pvc_rendered_path: $(ghpc_stage("./chs-pvc.yaml.tftpl"))
61+
5362
deployment_groups:
5463
- group: primary
5564
modules:
@@ -89,6 +98,7 @@ deployment_groups:
8998
- stackdriver.resourceMetadata.writer
9099
- storage.objectAdmin
91100
- artifactregistry.reader
101+
- container.admin
92102

93103
- id: gpunets
94104
source: modules/network/multivpc
@@ -137,6 +147,27 @@ deployment_groups:
137147
source: modules/management/kubectl-apply
138148
use: [gke_cluster]
139149
settings:
150+
apply_manifests:
151+
- source: $(vars.permissions_file_staged_path)
152+
template_vars:
153+
project_id: $(vars.project_id)
154+
- source: $(vars.chs_pvc_rendered_path)
155+
enable: $(vars.enable_periodic_health_checks)
156+
template_vars:
157+
pvc_name: $(vars.chs_pvc_claim_name)
158+
access_mode: ReadWriteOnce
159+
capacity: 1Gi
160+
storage_class_name: standard-rwo
161+
- source: $(vars.chs_cronjob_rendered_path)
162+
enable: $(vars.enable_periodic_health_checks)
163+
template_vars:
164+
project_id: $(vars.project_id)
165+
deployment_name: $(vars.deployment_name)
166+
region: $(vars.region)
167+
machine_type: a3-megagpu-8g
168+
gcs_bucket: $(vars.chs_output_bucket_name)
169+
gcs_pvc: $(vars.chs_pvc_claim_name)
170+
cronjob_schedule: $(vars.health_check_schedule)
140171
kueue:
141172
install: true
142173
config_path: $(vars.kueue_configuration_path)

‎modules/management/kubectl-apply/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ limitations under the License.
155155

156156
| Name | Description | Type | Default | Required |
157157
|------|-------------|------|---------|:--------:|
158-
| <a name="input_apply_manifests"></a> [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md). | <pre>list(object({<br/> content = optional(string, null)<br/> source = optional(string, null)<br/> template_vars = optional(map(any), null)<br/> server_side_apply = optional(bool, false)<br/> wait_for_rollout = optional(bool, true)<br/> }))</pre> | `[]` | no |
158+
| <a name="input_apply_manifests"></a> [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md). | <pre>list(object({<br/> enable = optional(bool, true)<br/> content = optional(string, null)<br/> source = optional(string, null)<br/> template_vars = optional(map(any), null)<br/> server_side_apply = optional(bool, false)<br/> wait_for_rollout = optional(bool, true)<br/> }))</pre> | `[]` | no |
159159
| <a name="input_cluster_id"></a> [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects/<project\_id>/locations/<region>/clusters/<name>. | `string` | n/a | yes |
160160
| <a name="input_gib"></a> [gib](#input\_gib) | Install the NCCL gIB plugin | <pre>object({<br/> install = bool<br/> path = string<br/> template_vars = object({<br/> image = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")<br/> version = string<br/> node_affinity = optional(any, {<br/> requiredDuringSchedulingIgnoredDuringExecution = {<br/> nodeSelectorTerms = [{<br/> matchExpressions = [{<br/> key = "cloud.google.com/gke-gpu",<br/> operator = "In",<br/> values = ["true"]<br/> }]<br/> }]<br/> }<br/> })<br/> accelerator_count = number<br/> })<br/> })</pre> | <pre>{<br/> "install": false,<br/> "path": "",<br/> "template_vars": {<br/> "accelerator_count": 0,<br/> "version": ""<br/> }<br/>}</pre> | no |
161161
| <a name="input_gke_cluster_exists"></a> [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no |

‎modules/management/kubectl-apply/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ locals {
2222

2323
apply_manifests_map = tomap({
2424
for index, manifest in var.apply_manifests : index => manifest
25+
if manifest.enable
2526
})
2627

2728
install_kueue = try(var.kueue.install, false)

‎modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11426,6 +11426,14 @@ metadata:
1142611426
control-plane: controller-manager
1142711427
name: kueue-manager-role
1142811428
rules:
11429+
- apiGroups:
11430+
- batch
11431+
resources:
11432+
- cronjobs
11433+
verbs:
11434+
- get
11435+
- list
11436+
- watch
1142911437
- apiGroups:
1143011438
- ""
1143111439
resources:

‎modules/management/kubectl-apply/variables.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ variable "cluster_id" {
7777
variable "apply_manifests" {
7878
description = "A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md)."
7979
type = list(object({
80+
enable = optional(bool, true)
8081
content = optional(string, null)
8182
source = optional(string, null)
8283
template_vars = optional(map(any), null)

0 commit comments

Comments
 (0)