Add deep health checkc status and ability to remove deep health checks from config

Author: iankouls-aws

Container-Root/hyperpod/ops/eks/nodes-list.sh

@@ -20,9 +20,9 @@ else
 	shift
 
 	if [ "$node" == "" ]; then
-		CMD="kubectl get nodes -L node.kubernetes.io/instance-type -L sagemaker.amazonaws.com/node-health-status $@"
+		CMD="kubectl get nodes -L node.kubernetes.io/instance-type -L sagemaker.amazonaws.com/node-health-status -L sagemaker.amazonaws.com/deep-health-check-status $@"
 	else
-		CMD="kubectl get nodes -L node.kubernetes.io/instance-type -L sagemaker.amazonaws.com/node-health-status  $@ | grep -E \"NODE-HEALTH-STATUS|$node\""
+		CMD="kubectl get nodes -L node.kubernetes.io/instance-type -L sagemaker.amazonaws.com/node-health-status -L sagemaker.amazonaws.com/deep-health-check-status $@ | grep -E \"NODE-HEALTH-STATUS|$node\""
 	fi
 
 	if [ "${VERBOSE}" == "true" ]; then

wd/conf/eks/aws-do-hyperpod-eks/generate-update-config.sh

@@ -3,6 +3,55 @@
 source ./env_input
 source ./env_vars
 
+if [ "$ONSTART_DEEP_HEALTHCHECKS" == "" ]; then
+
+cat > hyperpod-update-config.json << EOL
+{
+    "ClusterName": "${HYPERPOD_NAME}",
+    "InstanceGroups": [
+      {
+        "InstanceGroupName": "worker-group-1",
+        "InstanceType": "${ACCEL_INSTANCE_TYPE}",
+        "InstanceCount": ${ACCEL_COUNT},
+        "InstanceStorageConfigs": [
+          {
+            "EbsVolumeConfig": {
+              "VolumeSizeInGB": ${ACCEL_VOLUME_SIZE}
+            }
+          }
+        ],
+        "LifeCycleConfig": {
+          "SourceS3Uri": "s3://${BUCKET_NAME}",
+          "OnCreate": "on_create.sh"
+        },
+        "ExecutionRole": "${EXECUTION_ROLE}",
+        "ThreadsPerCore": 1
+      },
+      {
+        "InstanceGroupName": "worker-group-2",
+        "InstanceType": "${GEN_INSTANCE_TYPE}",
+        "InstanceCount": ${GEN_COUNT},
+        "InstanceStorageConfigs": [
+          {
+            "EbsVolumeConfig": {
+              "VolumeSizeInGB": ${GEN_VOLUME_SIZE}
+            }
+          }
+        ],
+        "LifeCycleConfig": {
+          "SourceS3Uri": "s3://${BUCKET_NAME}",
+          "OnCreate": "on_create.sh"
+        },
+        "ExecutionRole": "${EXECUTION_ROLE}",
+        "ThreadsPerCore": 1
+      }
+    ],
+    "NodeRecovery": "${NODE_RECOVERY}"
+}
+EOL
+
+else
+
 cat > hyperpod-update-config.json << EOL
 {
     "ClusterName": "${HYPERPOD_NAME}",
@@ -48,3 +97,5 @@ cat > hyperpod-update-config.json << EOL
     "NodeRecovery": "${NODE_RECOVERY}"
 }
 EOL
+
+fi