Show all commands before being executed unless VERBOSE=false

iankouls-aws · iankouls-aws · commit 107c0d7891fd · 2026-06-23T16:28:53.000-07:00
diff --git a/.env b/.env
@@ -53,7 +53,7 @@ fi
 export IMAGE=aws-do-eks
 ## VERSION: [optional] - Version tag for this Docker image. Example: v20180302
 #export VERSION=v$(date +%Y%m%d)
-export VERSION=v20260611
+export VERSION=v20260623
 export TAG=$(if [ -z "${VERSION}" ]; then echo ""; else echo ":${VERSION}"; fi) 
 ## BUILD_OPTS: [optional] - arguments for the docker image build command
 export BUILD_OPTS="--progress plain --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} --build-arg no_proxy=${no_proxy}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/config.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/config.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
 
-vi .env
+export CMD="vi .env"
+
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
 
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/run.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/run.sh
@@ -6,18 +6,25 @@ if [ "${MANIFEST_TYPE}" == "" ]; then
 	export MANIFEST_TYPE=deployment
 fi
 
+export CMD=""
+
 if [ "${MANIFEST_TYPE}" == "deployment" ]; then
 
 	cat deployment.yaml-template | envsubst > deployment.yaml
+	export CMD="kubectl apply -f ./deployment.yaml"
 
-	kubectl apply -f ./deployment.yaml
 elif [ "${MANIFEST_TYPE}" == "lws" ]; then
 
 	cat lws.yaml-template | envsubst > lws.yaml
+	export CMD="kubectl apply -f ./lws.yaml"
 
-	kubectl apply -f ./lws.yaml
 else
+
 	echo "Unknown MANIFEST_TYPE ${MANIFEST_TYPE}"
+
 fi
 
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
+
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/stop.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/stop.sh
@@ -6,21 +6,28 @@ if [ "${MANIFEST_TYPE}" == "" ]; then
 	export MANIFEST_TYPE=deployment
 fi
 
+export CMD=""
+
 if [ "${MANIFEST_TYPE}" == "deployment" ]; then
 
 	cat deployment.yaml-template | envsubst > deployment.yaml
-
-	kubectl delete -f ./deployment.yaml
+	export CMD="kubectl delete -f ./deployment.yaml"
 
 elif [ "${MANIFEST_TYPE}" == "lws" ]; then
 
 	cat lws.yaml-template | envsubst > lws.yaml
+	export CMD="kubectl delete -f ./lws.yaml"
 
-	kubectl delete -f ./lws.yaml
 else
+
 	echo "Unknown MANIFEST_TYPE ${MANIFEST_TYPE}"
+
 fi
 
-kubectl delete pods $(kubectl get pods | grep ${DEPLOYMENT_NAME} | cut -d ' ' -f 1)
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
 
+export CMD="kubectl delete pods \$(kubectl get pods | grep ${DEPLOYMENT_NAME} | cut -d ' ' -f 1)"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
 
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/config.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/config.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
 
-vi .env
+export CMD="vi .env"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/run.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/run.sh
@@ -6,17 +6,24 @@ if [ "${MANIFEST_TYPE}" == "" ]; then
 	export MANIFEST_TYPE=deployment
 fi
 
+export CMD=""
+
 if [ "${MANIFEST_TYPE}" == "deployment" ]; then
 
 	cat deployment.yaml-template | envsubst > deployment.yaml
-	kubectl apply -f ./deployment.yaml
+	export CMD="kubectl apply -f ./deployment.yaml"
 
 elif [ "${MANIFEST_TYPE}" == "lws" ]; then
 
 	cat lws.yaml-template | envsubst > lws.yaml
-	kubectl apply -f ./lws.yaml
+	export CMD="kubectl apply -f ./lws.yaml"
 
 else
+
 	echo "Unknown MANIFEST_TYPE ${MANIFEST_TYPE}"
+
 fi
 
+
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/stop.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/stop.sh
@@ -6,20 +6,27 @@ if [ "${MANIFEST_TYPE}" == "" ]; then
 	export MANIFEST_TYPE=deployment
 fi
 
+export CMD=""
+
 if [ "${MANIFEST_TYPE}" == "deployment" ]; then
 
 	cat deployment.yaml-template | envsubst > deployment.yaml
-
-	kubectl delete -f ./deployment.yaml
+	export CMD="kubectl delete -f ./deployment.yaml"
 
 elif [ "${MANIFEST_TYPE}" == "lws" ]; then
 
 	cat lws.yaml-template | envsubst > lws.yaml
+	export CMD="kubectl delete -f ./lws.yaml"
 
-	kubectl delete -f ./lws.yaml
 else
+
 	echo "Unknown MANIFEST_TYPE ${MANIFEST_TYPE}"
+
 fi
 
-kubectl delete pods $(kubectl get pods | grep ${DEPLOYMENT_NAME} | cut -d ' ' -f 1)
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
 
+export CMD="kubectl delete pods \$(kubectl get pods | grep ${DEPLOYMENT_NAME} | cut -d ' ' -f 1)"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "$CMD"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/config.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/config.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
 
-vi .env
-
+export CMD="vi .env"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-exec.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-exec.sh
@@ -2,5 +2,7 @@
 
 echo ""
 echo "Opening do-hf shell ..."
-kubectl exec -it do-hf -- bash
 
+export CMD="kubectl exec -it do-hf -- bash"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-run.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-run.sh
@@ -3,18 +3,18 @@
 echo ""
 echo "Starting do-hf pod ..."
 
-# Multi line with node selector and volume mount
-kubectl run do-hf \
+export CMD="kubectl run do-hf \
   --image=iankoulski/do-hf \
   --overrides='{
-    "spec": {
-      "nodeSelector": {"nvidia.com/gpu.present": "true"},
-      "containers": [{
-        "name": "do-hf",
-        "image": "iankoulski/do-hf",
-        "volumeMounts": [{"name": "fsx-vol", "mountPath": "/shared"}]
+    \"spec\": {
+      \"nodeSelector\": {\"nvidia.com/gpu.present\": \"true\"},
+      \"containers\": [{
+        \"name\": \"do-hf\",
+        \"image\": \"iankoulski/do-hf\",
+        \"volumeMounts\": [{\"name\": \"fsx-vol\", \"mountPath\": \"/shared\"}]
       }],
-      "volumes": [{"name": "fsx-vol", "persistentVolumeClaim": {"claimName": "fsx-pvc"}}]
+      \"volumes\": [{\"name\": \"fsx-vol\", \"persistentVolumeClaim\": {\"claimName\": \"fsx-pvc\"}}]
     }
-  }'
-
+  }'"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-stop.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/hf-pod-stop.sh
@@ -2,5 +2,7 @@
 
 echo ""
 echo "Removing do-hf pod ..."
-kubectl delete pod do-hf --force --grace-period=0
 
+export CMD="kubectl delete pod do-hf --force --grace-period=0"
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/model-download.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/download/model-download.sh
@@ -2,17 +2,17 @@
 
 source .env
 
-# Start the do-aiperf pod
+# Start the do-hf pod
 source ./hf-pod-run.sh
 
 # Wait for pod to be running
 echo "Waiting for the do-hf pod to be ready..."
 kubectl wait --for=condition=Ready pod/do-hf --timeout=120s
 
-# Execute the aiperf command interactively
-echo "Executing hf command in do-hf pod ..."
-kubectl exec -it do-hf -- bash -c "export HF_TOKEN=${HF_TOKEN}; hf download ${MODEL_NAME} --local-dir ${MODEL_PATH}"
+# Execute the hf command in do-hf pod
+export CMD="kubectl exec -it do-hf -- bash -c \"export HF_TOKEN=${HF_TOKEN}; hf download ${MODEL_NAME} --local-dir ${MODEL_PATH}\""
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
 
 # Clean up
 source ./hf-pod-stop.sh
-
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-pod-run.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-pod-run.sh
@@ -5,27 +5,8 @@
 echo ""
 echo "Starting do-aiperf pod ..."
 
-# One line simple
-#kubectl run -it --rm do-aiperf --image iankoulski/do-aiperf -- bash
-
-# One line with node selector and volume mount
-#kubectl run do-aiperf --image=iankoulski/do-aiperf --overrides='{"spec": {"nodeSelector": {"nvidia.com/gpu.present": "true"},"containers": [{"name": "do-aiperf","image": "iankoulski/do-aiperf","volumeMounts": [{"name": "fsx-vol", "mountPath": "/shared"}]}],"volumes": [{"name": "fsx-vol", "persistentVolumeClaim": {"claimName": "fsx-pvc"}}]}}' -- bash 
-
-# Multi line with node selector and volume mount
-kubectl run do-aiperf \
-  --image=iankoulski/do-aiperf \
-  --overrides='{
-    "spec": {
-      "nodeSelector": {"nvidia.com/gpu.present": "true"},
-      "containers": [{
-        "name": "do-aiperf",
-        "image": "iankoulski/do-aiperf",
-        "volumeMounts": [{"name": "fsx-vol", "mountPath": "/shared"}]
-      }],
-      "volumes": [{"name": "fsx-vol", "persistentVolumeClaim": {"claimName": "fsx-pvc"}}]
-    }
-  }' 
-
-
+export CMD="kubectl run do-aiperf --image=iankoulski/do-aiperf --overrides='{\"spec\": {\"nodeSelector\": {\"nvidia.com/gpu.present\": \"true\"}, \"containers\": [{\"name\": \"do-aiperf\", \"image\": \"iankoulski/do-aiperf\", \"volumeMounts\": [{\"name\": \"fsx-vol\", \"mountPath\": \"/shared\"}]}], \"volumes\": [{\"name\": \"fsx-vol\", \"persistentVolumeClaim\": {\"claimName\": \"fsx-pvc\"}}]}}'"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
 
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-pod-stop.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-pod-stop.sh
@@ -3,5 +3,9 @@
 # Ref: https://github.com/iankoulski/do-aiperf
 
 echo "Removing do-aiperf pod ..."
-kubectl delete pod do-aiperf
 
+export CMD="kubectl delete pod do-aiperf"
+
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-logs.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-logs.sh
@@ -2,5 +2,8 @@
 
 source .env
 
-kubectl -n ${NAMESPACE} logs -f $(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1)
+export CMD="kubectl -n ${NAMESPACE} logs -f \$(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1)"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-run.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-run.sh
@@ -4,6 +4,8 @@ source .env
 
 cat aiperf-sweep.yaml-template | envsubst > aiperf-sweep.yaml
 
-kubectl apply -f ./aiperf-sweep.yaml
+export CMD="kubectl apply -f ./aiperf-sweep.yaml"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
 
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-stop.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-stop.sh
@@ -2,5 +2,8 @@
 
 source .env
 
-kubectl -n ${NAMESPACE} delete pod $(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1)
+export CMD="kubectl -n ${NAMESPACE} delete pod \$(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1)"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-summary.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/aiperf-sweep-summary.sh
@@ -2,7 +2,14 @@
 
 source .env
 
-kubectl -n ${NAMESPACE} cp -f $(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1):/tmp/run-bundle/summary.json ./sweep-summary.json
+export CMD="kubectl -n ${NAMESPACE} cp -f \$(kubectl get pods | grep aiperf-sweep | cut -d ' ' -f 1):/tmp/run-bundle/summary.json ./sweep-summary.json"
 
-cat ./aiperf-sweep-summary.json
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
 
+eval "${CMD}"
+
+export CMD="cat ./sweep-summary.json"
+
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/config.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/config.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
 
-vi .env
+export CMD="vi .env"
 
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+eval "${CMD}"
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/models-health.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/models-health.sh
@@ -1,14 +1,11 @@
 #!/bin/bash
 
-# Run a curl pod and send a request to the /v1/models endpoin
+# Run a curl pod and send a request to the /health endpoint
 
 source .env
 
-CMD="kubectl run -it --rm do-curl --image iankoulski/do-curl --restart Never -- bash -c \"curl ${SERVICE_URL}/health | jq .\""
+export CMD="kubectl run -it --rm do-curl --image iankoulski/do-curl --restart Never -- bash -c \"curl ${SERVICE_URL}/health | jq .\""
 
-if [ ! "$verbose" == "false" ]; then
-	echo -e "\n${CMD}\n"
-fi
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
 
 eval "${CMD}"
-
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/models-list.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/models-list.sh
@@ -1,14 +1,11 @@
 #!/bin/bash
 
-# Run a curl pod and send a request to the /v1/models endpoin
+# Run a curl pod and send a request to the /v1/models endpoint
 
 source .env
 
-CMD="kubectl run -it --rm do-curl --image iankoulski/do-curl --restart Never -- bash -c \"curl ${SERVICE_URL}/v1/models | jq .\""
+export CMD="kubectl run -it --rm do-curl --image iankoulski/do-curl --restart Never -- bash -c \"curl ${SERVICE_URL}/v1/models | jq .\""
 
-if [ ! "$verbose" == "false" ]; then
-	echo -e "\n${CMD}\n"
-fi
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
 
 eval "${CMD}"
-
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-aiperf.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-aiperf.sh
@@ -5,7 +5,7 @@
 
 source .env
 
-# Start the do-aiperf pod 
+# Start the do-aiperf pod
 source ./aiperf-pod-run.sh
 
 # Wait for pod to be running
@@ -14,22 +14,12 @@ kubectl wait --for=condition=Ready pod/do-aiperf --timeout=120s
 
 # Execute the aiperf command interactively
 echo "Executing aiperf command in do-aiperf pod ..."
-kubectl exec -it do-aiperf -- \
-  aiperf profile \
-    --model "${MODEL_NAME}" \
-    --tokenizer "${MODEL_PATH}" \
-    --url "${SERVICE_URL}" \
-    --transport http \
-    --endpoint-type chat \
-    --streaming \
-    --concurrency 10 \
-    --request-count 100 \
-    --synthetic-input-tokens-mean 1024 \
-    --synthetic-input-tokens-stddev 0 \
-    --output-tokens-mean 512 \
-    --extra-inputs "ignore_eos:true" \
-    --random-seed 42
+
+export CMD="kubectl exec -it do-aiperf -- aiperf profile --model \"${MODEL_NAME}\" --tokenizer \"${MODEL_PATH}\" --url \"${SERVICE_URL}\" --transport http --endpoint-type chat --streaming --concurrency 10 --request-count 100 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 512 --extra-inputs \"ignore_eos:true\" --random-seed 42"
+
+if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
+
+eval "${CMD}"
 
 # Clean up
 source ./aiperf-pod-stop.sh
-
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-chat-completions.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-chat-completions.sh
diff --git a/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-completions.sh b/Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/test/test-completions.sh
