Skip to content

Commit 87a108d

Browse files
smoellsmoell
committed
Lambda fix for CDK
1 parent b02e9d6 commit 87a108d

2 files changed

Lines changed: 294 additions & 60 deletions

File tree

‎infrastructure/scripts/cleanup/monitoring.sh‎

Lines changed: 285 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,44 +2,299 @@
22

33
set -euo pipefail
44

5+
log() {
6+
echo "[$(date +'%H:%M:%S')] $*"
7+
}
8+
9+
# --- Configuration ---
510
NAMESPACE="monitoring"
11+
GRAFANA_SECRET_NAME="grafana-admin"
12+
SECRET_NAME="grafana-webhook-credentials"
13+
PARAM_NAME="/unicornstore/prometheus/internal-dns"
14+
15+
# Temporary files to clean up
16+
VALUES_FILE="prometheus-values.yaml"
17+
EXTRA_SCRAPE_FILE="extra-scrape-configs.yaml"
18+
DATASOURCE_FILE="grafana-datasource.yaml"
19+
DASHBOARD_JSON_FILE="jvm-dashboard.json"
20+
DASHBOARD_PROVISIONING_FILE="dashboard-provisioning.yaml"
21+
ALERT_RULE_FILE="grafana-alert-rules.yaml"
22+
GRAFANA_VALUES_FILE="grafana-values.yaml"
23+
LAMBDA_ALERT_RULE_FILE="lambda-alert-rule.json"
24+
NOTIFICATION_POLICY_CONFIGMAP_FILE="notification-policy.yaml"
25+
26+
cleanup_temp_files() {
27+
log "🧹 Cleaning up temporary files..."
28+
rm -f "$VALUES_FILE" "$EXTRA_SCRAPE_FILE" "$DATASOURCE_FILE" "$NOTIFICATION_POLICY_CONFIGMAP_FILE" \
29+
"$DASHBOARD_JSON_FILE" "$DASHBOARD_PROVISIONING_FILE" \
30+
"$ALERT_RULE_FILE" "$GRAFANA_VALUES_FILE" "$LAMBDA_ALERT_RULE_FILE" \
31+
"grafana-credentials.txt"
32+
}
33+
trap cleanup_temp_files EXIT
34+
35+
log "🚨 Starting monitoring stack cleanup..."
36+
37+
# --- Get Grafana credentials before cleanup ---
38+
GRAFANA_USER="admin"
39+
GRAFANA_PASSWORD=""
40+
41+
if kubectl get secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
42+
GRAFANA_PASSWORD=$(kubectl get secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" -o jsonpath="{.data.password}" | base64 --decode)
43+
log "📋 Retrieved Grafana password from existing secret"
44+
fi
45+
46+
# Get Grafana LoadBalancer hostname before cleanup
47+
GRAFANA_LB=$(kubectl get svc grafana -n "$NAMESPACE" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || true)
48+
if [[ -n "$GRAFANA_LB" && "$GRAFANA_LB" != "<no value>" ]]; then
49+
GRAFANA_URL="http://$GRAFANA_LB"
50+
log "📋 Found Grafana URL: $GRAFANA_URL"
51+
fi
52+
53+
# Get Prometheus LoadBalancer hostname before cleanup
54+
PROM_LB_HOSTNAME=$(kubectl get svc prometheus-server -n "$NAMESPACE" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || true)
55+
if [[ -n "$PROM_LB_HOSTNAME" && "$PROM_LB_HOSTNAME" != "<no value>" ]]; then
56+
log "📋 Found Prometheus hostname: $PROM_LB_HOSTNAME"
57+
fi
58+
59+
# --- Clean up Grafana alert rules (if Grafana is accessible) ---
60+
if [[ -n "$GRAFANA_LB" && -n "$GRAFANA_PASSWORD" ]]; then
61+
log "🔧 Cleaning up Grafana alert rules..."
62+
63+
# Wait briefly for Grafana to be accessible
64+
for i in {1..5}; do
65+
if curl -s -o /dev/null -w "%{http_code}" -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/health" | grep -q "200"; then
66+
log "✅ Grafana is accessible for cleanup"
67+
break
68+
fi
69+
log "⏳ Waiting for Grafana access... ($i/5)"
70+
sleep 2
71+
done
72+
73+
# Delete alert rules
74+
ALERT_RULES=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/v1/provisioning/alert-rules" 2>/dev/null || echo "[]")
75+
if [[ "$ALERT_RULES" != "[]" ]]; then
76+
echo "$ALERT_RULES" | jq -r '.[].uid' | while read -r rule_uid; do
77+
if [[ -n "$rule_uid" && "$rule_uid" != "null" ]]; then
78+
log "🗑️ Deleting alert rule: $rule_uid"
79+
curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \
80+
"$GRAFANA_URL/api/v1/provisioning/alert-rules/$rule_uid" >/dev/null || true
81+
fi
82+
done
83+
fi
84+
85+
# Delete contact points
86+
CONTACT_POINTS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/v1/provisioning/contact-points" 2>/dev/null || echo "[]")
87+
if [[ "$CONTACT_POINTS" != "[]" ]]; then
88+
echo "$CONTACT_POINTS" | jq -r '.[] | select(.name=="lambda-webhook") | .uid' | while read -r cp_uid; do
89+
if [[ -n "$cp_uid" && "$cp_uid" != "null" ]]; then
90+
log "🗑️ Deleting contact point: $cp_uid"
91+
curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \
92+
"$GRAFANA_URL/api/v1/provisioning/contact-points/$cp_uid" >/dev/null || true
93+
fi
94+
done
95+
fi
96+
97+
# Delete folders
98+
FOLDERS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/folders" 2>/dev/null || echo "[]")
99+
if [[ "$FOLDERS" != "[]" ]]; then
100+
echo "$FOLDERS" | jq -r '.[] | select(.title=="Unicorn Store Dashboards") | .uid' | while read -r folder_uid; do
101+
if [[ -n "$folder_uid" && "$folder_uid" != "null" ]]; then
102+
log "🗑️ Deleting folder: $folder_uid"
103+
curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \
104+
"$GRAFANA_URL/api/folders/$folder_uid" >/dev/null || true
105+
fi
106+
done
107+
fi
108+
fi
109+
110+
# --- Clean up Prometheus LoadBalancer Security Group rules ---
111+
if [[ -n "$PROM_LB_HOSTNAME" ]]; then
112+
log "🔐 Cleaning up Prometheus LoadBalancer Security Group rules..."
113+
114+
VPC_ID=$(aws ec2 describe-vpcs --filters "Name=tag:Name,Values=unicornstore-vpc" --query "Vpcs[0].VpcId" --output text 2>/dev/null || true)
115+
if [[ -n "$VPC_ID" && "$VPC_ID" != "None" ]]; then
116+
VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --query "Vpcs[0].CidrBlock" --output text 2>/dev/null || true)
117+
118+
LB_ARN=$(aws elbv2 describe-load-balancers --output json 2>/dev/null | jq -r \
119+
--arg dns "$PROM_LB_HOSTNAME" '
120+
.LoadBalancers[] | select(.DNSName == $dns) | .LoadBalancerArn' || true)
121+
122+
if [[ -n "$LB_ARN" ]]; then
123+
ILB_SG_ID=$(aws elbv2 describe-load-balancers \
124+
--load-balancer-arns "$LB_ARN" \
125+
--query "LoadBalancers[0].SecurityGroups[0]" \
126+
--output text 2>/dev/null || true)
127+
128+
if [[ -n "$ILB_SG_ID" && "$ILB_SG_ID" != "None" ]]; then
129+
log "🗑️ Removing security group rule from $ILB_SG_ID"
130+
aws ec2 revoke-security-group-ingress \
131+
--group-id "$ILB_SG_ID" \
132+
--protocol tcp \
133+
--port 9090 \
134+
--cidr "$VPC_CIDR" \
135+
--output text 2>/dev/null || log "ℹ️ Security group rule may not exist"
136+
fi
137+
fi
138+
fi
139+
fi
140+
141+
# --- Uninstall Helm releases ---
142+
log "🗑️ Uninstalling Helm releases..."
143+
144+
if helm list -n "$NAMESPACE" | grep -q "grafana"; then
145+
log "🗑️ Uninstalling Grafana..."
146+
helm uninstall grafana --namespace "$NAMESPACE" || log "⚠️ Failed to uninstall Grafana"
147+
fi
148+
149+
if helm list -n "$NAMESPACE" | grep -q "prometheus"; then
150+
log "🗑️ Uninstalling Prometheus..."
151+
helm uninstall prometheus --namespace "$NAMESPACE" || log "⚠️ Failed to uninstall Prometheus"
152+
fi
153+
154+
# --- Clean up Kubernetes resources ---
155+
log "🗑️ Cleaning up Kubernetes resources..."
156+
157+
# Delete ConfigMaps
158+
kubectl delete configmap unicornstore-datasource -n "$NAMESPACE" 2>/dev/null || log "ℹ️ ConfigMap unicornstore-datasource not found"
159+
kubectl delete configmap unicornstore-dashboard -n "$NAMESPACE" 2>/dev/null || log "ℹ️ ConfigMap unicornstore-dashboard not found"
160+
kubectl delete configmap prometheus-extra-scrape -n "$NAMESPACE" 2>/dev/null || log "ℹ️ ConfigMap prometheus-extra-scrape not found"
161+
kubectl delete configmap unicornstore-notification-policy -n "$NAMESPACE" 2>/dev/null || log "ℹ️ ConfigMap unicornstore-notification-policy not found"
162+
163+
# Delete Secrets
164+
kubectl delete secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" 2>/dev/null || log "ℹ️ Secret $GRAFANA_SECRET_NAME not found"
165+
166+
# Delete PVCs (Persistent Volume Claims)
167+
log "🗑️ Cleaning up Persistent Volume Claims..."
168+
kubectl get pvc -n "$NAMESPACE" -o name 2>/dev/null | while read -r pvc; do
169+
if [[ -n "$pvc" ]]; then
170+
log "🗑️ Deleting $pvc"
171+
kubectl delete "$pvc" -n "$NAMESPACE" || log "⚠️ Failed to delete $pvc"
172+
fi
173+
done
174+
175+
# Wait for PVCs to be deleted
176+
log "⏳ Waiting for PVCs to be fully deleted..."
177+
for i in {1..30}; do
178+
PVC_COUNT=$(kubectl get pvc -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l || echo "0")
179+
if [[ "$PVC_COUNT" -eq 0 ]]; then
180+
log "✅ All PVCs deleted"
181+
break
182+
fi
183+
log "⏳ Waiting for $PVC_COUNT PVCs to be deleted... ($i/30)"
184+
sleep 5
185+
done
186+
187+
# --- Delete namespace ---
188+
log "🗑️ Deleting namespace $NAMESPACE..."
189+
kubectl delete namespace "$NAMESPACE" --timeout=60s 2>/dev/null || log "⚠️ Failed to delete namespace or namespace not found"
190+
191+
# Wait for namespace deletion
192+
log "⏳ Waiting for namespace deletion..."
193+
for i in {1..30}; do
194+
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
195+
log "✅ Namespace $NAMESPACE deleted"
196+
break
197+
fi
198+
log "⏳ Waiting for namespace deletion... ($i/30)"
199+
sleep 5
200+
done
201+
202+
# --- Clean up AWS resources ---
203+
log "🗑️ Cleaning up AWS resources..."
204+
205+
# Delete SSM Parameter
206+
if aws ssm get-parameter --name "$PARAM_NAME" >/dev/null 2>&1; then
207+
log "🗑️ Deleting SSM parameter $PARAM_NAME"
208+
aws ssm delete-parameter --name "$PARAM_NAME" || log "⚠️ Failed to delete SSM parameter"
209+
else
210+
log "ℹ️ SSM parameter $PARAM_NAME not found"
211+
fi
212+
213+
# Delete Secrets Manager secret
214+
if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then
215+
log "🗑️ Deleting Secrets Manager secret $SECRET_NAME"
216+
aws secretsmanager delete-secret --secret-id "$SECRET_NAME" --force-delete-without-recovery || log "⚠️ Failed to delete secret"
217+
else
218+
log "ℹ️ Secrets Manager secret $SECRET_NAME not found"
219+
fi
220+
221+
# --- Clean up Lambda Function URL (optional) ---
222+
log "🔧 Checking Lambda Function URL..."
223+
LAMBDA_FUNCTION_NAME="unicornstore-thread-dump-lambda"
224+
225+
if aws lambda get-function --function-name "$LAMBDA_FUNCTION_NAME" >/dev/null 2>&1; then
226+
# Check if Function URL exists
227+
if aws lambda get-function-url-config --function-name "$LAMBDA_FUNCTION_NAME" >/dev/null 2>&1; then
228+
log "🗑️ Removing Lambda Function URL..."
229+
aws lambda delete-function-url-config --function-name "$LAMBDA_FUNCTION_NAME" || log "⚠️ Failed to delete Function URL"
230+
231+
# Remove the permission
232+
aws lambda remove-permission \
233+
--function-name "$LAMBDA_FUNCTION_NAME" \
234+
--statement-id AllowPublicAccess 2>/dev/null || log "ℹ️ Permission may not exist"
235+
else
236+
log "ℹ️ Lambda Function URL not found"
237+
fi
238+
else
239+
log "ℹ️ Lambda function $LAMBDA_FUNCTION_NAME not found"
240+
fi
241+
242+
# --- Remove Helm repositories (optional) ---
243+
log "🗑️ Cleaning up Helm repositories..."
244+
helm repo remove prometheus-community 2>/dev/null || log "ℹ️ prometheus-community repo not found"
245+
helm repo remove grafana 2>/dev/null || log "ℹ️ grafana repo not found"
6246

7-
# ConfigMap-Namen
8-
ALERT_RULE_CONFIGMAP_NAME="unicornstore-alert-rule"
9-
CONTACT_POINT_CONFIGMAP_NAME="unicornstore-contact-point"
10-
DATASOURCE_CONFIGMAP_NAME="unicornstore-datasource"
11-
DASHBOARD_CONFIGMAP_NAME="unicornstore-dashboard"
12-
SCRAPE_CONFIGMAP_NAME="prometheus-extra-scrape"
247+
# --- Final validation ---
248+
log "🔍 Validating cleanup..."
13249

14-
# Helm Releases
15-
GRAFANA_HELM_RELEASE="grafana"
16-
PROMETHEUS_HELM_RELEASE="prometheus"
250+
# Check if namespace still exists
251+
if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
252+
log "⚠️ Warning: Namespace $NAMESPACE still exists"
253+
else
254+
log "✅ Namespace $NAMESPACE successfully deleted"
255+
fi
17256

18-
echo "🧹 Starting cleanup of Kubernetes-based monitoring stack..."
257+
# Check if Helm releases still exist
258+
REMAINING_RELEASES=$(helm list -A | grep -E "(prometheus|grafana)" || true)
259+
if [[ -n "$REMAINING_RELEASES" ]]; then
260+
log "⚠️ Warning: Some Helm releases may still exist:"
261+
echo "$REMAINING_RELEASES"
262+
else
263+
log "✅ All monitoring Helm releases cleaned up"
264+
fi
19265

20-
# Delete alert rule configmap
21-
echo "🔸 Deleting Alert Rule ConfigMap: $ALERT_RULE_CONFIGMAP_NAME"
22-
kubectl delete configmap "$ALERT_RULE_CONFIGMAP_NAME" -n "$NAMESPACE" --ignore-not-found
266+
# Check AWS resources
267+
if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then
268+
log "⚠️ Warning: Secrets Manager secret $SECRET_NAME still exists"
269+
else
270+
log "✅ Secrets Manager secret cleaned up"
271+
fi
23272

24-
# Delete contact point configmap
25-
echo "🔸 Deleting Contact Point ConfigMap: $CONTACT_POINT_CONFIGMAP_NAME"
26-
kubectl delete configmap "$CONTACT_POINT_CONFIGMAP_NAME" -n "$NAMESPACE" --ignore-not-found
273+
if aws ssm get-parameter --name "$PARAM_NAME" >/dev/null 2>&1; then
274+
log "⚠️ Warning: SSM parameter $PARAM_NAME still exists"
275+
else
276+
log "✅ SSM parameter cleaned up"
277+
fi
27278

28-
# Delete datasource configmap
29-
echo "🔸 Deleting Datasource ConfigMap: $DATASOURCE_CONFIGMAP_NAME"
30-
kubectl delete configmap "$DATASOURCE_CONFIGMAP_NAME" -n "$NAMESPACE" --ignore-not-found
279+
log "✅ Monitoring stack cleanup completed!"
280+
log "ℹ️ Note: Some AWS resources (like Load Balancers) may take additional time to fully terminate"
281+
log "ℹ️ Note: Persistent Volumes may need manual cleanup if they were not automatically deleted"
31282

32-
# Delete dashboard configmap
33-
echo "🔸 Deleting Dashboard ConfigMap: $DASHBOARD_CONFIGMAP_NAME"
34-
kubectl delete configmap "$DASHBOARD_CONFIGMAP_NAME" -n "$NAMESPACE" --ignore-not-found
283+
# --- Optional: List remaining resources for manual cleanup ---
284+
log "📋 Checking for any remaining resources that may need manual cleanup..."
35285

36-
# Delete extra scrape config
37-
echo "🔸 Deleting Extra Scrape ConfigMap: $SCRAPE_CONFIGMAP_NAME"
38-
kubectl delete configmap "$SCRAPE_CONFIGMAP_NAME" -n "$NAMESPACE" --ignore-not-found
286+
# Check for remaining PVs
287+
REMAINING_PVS=$(kubectl get pv | grep "$NAMESPACE" || true)
288+
if [[ -n "$REMAINING_PVS" ]]; then
289+
log "⚠️ Warning: Found Persistent Volumes that may need manual cleanup:"
290+
echo "$REMAINING_PVS"
291+
fi
39292

40-
# Uninstall Helm releases
41-
echo "🔸 Uninstalling Helm release: $GRAFANA_HELM_RELEASE"
42-
helm uninstall "$GRAFANA_HELM_RELEASE" -n "$NAMESPACE" || echo "⚠️ Grafana release not found or already uninstalled."
293+
# Check for remaining Load Balancers
294+
REMAINING_LBS=$(aws elbv2 describe-load-balancers --output table | grep -E "(prometheus|grafana)" || true)
295+
if [[ -n "$REMAINING_LBS" ]]; then
296+
log "⚠️ Warning: Found Load Balancers that may need manual cleanup:"
297+
echo "$REMAINING_LBS"
298+
fi
43299

44-
echo "🔸 Uninstalling Helm release: $PROMETHEUS_HELM_RELEASE"
45-
helm uninstall "$PROMETHEUS_HELM_RELEASE" -n "$NAMESPACE"
300+
log "🎉 Cleanup script execution completed!"

‎infrastructure/scripts/setup/monitoring.sh‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,11 @@ log() {
66
echo "[$(date +'%H:%M:%S')] $*"
77
}
88

9-
set -x
10-
119
# --- Configuration ---
1210
NAMESPACE="monitoring"
1311
GRAFANA_SECRET_NAME="grafana-admin"
1412
GRAFANA_USER="admin"
15-
16-
source /etc/profile.d/workshop.sh
17-
18-
# Use IDE_PASSWORD if set; otherwise generate random Grafana password
19-
if [ -z "$IDE_PASSWORD" ]; then
20-
echo "⚠️ Warning: GRAFANA_PASSWORD is not set via IDE_PASSWORD. A random password will be generated."
21-
GRAFANA_PASSWORD="$(openssl rand -base64 16 | tr -d '\n')"
22-
else
23-
GRAFANA_PASSWORD="$IDE_PASSWORD"
24-
fi
25-
26-
echo "GRAFANA_PASSWORD is $GRAFANA_PASSWORD"
13+
GRAFANA_PASSWORD="$(openssl rand -base64 16 | tr -d '\n')"
2714

2815
VALUES_FILE="prometheus-values.yaml"
2916
EXTRA_SCRAPE_FILE="extra-scrape-configs.yaml"
@@ -45,23 +32,13 @@ trap cleanup EXIT
4532

4633
# -- Generate secure username and password for webhook
4734
WEBHOOK_USER="grafana-alerts"
48-
WEBHOOK_PASSWORD="$(openssl rand -base64 16 | tr -d '\n')"
49-
SECRET_NAME="grafana-webhook-credentials"
50-
SECRET_VALUE="{\"username\":\"$WEBHOOK_USER\",\"password\":\"$WEBHOOK_PASSWORD\"}"
51-
52-
# -- Create or update the secret for webhook
53-
if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then
54-
echo "🔁 Secret '$SECRET_NAME' exists. Updating..."
55-
aws secretsmanager put-secret-value \
56-
--secret-id "$SECRET_NAME" \
57-
--secret-string "$SECRET_VALUE"
58-
else
59-
echo "➕ Creating secret '$SECRET_NAME'..."
60-
aws secretsmanager create-secret \
61-
--name "$SECRET_NAME" \
35+
WEBHOOK_PASSWORD=$(openssl rand -base64 16 | tr -d '\n')
36+
37+
# -- Create the secret for webhook
38+
aws secretsmanager create-secret \
39+
--name grafana-webhook-credentials \
6240
--description "Basic auth credentials for Grafana webhook to Lambda" \
63-
--secret-string "$SECRET_VALUE"
64-
fi
41+
--secret-string "{\"username\":\"$WEBHOOK_USER\",\"password\":\"$WEBHOOK_PASSWORD\"}"
6542

6643
echo "Webhook credentials created:"
6744
echo "Username: $WEBHOOK_USER"
@@ -264,6 +241,8 @@ GRAFANA_URL="http://$GRAFANA_LB"
264241

265242
# 1. Search for a dashboard containing "JVM" in the title
266243

244+
set -x
245+
267246
log "⏳ Waiting for Grafana to become healthy..."
268247
for i in {1..20}; do
269248
STATUS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/health" | jq -r .database || true)

0 commit comments

Comments
 (0)