|
| 1 | +#!/usr/bin/env bash |
| 2 | +# cleanup-ephemeral-stacks.sh — Delete ephemeral CloudFormation stacks older than MAX_AGE_HOURS. |
| 3 | +# |
| 4 | +# Targets stacks deployed by this CDK app that do NOT have termination protection. |
| 5 | +# Handles stuck ENI cleanup (AgentCore/Lambda Hyperplane ENIs) before deletion. |
| 6 | +# |
| 7 | +# Usage: |
| 8 | +# AWS_PROFILE=abca ./scripts/cleanup-ephemeral-stacks.sh [--dry-run] [--max-age-hours N] [--prefix PREFIX] [--region REGION] |
| 9 | +# |
| 10 | +# Options: |
| 11 | +# --dry-run Show what would be deleted without acting |
| 12 | +# --max-age-hours N Delete stacks older than N hours (default: 48) |
| 13 | +# --prefix PREFIX Only target stacks matching this prefix (default: all ABCA stacks) |
| 14 | +# --region REGION AWS region to operate in (default: $AWS_DEFAULT_REGION or us-east-1) |
| 15 | +# |
| 16 | +# Safety: |
| 17 | +# - Never touches stacks with termination protection enabled |
| 18 | +# - Only targets stacks whose description starts with "ABCA Development Stack" |
| 19 | +# - Skips stacks in UPDATE_IN_PROGRESS or CREATE_IN_PROGRESS states |
| 20 | + |
| 21 | +set -euo pipefail |
| 22 | + |
| 23 | +MAX_AGE_HOURS=${MAX_AGE_HOURS:-48} |
| 24 | +DRY_RUN=false |
| 25 | +PREFIX="" |
| 26 | +REGION="${AWS_DEFAULT_REGION:-us-east-1}" |
| 27 | + |
| 28 | +while [[ $# -gt 0 ]]; do |
| 29 | + case $1 in |
| 30 | + --dry-run) DRY_RUN=true; shift ;; |
| 31 | + --max-age-hours) |
| 32 | + [[ $# -ge 2 ]] || { echo "Error: --max-age-hours requires a value" >&2; exit 1; } |
| 33 | + MAX_AGE_HOURS="$2"; shift 2 ;; |
| 34 | + --prefix) |
| 35 | + [[ $# -ge 2 ]] || { echo "Error: --prefix requires a value" >&2; exit 1; } |
| 36 | + PREFIX="$2"; shift 2 ;; |
| 37 | + --region) |
| 38 | + [[ $# -ge 2 ]] || { echo "Error: --region requires a value" >&2; exit 1; } |
| 39 | + REGION="$2"; shift 2 ;; |
| 40 | + *) echo "Unknown option: $1" >&2; exit 1 ;; |
| 41 | + esac |
| 42 | +done |
| 43 | + |
| 44 | +# Validate numeric input — guards the age arithmetic against injection/garbage. |
| 45 | +if ! [[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]]; then |
| 46 | + echo "Error: --max-age-hours must be a non-negative integer (got: '$MAX_AGE_HOURS')" >&2 |
| 47 | + exit 1 |
| 48 | +fi |
| 49 | + |
| 50 | +MAX_AGE_SECONDS=$((MAX_AGE_HOURS * 3600)) |
| 51 | +NOW=$(date +%s) |
| 52 | + |
| 53 | +# Surface the blast radius before touching anything. Confirms the operator is |
| 54 | +# pointed at the account/identity they think they are (defense in depth). |
| 55 | +CALLER_IDENTITY=$(aws sts get-caller-identity \ |
| 56 | + --region "$REGION" \ |
| 57 | + --query '[Account,Arn]' --output text 2>/dev/null) || { |
| 58 | + echo "Error: unable to resolve AWS identity (sts:GetCallerIdentity failed). Check credentials." >&2 |
| 59 | + exit 1 |
| 60 | +} |
| 61 | +ACCOUNT_ID=$(echo "$CALLER_IDENTITY" | cut -f1) |
| 62 | +CALLER_ARN=$(echo "$CALLER_IDENTITY" | cut -f2) |
| 63 | + |
| 64 | +echo "=== Ephemeral Stack Cleanup ===" |
| 65 | +echo " Account: $ACCOUNT_ID" |
| 66 | +echo " Identity: $CALLER_ARN" |
| 67 | +echo " Region: $REGION" |
| 68 | +echo " Max age: ${MAX_AGE_HOURS}h" |
| 69 | +echo " Dry run: $DRY_RUN" |
| 70 | +echo " Prefix filter: ${PREFIX:-<none>}" |
| 71 | +echo "" |
| 72 | + |
| 73 | +# List all stacks (excluding deleted ones). |
| 74 | +# DELETE_FAILED is included so a stack that previously failed to delete is |
| 75 | +# re-targeted on the next run (delete-stack is idempotent and retries it). |
| 76 | +# Capture the exit code separately from emptiness: an API failure (auth, |
| 77 | +# throttle, IAM) must NOT look like "nothing to clean" — that would exit 0 |
| 78 | +# and silently skip the whole run. |
| 79 | +if ! STACKS=$(aws cloudformation list-stacks \ |
| 80 | + --region "$REGION" \ |
| 81 | + --stack-status-filter \ |
| 82 | + CREATE_COMPLETE UPDATE_COMPLETE ROLLBACK_COMPLETE \ |
| 83 | + UPDATE_ROLLBACK_COMPLETE DELETE_FAILED \ |
| 84 | + --query 'StackSummaries[*].[StackName,CreationTime]' \ |
| 85 | + --output text); then |
| 86 | + echo "Error: cloudformation:ListStacks failed. Check credentials/permissions/region." >&2 |
| 87 | + exit 1 |
| 88 | +fi |
| 89 | + |
| 90 | +if [[ -z "$STACKS" ]]; then |
| 91 | + echo "No stacks found." |
| 92 | + exit 0 |
| 93 | +fi |
| 94 | + |
| 95 | +DELETED=0 |
| 96 | +SKIPPED=0 |
| 97 | +FAILED=0 |
| 98 | + |
| 99 | +while IFS=$'\t' read -r STACK_NAME CREATION_TIME; do |
| 100 | + # Apply prefix filter |
| 101 | + if [[ -n "$PREFIX" && "$STACK_NAME" != "$PREFIX"* ]]; then |
| 102 | + continue |
| 103 | + fi |
| 104 | + |
| 105 | + # Get stack details (description, termination protection, tags) |
| 106 | + STACK_INFO=$(aws cloudformation describe-stacks \ |
| 107 | + --region "$REGION" \ |
| 108 | + --stack-name "$STACK_NAME" \ |
| 109 | + --query 'Stacks[0].[Description,EnableTerminationProtection,StackStatus]' \ |
| 110 | + --output text 2>/dev/null) || continue |
| 111 | + |
| 112 | + DESCRIPTION=$(echo "$STACK_INFO" | cut -f1) |
| 113 | + TERMINATION_PROTECTED=$(echo "$STACK_INFO" | cut -f2) |
| 114 | + STATUS=$(echo "$STACK_INFO" | cut -f3) |
| 115 | + |
| 116 | + # Only target stacks from this CDK app. Prefix-match, not exact-match: the |
| 117 | + # app description carries an optional solution-id suffix (e.g. |
| 118 | + # "ABCA Development Stack (uksb-...)", see cdk/src/main.ts) that operators may |
| 119 | + # add or strip. An exact-equality check silently matches zero stacks whenever |
| 120 | + # the suffix is present. (Tag-based filtering would be even more robust — |
| 121 | + # tracked as a follow-up.) |
| 122 | + if [[ "$DESCRIPTION" != "ABCA Development Stack"* ]]; then |
| 123 | + continue |
| 124 | + fi |
| 125 | + |
| 126 | + # Never touch termination-protected stacks |
| 127 | + if [[ "$TERMINATION_PROTECTED" == "True" ]]; then |
| 128 | + echo " SKIP (protected): $STACK_NAME" |
| 129 | + ((SKIPPED++)) || true |
| 130 | + continue |
| 131 | + fi |
| 132 | + |
| 133 | + # Skip stacks in active transitions |
| 134 | + if [[ "$STATUS" == *"IN_PROGRESS"* ]]; then |
| 135 | + echo " SKIP (in progress): $STACK_NAME ($STATUS)" |
| 136 | + ((SKIPPED++)) || true |
| 137 | + continue |
| 138 | + fi |
| 139 | + |
| 140 | + # Check age. Parse the CreationTime to epoch seconds (GNU date, then BSD date). |
| 141 | + # CreationTime is UTC (e.g. 2026-06-18T00:23:10.123Z). The BSD branch strips |
| 142 | + # the fractional seconds and trailing Z, so it MUST parse as UTC (-u) — without |
| 143 | + # it, BSD `date -j` assumes local time and a stack reads N hours off (8h on a |
| 144 | + # PST Mac), wrongly skipping/deleting near the age boundary. GNU `date -d` |
| 145 | + # honours the Z natively. |
| 146 | + # FAIL CLOSED: if both parsers fail we cannot trust the age, so SKIP rather than |
| 147 | + # risk deleting a stack we can't prove is old enough. |
| 148 | + CREATED_EPOCH=$(date -d "$CREATION_TIME" +%s 2>/dev/null || date -j -u -f "%Y-%m-%dT%H:%M:%S" "${CREATION_TIME%%.*}" +%s 2>/dev/null || echo "") |
| 149 | + if ! [[ "$CREATED_EPOCH" =~ ^[0-9]+$ ]]; then |
| 150 | + echo " SKIP (unparseable creation time '$CREATION_TIME'): $STACK_NAME" |
| 151 | + ((SKIPPED++)) || true |
| 152 | + continue |
| 153 | + fi |
| 154 | + AGE_SECONDS=$((NOW - CREATED_EPOCH)) |
| 155 | + |
| 156 | + if [[ $AGE_SECONDS -lt $MAX_AGE_SECONDS ]]; then |
| 157 | + AGE_HOURS=$((AGE_SECONDS / 3600)) |
| 158 | + echo " SKIP (too young: ${AGE_HOURS}h): $STACK_NAME" |
| 159 | + ((SKIPPED++)) || true |
| 160 | + continue |
| 161 | + fi |
| 162 | + |
| 163 | + AGE_HOURS=$((AGE_SECONDS / 3600)) |
| 164 | + echo " TARGET: $STACK_NAME (age: ${AGE_HOURS}h, status: $STATUS)" |
| 165 | + |
| 166 | + if [[ "$DRY_RUN" == "true" ]]; then |
| 167 | + echo " [dry-run] Would delete $STACK_NAME" |
| 168 | + ((DELETED++)) || true |
| 169 | + continue |
| 170 | + fi |
| 171 | + |
| 172 | + # --- ENI cleanup (handles stuck VPC deletion) --- |
| 173 | + # Find security groups owned by this stack |
| 174 | + SG_IDS=$(aws cloudformation list-stack-resources \ |
| 175 | + --region "$REGION" \ |
| 176 | + --stack-name "$STACK_NAME" \ |
| 177 | + --query "StackResourceSummaries[?ResourceType=='AWS::EC2::SecurityGroup'].PhysicalResourceId" \ |
| 178 | + --output text 2>/dev/null) || true |
| 179 | + |
| 180 | + if [[ -n "$SG_IDS" && "$SG_IDS" != "None" ]]; then |
| 181 | + for SG_ID in $SG_IDS; do |
| 182 | + # Find ENIs attached to this security group. |
| 183 | + # shellcheck disable=SC2016 # backticks are JMESPath literal syntax for --query, must NOT expand |
| 184 | + ENIS=$(aws ec2 describe-network-interfaces \ |
| 185 | + --region "$REGION" \ |
| 186 | + --filters "Name=group-id,Values=$SG_ID" \ |
| 187 | + --query 'NetworkInterfaces[?Status==`in-use`].[NetworkInterfaceId,Attachment.AttachmentId]' \ |
| 188 | + --output text 2>/dev/null) || true |
| 189 | + |
| 190 | + if [[ -n "$ENIS" && "$ENIS" != "None" ]]; then |
| 191 | + echo " Cleaning up ENIs in security group $SG_ID..." |
| 192 | + while IFS=$'\t' read -r ENI_ID ATTACHMENT_ID; do |
| 193 | + if [[ -n "$ENI_ID" && "$ENI_ID" != "None" ]]; then |
| 194 | + echo " Force-detaching $ENI_ID ($ATTACHMENT_ID)" |
| 195 | + aws ec2 detach-network-interface \ |
| 196 | + --region "$REGION" \ |
| 197 | + --attachment-id "$ATTACHMENT_ID" \ |
| 198 | + --force 2>/dev/null || true |
| 199 | + fi |
| 200 | + done <<< "$ENIS" |
| 201 | + |
| 202 | + # Wait briefly for detachment |
| 203 | + echo " Waiting 15s for ENI detachment..." |
| 204 | + sleep 15 |
| 205 | + |
| 206 | + # Delete the ENIs |
| 207 | + AVAILABLE_ENIS=$(aws ec2 describe-network-interfaces \ |
| 208 | + --region "$REGION" \ |
| 209 | + --filters "Name=group-id,Values=$SG_ID" "Name=status,Values=available" \ |
| 210 | + --query 'NetworkInterfaces[*].NetworkInterfaceId' \ |
| 211 | + --output text 2>/dev/null) || true |
| 212 | + |
| 213 | + for ENI_ID in $AVAILABLE_ENIS; do |
| 214 | + if [[ -n "$ENI_ID" && "$ENI_ID" != "None" ]]; then |
| 215 | + echo " Deleting $ENI_ID" |
| 216 | + aws ec2 delete-network-interface \ |
| 217 | + --region "$REGION" \ |
| 218 | + --network-interface-id "$ENI_ID" 2>/dev/null || true |
| 219 | + fi |
| 220 | + done |
| 221 | + fi |
| 222 | + done |
| 223 | + fi |
| 224 | + |
| 225 | + # --- Delete the stack --- |
| 226 | + # Only count a deletion we actually initiated. Tolerate a single failure |
| 227 | + # (e.g. AccessDenied, transient throttling) without aborting the whole run — |
| 228 | + # set -e would otherwise kill the loop mid-pass and orphan later stacks. |
| 229 | + echo " Deleting stack $STACK_NAME..." |
| 230 | + # Let stderr through: this is the one call where the API error matters — |
| 231 | + # AccessDenied vs ValidationError vs throttling are diagnosed differently. |
| 232 | + # Suppressing it would leave the operator with only a generic failure line. |
| 233 | + if aws cloudformation delete-stack \ |
| 234 | + --region "$REGION" \ |
| 235 | + --stack-name "$STACK_NAME"; then |
| 236 | + ((DELETED++)) || true |
| 237 | + else |
| 238 | + echo " ERROR: delete-stack failed for $STACK_NAME (continuing)" >&2 |
| 239 | + ((FAILED++)) || true |
| 240 | + fi |
| 241 | + |
| 242 | +done <<< "$STACKS" |
| 243 | + |
| 244 | +echo "" |
| 245 | +echo "=== Summary ===" |
| 246 | +echo " Deleted: $DELETED" |
| 247 | +echo " Skipped: $SKIPPED" |
| 248 | +echo " Failed: $FAILED" |
| 249 | + |
| 250 | +if [[ "$DELETED" -gt 0 && "$DRY_RUN" == "false" ]]; then |
| 251 | + echo "" |
| 252 | + echo "Note: Stack deletion is asynchronous. Monitor with:" |
| 253 | + echo " aws cloudformation list-stacks --stack-status-filter DELETE_IN_PROGRESS --region $REGION" |
| 254 | +fi |
0 commit comments