Add orphaned test-VPC reaper and standardize dry-run across cleaners

clean_eks: after the cluster pass, reap dedicated EFA test VPCs
(tag:Name efa-test-vpc-*) left behind by failed terraform destroys.
Tears the stack down in dependency order — VPC endpoints, NAT gateways,
Elastic IPs, leftover ENIs, internet gateways, subnets, route tables,
security groups (revoke-then-delete), then the VPC — behind three safety
gates: not backing a live EKS cluster (prod + best-effort beta), no
active EC2 instances (re-checked before subnet deletion), and an age
gate from the newest NAT gateway/VPC endpoint. A cleaner:reaping tag lets
a partially-torn-down VPC resume on a later run instead of being stranded
once its age signals are gone.

Shared dry-run: add tool/clean/clean_flags.go (clean.DryRun,
RegisterCommonFlags, Skip) and gate every mutating call across all 12
cleaners. Add a -dry-run flag to each tool and a workflow_dispatch
dry_run input (DRY_RUN env: scheduled runs stay live, manual defaults to
dry-run). Normalize --tags=clean to build-flag position and move
ecs/host region args to flag.Arg(0).

Also fix pre-existing bugs surfaced while touching these files:
- clean_ecs: inverted-sign expiration deleted clusters with active tasks
- clean_security_group: break-in-select spun pagination to timeout
- clean_ebs / clean_file_system: age no-op and stale-error under-deletion

Author: sky333999

.github/workflows/clean-aws-resources.yml

@@ -7,6 +7,20 @@ on:
   schedule:
     - cron: "0 0 * * *" # Run Every Day At Midnight
   workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run: log the resources that would be deleted without deleting them"
+        type: boolean
+        required: false
+        default: true
+
+env:
+  # Scheduled (cron) runs perform real deletions. Any manual (workflow_dispatch)
+  # run defaults to a dry run unless dry_run=false is explicitly passed — so a
+  # manual trigger, including an API dispatch that omits the input, never deletes
+  # by accident. (github.event.inputs does not receive the input's UI default,
+  # hence the explicit event_name gate.)
+  DRY_RUN: ${{ github.event_name == 'schedule' && 'false' || (github.event.inputs.dry_run || 'true') }}
 
 jobs:
   clean-opensource-ami:
@@ -26,7 +40,7 @@ jobs:
 
       - name: Clean old ami
         working-directory: tool/clean
-        run: go run ./clean_ami/clean_ami.go --tags=clean
+        run: go run --tags=clean ./clean_ami/clean_ami.go -dry-run=${DRY_RUN}
 
   clean-old-file-systems:
     runs-on: ubuntu-latest
@@ -45,7 +59,7 @@ jobs:
 
       - name: Clean old file system
         working-directory: tool/clean
-        run: go run ./clean_file_system/clean_file_system.go --tags=clean
+        run: go run --tags=clean ./clean_file_system/clean_file_system.go -dry-run=${DRY_RUN}
 
   clean-opensource-dedicated-hosts:
     runs-on: ubuntu-latest
@@ -64,7 +78,7 @@ jobs:
 
       - name: Clean old dedicated host
         working-directory: tool/clean
-        run: go run ./clean_dedicated_host/clean_dedicated_host.go --tags=clean
+        run: go run --tags=clean ./clean_dedicated_host/clean_dedicated_host.go -dry-run=${DRY_RUN}
 
   clean-internal-dedicated-hosts:
     runs-on: ubuntu-latest
@@ -98,7 +112,7 @@ jobs:
 
       - name: Clean old dedicated host
         working-directory: tool/clean
-        run: go run ./clean_dedicated_host/clean_dedicated_host.go --tags=clean
+        run: go run --tags=clean ./clean_dedicated_host/clean_dedicated_host.go -dry-run=${DRY_RUN}
 
   clean-hosts:
     runs-on: ubuntu-latest
@@ -138,7 +152,7 @@ jobs:
 
       - name: Clean old host
         working-directory: tool/clean
-        run: go run ./clean_host/clean_host.go ${{ matrix.region }}
+        run: go run ./clean_host/clean_host.go -dry-run=${DRY_RUN} ${{ matrix.region }}
 
   clean-hosts-china:
     runs-on: ubuntu-latest
@@ -157,7 +171,7 @@ jobs:
 
       - name: Clean old hosts
         working-directory: tool/clean
-        run: go run ./clean_host/clean_host.go cn-north-1
+        run: go run ./clean_host/clean_host.go -dry-run=${DRY_RUN} cn-north-1
 
   clean-ecs-resources:
     runs-on: ubuntu-latest
@@ -176,7 +190,7 @@ jobs:
 
       - name: Clean old ecs resources
         working-directory: tool/clean
-        run: go run --tags=clean ./clean_ecs/clean_ecs.go us-west-2
+        run: go run --tags=clean ./clean_ecs/clean_ecs.go -dry-run=${DRY_RUN} us-west-2
 
   clean-eks-clusters:
     runs-on: ubuntu-latest
@@ -195,7 +209,7 @@ jobs:
 
       - name: Clean old eks cluster
         working-directory: tool/clean
-        run: go run ./clean_eks/clean_eks.go --tags=clean
+        run: go run ./clean_eks/clean_eks.go -dry-run=${DRY_RUN}
   clean-ebs-volumes:
     runs-on: ubuntu-latest
     permissions:
@@ -213,7 +227,7 @@ jobs:
 
       - name: Clean old unused ebs volumes
         working-directory: tool/clean
-        run: go run ./clean_ebs/clean_ebs.go --tags=clean
+        run: go run ./clean_ebs/clean_ebs.go -dry-run=${DRY_RUN}
 
   clean-asg:
     runs-on: ubuntu-latest
@@ -232,7 +246,7 @@ jobs:
 
       - name: Clean old asg
         working-directory: tool/clean
-        run: go run ./clean_auto_scaling_groups/clean_auto_scaling_groups.go --tags=clean
+        run: go run ./clean_auto_scaling_groups/clean_auto_scaling_groups.go -dry-run=${DRY_RUN}
 
   clean-launch-configs:
     runs-on: ubuntu-latest
@@ -251,7 +265,7 @@ jobs:
 
       - name: Clean old launch configuration
         working-directory: tool/clean
-        run: go run ./clean_launch_configuration/clean_launch_configuration.go --tags=clean
+        run: go run ./clean_launch_configuration/clean_launch_configuration.go -dry-run=${DRY_RUN}
   clean-iam-roles:
     runs-on: ubuntu-latest
     permissions:
@@ -269,7 +283,7 @@ jobs:
 
       - name: Clean old IAM roles
         working-directory: tool/clean
-        run: go run ./clean_iam_roles/clean_iam_roles.go --tags=clean
+        run: go run ./clean_iam_roles/clean_iam_roles.go -dry-run=${DRY_RUN}
   clean-log-groups:
     runs-on: ubuntu-latest
     permissions:
@@ -287,7 +301,7 @@ jobs:
 
       - name: Clean old Log Groups
         working-directory: tool/clean
-        run: go run ./clean_log_group/clean_log_group.go 
+        run: go run ./clean_log_group/clean_log_group.go -dry-run=${DRY_RUN}
   clean-security-groups:
     runs-on: ubuntu-latest
     permissions:
@@ -307,5 +321,5 @@ jobs:
         working-directory: tool/clean
         run: |
           set -e
-          go run ./clean_security_group/clean_security_group.go || { echo "Failed to clean security groups"; exit 1; }
+          go run ./clean_security_group/clean_security_group.go -dry-run=${DRY_RUN} || { echo "Failed to clean security groups"; exit 1; }
 

tool/clean/clean_ami/clean_ami.go

@@ -9,6 +9,7 @@ package main
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"log"
 	"sort"
@@ -56,6 +57,8 @@ var imagePrefixes = []string{
 }
 
 func main() {
+	clean.RegisterCommonFlags()
+	flag.Parse()
 	err := cleanAMIs()
 	if err != nil {
 		log.Fatalf("errors cleaning %v", err)
@@ -88,6 +91,9 @@ func deregisterAMIs(ctx context.Context, ec2client *ec2.Client, images []types.I
 	for _, image := range images {
 		if image.Name != nil && image.ImageId != nil && image.CreationDate != nil {
 			log.Printf("Try to delete ami %v tags %v image id %v image creation date raw %v", *image.Name, image.Tags, *image.ImageId, *image.CreationDate)
+			if clean.Skip("deregister ami %v (%v)", *image.Name, *image.ImageId) {
+				continue
+			}
 			deregisterImageInput := &ec2.DeregisterImageInput{ImageId: image.ImageId}
 			_, err := ec2client.DeregisterImage(ctx, deregisterImageInput)
 

tool/clean/clean_auto_scaling_groups/clean_auto_scaling_groups.go

@@ -5,6 +5,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"log"
 	"time"
 
@@ -18,6 +19,8 @@ import (
 
 // Clean eks clusters if they have been open longer than 7 day
 func main() {
+	clean.RegisterCommonFlags()
+	flag.Parse()
 	err := cleanAutoScalingGroups()
 	if err != nil {
 		log.Fatalf("errors cleaning %v", err)
@@ -60,6 +63,9 @@ func terminateAutoScaling(ctx context.Context, client *autoscaling.Client, filte
 	for _, group := range describeAutoScalingGroupsOutput.AutoScalingGroups {
 		if expirationDateCluster.After(*group.CreatedTime) {
 			log.Printf("try to delete auto scaling group %s", *group.AutoScalingGroupName)
+			if clean.Skip("delete auto scaling group %s", *group.AutoScalingGroupName) {
+				continue
+			}
 			deleteAutoScalingGroupInput := autoscaling.DeleteAutoScalingGroupInput{
 				AutoScalingGroupName: group.AutoScalingGroupName,
 				ForceDelete:          aws.Bool(true),

tool/clean/clean_dedicated_host/clean_dedicated_host.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"log"
 	"time"
 
@@ -24,6 +25,8 @@ const tagName = "tag:Name"
 const tagValue = "IntegrationTestMacDedicatedHost"
 
 func main() {
+	clean.RegisterCommonFlags()
+	flag.Parse()
 	err := cleanDedicatedHost()
 	if err != nil {
 		log.Fatalf("errors cleaning %v", err)
@@ -62,6 +65,9 @@ func cleanDedicatedHost() error {
 	}
 
 	log.Printf("Dedicated hosts to release %v", dedicatedHostIds)
+	if clean.Skip("release dedicated hosts %v", dedicatedHostIds) {
+		return nil
+	}
 	releaseDedicatedHost := ec2.ReleaseHostsInput{HostIds: dedicatedHostIds}
 	_, err = ec2client.ReleaseHosts(cxt, &releaseDedicatedHost)
 	return err

tool/clean/clean_ebs/clean_ebs.go

@@ -5,6 +5,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"log"
 	"time"
 
@@ -18,6 +19,8 @@ import (
 
 // Clean ebs volumes if they have been open longer than 7 day and unused
 func main() {
+	clean.RegisterCommonFlags()
+	flag.Parse()
 	err := cleanVolumes()
 	if err != nil {
 		log.Fatalf("errors cleaning %v", err)
@@ -39,6 +42,10 @@ func cleanVolumes() error {
 
 func deleteUnusedVolumes(ctx context.Context, client *ec2.Client) error {
 
+	// KeepDurationOneWeek is negative, so expirationDate is "now minus one week".
+	// A volume created before that is older than the retention window.
+	expirationDate := time.Now().UTC().Add(clean.KeepDurationOneWeek)
+
 	input := &ec2.DescribeVolumesInput{
 		Filters: []types.Filter{
 			{
@@ -54,14 +61,17 @@ func deleteUnusedVolumes(ctx context.Context, client *ec2.Client) error {
 		return err
 	}
 	for _, volume := range volumes.Volumes {
-		if time.Since(*volume.CreateTime) > clean.KeepDurationOneWeek && len(volume.Attachments) == 0 {
+		if expirationDate.After(*volume.CreateTime) && len(volume.Attachments) == 0 {
+			if clean.Skip("delete unused volume %s", *volume.VolumeId) {
+				continue
+			}
 			log.Printf("Deleting unused volume %s", *volume.VolumeId)
 			_, err = client.DeleteVolume(ctx, &ec2.DeleteVolumeInput{
 				VolumeId: volume.VolumeId,
 			})
-		}
-		if err != nil {
-			log.Printf("Error deleting volume %s: %v", *volume.VolumeId, err)
+			if err != nil {
+				log.Printf("Error deleting volume %s: %v", *volume.VolumeId, err)
+			}
 		}
 	}
 	return nil

tool/clean/clean_ecs/clean_ecs.go

@@ -8,8 +8,8 @@ package main
 
 import (
 	"context"
+	"flag"
 	"log"
-	"os"
 	"strings"
 	"time"
 
@@ -23,15 +23,23 @@ import (
 
 // Clean ECS clusters if they have been running longer than 7 days
 
-var expirationTimeOneWeek = time.Now().UTC().Add(-clean.KeepDurationOneWeek)
+// KeepDurationOneWeek is negative, so this is "now minus one week"; a task that
+// started after this cutoff is younger than a week and keeps its cluster alive.
+var expirationTimeOneWeek = time.Now().UTC().Add(clean.KeepDurationOneWeek)
 
 const clusterPrefix = "cwagent-integ-test-cluster-"
 
 var taskdefPrefixes = []string{"cwagent-integ-test-", "extra-apps-family-", "cwagent-task-family-"}
 
 func main() {
+	clean.RegisterCommonFlags()
+	flag.Parse()
+	region := flag.Arg(0)
+	if region == "" {
+		log.Fatal("Region argument is required, e.g. clean_ecs.go <region>")
+	}
 	ctx := context.Background()
-	defaultConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(os.Args[1]))
+	defaultConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(region))
 	if err != nil {
 		log.Fatalf("Error loading AWS config for ECS cleanup: %v", err)
 	}
@@ -98,6 +106,9 @@ func terminateClusters(ctx context.Context, client *ecs.Client) {
 	// Deletion Logic
 	for _, clusterId := range clusterIds {
 		log.Printf("Cluster to terminate: %s", *clusterId)
+		if clean.Skip("delete ECS cluster %s (scale down and delete its services, deregister container instances)", *clusterId) {
+			continue
+		}
 
 		// Delete cluster services
 		serviceInput := ecs.ListServicesInput{Cluster: clusterId}
@@ -190,6 +201,10 @@ func deleteInactiveTaskDefinitions(ctx context.Context, client *ecs.Client) {
 
 	log.Printf("Found %d inactive task definitions to delete", len(taskDefsToDelete))
 
+	if clean.Skip("delete %d inactive task definitions", len(taskDefsToDelete)) {
+		return
+	}
+
 	// Batch delete task definitions (API supports up to 10 at a time)
 	const batchSize = 10
 	totalDeleted := 0