fix: GCP dynamic zone discovery; guaranteed CI cleanup on failure

GCP: replace hardcoded zone with dynamic discovery at plan time using
data "google_compute_zones" so the cluster is never deployed into an
exhausted or unavailable zone.

CI workflows: add Taskfile defer to all four workflows so terraform
destroy is guaranteed to run on exit regardless of which step fails,
preventing resource accumulation on cancellation or mid-run failures.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: PrzemekZglinicki

.tasks/ci.yml

@@ -15,6 +15,8 @@ tasks:
   aws:rp:
     desc: CI workflow - AWS basic Redpanda
     cmds:
+      - defer:
+          task: :infra:aws:destroy
       - task: :tools:keygen
       - task: :infra:aws:build
       - task: :cluster:provision
@@ -30,6 +32,10 @@ tasks:
       ENABLE_CONNECT: "true"
       DISTRO: "43"
     cmds:
+      - defer:
+          task: :infra:aws:destroy
+      - defer:
+          task: :infra:extra:aws:destroy
       - task: :tools:keygen
       - task: :infra:aws:build
         vars: {ENABLE_CONNECT: "true", DISTRO: "43"}
@@ -51,6 +57,8 @@ tasks:
     vars:
       TIERED_STORAGE_ENABLED: "true"
     cmds:
+      - defer:
+          task: :infra:aws:destroy
       - task: :tools:keygen
       - task: :infra:aws:build
         vars: {TIERED_STORAGE_ENABLED: "true"}
@@ -69,6 +77,10 @@ tasks:
       TIERED_STORAGE_ENABLED: "true"
       DISTRO: "43"
     cmds:
+      - defer:
+          task: :infra:aws:destroy
+      - defer:
+          task: :infra:extra:aws:destroy
       - task: :tools:keygen
       - task: :infra:aws:build
         vars: {ENABLE_CONNECT: "true", TIERED_STORAGE_ENABLED: "true", DISTRO: "43"}
@@ -92,6 +104,8 @@ tasks:
   gcp:rp:
     desc: CI workflow - GCP basic Redpanda
     cmds:
+      - defer:
+          task: :infra:gcp:destroy
       - task: :tools:keygen
       - task: :infra:gcp:build
       - task: :cluster:provision
@@ -106,6 +120,8 @@ tasks:
     vars:
       TIERED_STORAGE_ENABLED: "true"
     cmds:
+      - defer:
+          task: :infra:gcp:destroy
       - task: :tools:keygen
       - task: :infra:gcp:build
         vars: {TIERED_STORAGE_ENABLED: "true"}

aws/main.tf

@@ -1,6 +1,5 @@
 ## we assume a default vpc. if you have one you want to use you will need to provide a vpc and subnet ID
 
-
 module "redpanda-cluster" {
   source                   = "redpanda-data/redpanda-cluster/aws"
   public_key_path          = var.public_key_path
@@ -21,9 +20,9 @@ module "redpanda-cluster" {
   client_instance_type     = var.client_instance_type
   prometheus_instance_type = var.prometheus_instance_type
   machine_architecture     = var.machine_architecture
-  connect_count = var.connect_count
-  connect_instance_type = var.broker_instance_type
-  enable_connect = var.enable_connect
+  connect_count            = var.connect_count
+  connect_instance_type    = var.broker_instance_type
+  enable_connect           = var.enable_connect
 }
 
 variable "broker_instance_type" {

gcp/main.tf

@@ -35,7 +35,7 @@ module "redpanda-cluster" {
   ssh_user              = var.ssh_user
   subnet                = coalesce(var.subnet, google_compute_subnetwork.test-subnet.id)
   image                 = var.image
-  availability_zone     = var.availability_zone
+  availability_zone     = local.availability_zones
   broker_count          = var.broker_count
   client_count          = var.client_count
   disks                 = var.disks
@@ -58,6 +58,18 @@ provider "google" {
   credentials = base64decode(var.gcp_creds)
 }
 
+data "google_compute_zones" "available" {
+  region = var.region
+  status = "UP"
+}
+
+locals {
+  availability_zones = length(var.availability_zone) > 0 ? var.availability_zone : [
+    for z in data.google_compute_zones.available.names :
+    trimprefix(z, "${var.region}-")
+  ]
+}
+
 variable "gcp_creds" {
   default     = ""
   type        = string
@@ -69,8 +81,8 @@ variable "region" {
 }
 
 variable "availability_zone" {
-  description = "The zone where the cluster will be deployed [a,b,...]"
-  default     = ["a"]
+  description = "The zone where the cluster will be deployed [a,b,...]. Leave empty to auto-discover available zones."
+  default     = []
   type        = list(string)
 }