phase/uninstall_mke: fall back to forced swarm dissolution on timeout

The uninstall-ucp bootstrapper deploys ucp-uninstall-agent as a global
Swarm service, then waits (~2 min hardcoded) for every node to report
back. On large or mixed-OS clusters with cold image caches this deadline
is missed, causing Reset() to fail even though the infrastructure will
be torn down by terraform destroy anyway.

Observed in CI:
  smoke-modern (MKE 3.9.2, 7 nodes): all nodes missed the deadline
  smoke-windows (MKE 3.8.8, Win2025): Win2025 node missed the deadline

MKE itself documents the recovery path when this happens:
  1. Remove the stuck ucp-uninstall-agent service.
  2. Force every node to leave the swarm.

Implement that as an automatic fallback inside UninstallMKE.Run():
- isUninstallTimeout() detects the specific 'Uninstalling UCP took too
  long' fatal line that Bootstrap surfaces from the MKE container.
- dissolveSwarm() removes the stuck service (best-effort), then forces
  all non-leader nodes to leave in parallel, then forces the leader to
  leave last. Per-node failures are logged as warnings so that a single
  unresponsive host does not block the rest.

Other uninstall-ucp errors (connection failures, image pull errors, etc.)
are still returned as hard failures unchanged.

Signed-off-by: James Nesbitt <jnesbitt@mirantis.com>

Author: james-nesbitt

pkg/mcr/mcr.go

@@ -16,18 +16,22 @@ var (
 )
 
 // DrainNode drains a node from the workload via docker drain command.
+// If the node is not part of a swarm (empty NodeID) the call is a no-op;
+// this is the expected state after a forced swarm dissolution.
 func DrainNode(lead *mkeconfig.Host, h *mkeconfig.Host) error {
 	nodeID, err := swarm.NodeID(h)
 	if err != nil {
 		return fmt.Errorf("failed to get node ID for %s: %w", h, err)
 	}
 
-	drainCmd := lead.Configurer.DockerCommandf("node update --availability drain %s", nodeID)
-	if err := lead.Exec(drainCmd); err != nil {
-		return fmt.Errorf("%s: failed to run MKE uninstaller: %w", lead, err)
+	if nodeID == "" {
+		log.Debugf("%s: not part of a swarm, skipping drain", h)
+		return nil
 	}
+
+	drainCmd := lead.Configurer.DockerCommandf("node update --availability drain %s", nodeID)
 	if err := lead.Exec(drainCmd); err != nil {
-		return fmt.Errorf("failed to drain node %s: %w", nodeID, err)
+		return fmt.Errorf("%s: failed to drain node %s: %w", lead, nodeID, err)
 	}
 
 	log.Infof("%s: node %s drained", lead, nodeID)

pkg/product/mke/phase/uninstall_mke.go

@@ -2,6 +2,7 @@ package phase
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/Mirantis/launchpad/pkg/mke"
 	"github.com/Mirantis/launchpad/pkg/phase"
@@ -34,15 +35,30 @@ func (p *UninstallMKE) Run() error {
 	uninstallFlags := commonconfig.Flags{"--id", swarm.ClusterID(leader), "--purge-config"}
 
 	if _, err := mke.Bootstrap("uninstall-ucp", *p.Config, mke.BootstrapOptions{OperationFlags: uninstallFlags, ExecOptions: []exec.Option{exec.StreamOutput()}}); err != nil {
-		return fmt.Errorf("%s: failed to run MKE uninstaller: %w", leader, err)
+		// The uninstall-ucp bootstrapper deploys ucp-uninstall-agent as a global
+		// Swarm service and waits (hardcoded ~2 minutes) for every node to report
+		// back. On large clusters or hosts with cold image caches this deadline is
+		// missed. When that happens, MKE itself recommends:
+		//   1. Remove the stuck ucp-uninstall-agent service.
+		//   2. Force every node to leave the swarm.
+		// We implement that as an automatic fallback so that reset can continue
+		// to MCR uninstall without leaving a broken cluster behind.
+		if isUninstallTimeout(err) {
+			log.Warnf("%s: uninstall-ucp timed out waiting for nodes; falling back to forced swarm dissolution", leader)
+			if dissolveErr := dissolveSwarm(leader, p.Config.Spec.Hosts); dissolveErr != nil {
+				return fmt.Errorf("%s: uninstall-ucp timed out and forced swarm dissolution failed: %w (original: %w)", leader, dissolveErr, err)
+			}
+			log.Infof("%s: swarm dissolved; continuing with MCR uninstall", leader)
+		} else {
+			return fmt.Errorf("%s: failed to run MKE uninstaller: %w", leader, err)
+		}
 	}
 
 	managers := p.Config.Spec.Managers()
 	_ = managers.ParallelEach(func(h *mkeconfig.Host) error {
 		log.Infof("%s: removing ucp-controller-server-certs volume", h)
-		err := h.Exec(h.Configurer.DockerCommandf("volume rm --force ucp-controller-server-certs"))
-		if err != nil {
-			log.Errorf("%s: failed to remove the volume", h)
+		if err := h.Exec(h.Configurer.DockerCommandf("volume rm --force ucp-controller-server-certs")); err != nil {
+			log.Errorf("%s: failed to remove the volume: %v", h, err)
 		}
 
 		if err := h.Reboot(); err != nil {
@@ -61,3 +77,50 @@ func (p *UninstallMKE) Run() error {
 
 	return nil
 }
+
+// isUninstallTimeout returns true when the error from Bootstrap is the
+// well-known "took too long" timeout from the uninstall-ucp bootstrapper.
+// MKE emits this as a fatal log line when its per-node acknowledgement
+// deadline expires; Bootstrap surfaces it verbatim in the returned error.
+func isUninstallTimeout(err error) bool {
+	return strings.Contains(err.Error(), "Uninstalling UCP took too long")
+}
+
+// dissolveSwarm forcibly tears down the Swarm cluster when uninstall-ucp
+// cannot do so cleanly. It follows the recovery steps documented by MKE:
+//
+//  1. Remove the stuck ucp-uninstall-agent / ucp-uninstall-agent-win services
+//     from the swarm leader (best-effort; they may already be gone).
+//  2. Force all non-leader nodes to leave the swarm in parallel.
+//  3. Force the leader to leave last.
+//
+// Errors from individual nodes are logged as warnings so that a single
+// unresponsive host does not prevent the rest of the cluster from being torn
+// down. Only the leader's final leave is treated as a hard failure.
+func dissolveSwarm(leader *mkeconfig.Host, hosts mkeconfig.Hosts) error {
+	// Step 1: remove the stuck uninstall-agent services (best-effort).
+	for _, svc := range []string{"ucp-uninstall-agent", "ucp-uninstall-agent-win"} {
+		log.Infof("%s: removing stuck service %s", leader, svc)
+		if err := leader.Exec(leader.Configurer.DockerCommandf("service rm %s", svc)); err != nil {
+			log.Debugf("%s: service rm %s: %v (may already be removed)", leader, svc, err)
+		}
+	}
+
+	// Step 2: force all non-leader nodes to leave the swarm.
+	nonLeaders := hosts.Filter(func(h *mkeconfig.Host) bool { return h != leader })
+	_ = nonLeaders.ParallelEach(func(h *mkeconfig.Host) error {
+		log.Infof("%s: force-leaving swarm", h)
+		if err := h.Exec(h.Configurer.DockerCommandf("swarm leave --force")); err != nil {
+			log.Warnf("%s: swarm leave --force failed: %v", h, err)
+		}
+		return nil // continue regardless; errors are warnings only
+	})
+
+	// Step 3: leader leaves last so it can still reach the other nodes above.
+	log.Infof("%s: force-leaving swarm (leader)", leader)
+	if err := leader.Exec(leader.Configurer.DockerCommandf("swarm leave --force")); err != nil {
+		return fmt.Errorf("swarm leader failed to leave: %w", err)
+	}
+
+	return nil
+}

pkg/product/mke/phase/uninstall_mke_test.go

@@ -0,0 +1,29 @@
+package phase
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestIsUninstallTimeout(t *testing.T) {
+	t.Run("matches MKE timeout error", func(t *testing.T) {
+		err := errors.New("mke bootstrap uninstall-ucp failure; MKE uninstall-ucp: Uninstalling UCP took too long!")
+		if !isUninstallTimeout(err) {
+			t.Errorf("expected isUninstallTimeout=true for MKE timeout error, got false")
+		}
+	})
+
+	t.Run("does not match unrelated error", func(t *testing.T) {
+		err := errors.New("mke bootstrap uninstall-ucp failure; MKE uninstall-ucp: unable to cleanly uninstall UCP")
+		if isUninstallTimeout(err) {
+			t.Errorf("expected isUninstallTimeout=false for generic uninstall error, got true")
+		}
+	})
+
+	t.Run("does not match connection error", func(t *testing.T) {
+		err := errors.New("[ssh] 1.2.3.4:22: connection refused")
+		if isUninstallTimeout(err) {
+			t.Errorf("expected isUninstallTimeout=false for connection error, got true")
+		}
+	})
+}