🌱 Do not reconcile on irrecoverable errors (#1887)

do not reconcile on irrecoverable errors

Do not reconcile or remidiate on irrecoverable errors like
`invalid_input` and `resource_unavailable` as reconciling again won't
change the result for these errors and it will rather stuck in a loop.

Signed-off-by: Dhairya Arora <dhairya.arora@syself.com>

Author: Dhairya-Arora01

api/v1beta1/conditions_const.go

@@ -48,6 +48,8 @@ const (
 	ServerTypeNotFoundReason = "ServerTypeNotFound"
 	// ServerCreateFailedReason indicates that server could not get created.
 	ServerCreateFailedReason = "ServerCreateFailedReason"
+	// ServerCreateFailedIrrecoverableErrorReason indicates that server creation failed with an irrecoverable error.
+	ServerCreateFailedIrrecoverableErrorReason = "ServerCreateFailedIrrecoverableError"
 )
 
 const (

controllers/hcloudremediation_controller.go

@@ -109,6 +109,19 @@ func (r *HCloudRemediationReconciler) Reconcile(ctx context.Context, req reconci
 
 	log = log.WithValues("HCloudMachine", klog.KObj(hcloudMachine))
 
+	// Skip remediation for machines that failed to create with irrecoverable errors (e.g. invalid_input, resource_unavailable).
+	// These errors cannot be fixed by rebooting or replacing the machine.
+	// We return without error so the MHC does not keep retrying remediation.
+	if conditions.IsFalse(hcloudMachine, infrav1.ServerCreateSucceededCondition) &&
+		conditions.GetReason(hcloudMachine, infrav1.ServerCreateSucceededCondition) == infrav1.ServerCreateFailedIrrecoverableErrorReason {
+		log.Info("Skipping remediation for machine with irrecoverable creation failure",
+			"reason", conditions.GetMessage(hcloudMachine, infrav1.ServerCreateSucceededCondition),
+		)
+
+		// signal remediation done.
+		return reconcile.Result{}, nil
+	}
+
 	// Fetch the Cluster.
 	cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta)
 	if err != nil {

pkg/services/hcloud/server/server.go

@@ -142,6 +142,21 @@ func (s *Service) Reconcile(ctx context.Context) (res reconcile.Result, err erro
 		// otherwise create server.
 		server, err = s.createServer(ctx)
 		if err != nil {
+			// Terminal errors like invalid_input (e.g. unsupported location for server type)
+			// or resource_unavailable (e.g. server location disabled) will never succeed on retry.
+			// Mark the machine as irrecoverably failed and stop reconciling.
+			if hcloud.IsError(err, hcloud.ErrorCodeInvalidInput) || hcloud.IsError(err, hcloud.ErrorCodeResourceUnavailable) {
+				conditions.MarkFalse(
+					s.scope.HCloudMachine,
+					infrav1.ServerCreateSucceededCondition,
+					infrav1.ServerCreateFailedIrrecoverableErrorReason,
+					clusterv1.ConditionSeverityError,
+					"%s",
+					err.Error(),
+				)
+				return reconcile.Result{}, nil
+			}
+
 			if errors.Is(err, errServerCreateNotPossible) {
 				return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 			}