Merge pull request #33 from alicefr/fix-31

cluster start: wait for HAProxy to be healthy before returning

Author: alicefr

.github/workflows/integration-tests.yml

@@ -135,7 +135,7 @@ jobs:
           key: podman-images-v2-integration-${{ steps.digests.outputs.hash }}
 
       - name: Run integration tests
-        run: sudo make test-integration TEST_PROCS=3
+        run: sudo make test-integration TEST_PROCS=2
         timeout-minutes: 90
         env:
           CONTAINER_HOST: unix:///run/podman/podman.sock

internal/cli/cluster/start.go

@@ -183,6 +183,9 @@ func runStart(ctx context.Context, logger *logrus.Logger, nodeName string, nodeI
 	if err := haproxyMgr.EnsureHAProxy(ctx, 0); err != nil {
 		return fmt.Errorf("creating HAProxy load balancer: %w", err)
 	}
+	if err := haproxyMgr.WaitForHealthy(ctx); err != nil {
+		return fmt.Errorf("waiting for HAProxy: %w", err)
+	}
 	logger.Info("")
 
 	logger.Info("✅ Cluster created successfully!")

internal/cluster/join.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bootc-dev/bink/internal/config"
 	"github.com/bootc-dev/bink/internal/ssh"
 )
 
@@ -101,17 +102,24 @@ func (c *Cluster) Join(ctx context.Context, opts JoinOptions) error {
 		c.logger.Info("")
 		c.logger.Infof("=== Labeling %s ===", nodeName)
 
-		containerName := fmt.Sprintf("k8s-%s-%s", c.name, controlPlane)
-		kubeClient, err := c.newKubeClient(ctx, cpSSHClient, containerName)
+		haproxyContainer := fmt.Sprintf("%s%s-%s", config.ContainerNamePrefix, c.name, config.HAProxyContainerName)
+		kubeClient, err := c.newKubeClient(ctx, cpSSHClient, haproxyContainer)
 		if err != nil {
-			c.logger.Warnf("Failed to create kubernetes client (non-fatal): %v", err)
-		} else {
-			if err := kubeClient.LabelNode(ctx, nodeName, labels); err != nil {
-				c.logger.Warnf("Failed to label node (non-fatal): %v", err)
-			} else {
-				c.logger.Infof("✅ Node %s labeled", nodeName)
+			return fmt.Errorf("labeling node %s: creating kubernetes client: %w", nodeName, err)
+		}
+		var labelErr error
+		for attempt := 1; attempt <= 5; attempt++ {
+			labelErr = kubeClient.LabelNode(ctx, nodeName, labels)
+			if labelErr == nil {
+				break
 			}
+			c.logger.Warnf("Failed to label node (attempt %d/5): %v", attempt, labelErr)
+			time.Sleep(5 * time.Second)
+		}
+		if labelErr != nil {
+			return fmt.Errorf("labeling node %s: %w", nodeName, labelErr)
 		}
+		c.logger.Infof("✅ Node %s labeled", nodeName)
 	}
 
 	c.logger.Info("")

internal/haproxy/haproxy.go

@@ -5,9 +5,12 @@ package haproxy
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/bootc-dev/bink/internal/config"
 	"github.com/bootc-dev/bink/internal/podman"
@@ -202,6 +205,56 @@ func (m *Manager) GetPublishedPort(ctx context.Context) (int, error) {
 	return m.podman.GetPublishedPort(ctx, m.containerName(), fmt.Sprintf("%d/tcp", config.HAProxyPort))
 }
 
+// WaitForHealthy polls the Kubernetes API /healthz endpoint through HAProxy
+// until it returns HTTP 200, indicating that HAProxy has healthy backends.
+func (m *Manager) WaitForHealthy(ctx context.Context) error {
+	port, err := m.GetPublishedPort(ctx)
+	if err != nil {
+		return fmt.Errorf("getting HAProxy published port: %w", err)
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	healthURL := fmt.Sprintf("https://localhost:%d/healthz", port)
+	timer := time.NewTimer(2 * time.Minute)
+	defer timer.Stop()
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	logrus.Info("Waiting for HAProxy to become healthy...")
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+			return fmt.Errorf("timed out waiting for HAProxy to become healthy on port %d", port)
+		case <-ticker.C:
+			reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, healthURL, nil)
+			if err != nil {
+				cancel()
+				continue
+			}
+			resp, err := client.Do(req)
+			if err != nil {
+				cancel()
+				continue
+			}
+			resp.Body.Close()
+			cancel()
+			if resp.StatusCode == http.StatusOK {
+				logrus.Info("HAProxy is healthy")
+				return nil
+			}
+		}
+	}
+}
+
 // discoverBackends finds all control-plane node containers in this cluster
 // and returns their bridge IPs as backends.
 func (m *Manager) discoverBackends(ctx context.Context) ([]backend, error) {