Merge pull request #31080 from dgoodwin/admin-kubeconfig-retry

OCPBUGS-84504: retry transient kubeconfig read failures in GetClientConfig

Author: openshift-merge-bot[bot]

test/extended/util/client.go

@@ -1272,19 +1272,36 @@ func WaitForAccess(c kubernetes.Interface, allowed bool, review *kubeauthorizati
 }
 
 func GetClientConfig(kubeConfigFile string) (*rest.Config, error) {
-	kubeConfigBytes, err := ioutil.ReadFile(kubeConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	kubeConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfigBytes)
-	if err != nil {
-		return nil, err
-	}
-	clientConfig, err := kubeConfig.ClientConfig()
+	var clientConfig *rest.Config
+	var lastErr error
+	err := wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (bool, error) {
+		kubeConfigBytes, err := os.ReadFile(kubeConfigFile)
+		if err != nil {
+			lastErr = err
+			if os.IsNotExist(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		if len(kubeConfigBytes) == 0 {
+			lastErr = fmt.Errorf("kubeconfig file %q is empty", kubeConfigFile)
+			return false, nil
+		}
+		kubeConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfigBytes)
+		if err != nil {
+			lastErr = err
+			return false, nil
+		}
+		clientConfig, err = kubeConfig.ClientConfig()
+		if err != nil {
+			lastErr = err
+			return false, nil
+		}
+		return true, nil
+	})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to read kubeconfig %q after retries: %w", kubeConfigFile, lastErr)
 	}
-
 	return clientConfig, nil
 }
 

test/extended/util/client_test.go

@@ -0,0 +1,112 @@
+package util
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const validKubeconfig = `apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: https://localhost:6443
+  name: test-cluster
+contexts:
+- context:
+    cluster: test-cluster
+    user: test-user
+  name: test-context
+current-context: test-context
+users:
+- name: test-user
+  user:
+    token: test-token
+`
+
+func TestGetClientConfigRetryOnMissingFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "kubeconfig")
+
+	if err := os.WriteFile(path, []byte(validKubeconfig), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		time.Sleep(200 * time.Millisecond)
+		os.Remove(path)
+		time.Sleep(800 * time.Millisecond)
+		os.WriteFile(path, []byte(validKubeconfig), 0644)
+	}()
+
+	time.Sleep(300 * time.Millisecond)
+
+	cfg, err := GetClientConfig(path)
+	if err != nil {
+		t.Fatalf("expected retry to succeed, got error: %v", err)
+	}
+	if cfg.Host != "https://localhost:6443" {
+		t.Fatalf("unexpected host: %s", cfg.Host)
+	}
+}
+
+func TestGetClientConfigRetryOnEmptyFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "kubeconfig")
+
+	if err := os.WriteFile(path, []byte{}, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		os.WriteFile(path, []byte(validKubeconfig), 0644)
+	}()
+
+	cfg, err := GetClientConfig(path)
+	if err != nil {
+		t.Fatalf("expected retry to succeed, got error: %v", err)
+	}
+	if cfg.Host != "https://localhost:6443" {
+		t.Fatalf("unexpected host: %s", cfg.Host)
+	}
+}
+
+func TestGetClientConfigFailsOnPermanentMissing(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "nonexistent", "kubeconfig")
+
+	start := time.Now()
+	_, err := GetClientConfig(path)
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("expected error for permanently missing file")
+	}
+	if elapsed > 15*time.Second {
+		t.Fatalf("took too long: %v", elapsed)
+	}
+}
+
+func TestGetClientConfigSucceedsImmediately(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "kubeconfig")
+
+	if err := os.WriteFile(path, []byte(validKubeconfig), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	start := time.Now()
+	cfg, err := GetClientConfig(path)
+	elapsed := time.Since(start)
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if cfg.Host != "https://localhost:6443" {
+		t.Fatalf("unexpected host: %s", cfg.Host)
+	}
+	if elapsed > 1*time.Second {
+		t.Fatalf("should have succeeded immediately, took %v", elapsed)
+	}
+}