OCPBUGS-29479: Add retries to async cache initialization (#13610)

* OCPBUGS-29479: Add retries to async cache initialization

Auth initialization can fail if the API server not ready yet. This is especially common during cluster install.

* Don't log errors on async cache retries.

* Use context.WithTimeout and wait.UntilWithContext instead of a for loop to retry auth async cache setup

* Use `wait.PollUntilContextTimeout` for async cache initialization

* Address review comments
- Log error on each retry
- Do not return error from PollUntilContextTimeout condition func
- Reduce retry interval to 10 seconds to match historical behavior

Author: TheRealJon

pkg/auth/asynccache.go

@@ -10,6 +10,12 @@ import (
 	"k8s.io/klog"
 )
 
+const (
+	initializationRetryInterval = 10 * time.Second
+	initializationTimeout       = 5 * time.Minute
+	initializeImmediately       = true
+)
+
 type cachingFuncType[T any] func(ctx context.Context) (T, error)
 
 type AsyncCache[T any] struct {
@@ -27,13 +33,26 @@ func NewAsyncCache[T any](ctx context.Context, reloadPeriod time.Duration, cachi
 		cachingFunc:  cachingFunc,
 	}
 
-	item, err := cachingFunc(ctx)
+	var err error
+	wait.PollUntilContextTimeout(
+		ctx,
+		initializationRetryInterval,
+		initializationTimeout,
+		initializeImmediately,
+		func(ctx context.Context) (bool, error) {
+			item, err := cachingFunc(ctx)
+			if err != nil {
+				klog.V(4).Infof("failed to setup an async cache (retrying in %v) - caching func returned error: %v", initializationRetryInterval, err)
+				return false, nil
+			}
+			c.cachedItem = item
+			return true, nil
+		},
+	)
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to setup an async cache - caching func returned error: %w", err)
 	}
-
-	c.cachedItem = item
-
 	return c, nil
 }