vault: retry allowlist auth on sync lag

Author: prashantkumar1982

core/capabilities/vault/allow_list_based_auth.go

@@ -14,6 +14,11 @@ import (
 	workflowsyncerv2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/v2"
 )
 
+const (
+	allowListBasedAuthRetryCount    = 3
+	allowListBasedAuthRetryInterval = 3 * time.Second
+)
+
 // AllowListBasedAuth validates Vault requests against the workflow registry's onchain allowlist.
 type AllowListBasedAuth interface {
 	AuthorizeRequest(ctx context.Context, req jsonrpc.Request[json.RawMessage]) (*AuthResult, error)
@@ -22,6 +27,8 @@ type AllowListBasedAuth interface {
 type allowListBasedAuth struct {
 	workflowRegistrySyncer workflowsyncerv2.WorkflowRegistrySyncer
 	lggr                   logger.Logger
+	retryCount             int
+	retryInterval          time.Duration
 }
 
 // AuthorizeRequest authorizes a request using AllowListBasedAuth.
@@ -43,14 +50,10 @@ func (r *allowListBasedAuth) AuthorizeRequest(ctx context.Context, req jsonrpc.R
 		r.lggr.Errorw("AllowListBasedAuth workflowRegistrySyncer is nil", "method", req.Method, "requestID", req.ID)
 		return nil, errors.New("internal error: workflowRegistrySyncer is nil")
 	}
-	allowedRequests := r.workflowRegistrySyncer.GetAllowlistedRequests(ctx)
-	allowedRequestsStrs := make([]string, 0, len(allowedRequests))
-	for _, rr := range allowedRequests {
-		allowedReqStr := fmt.Sprintf("AuthorizedOwner: %s, RequestDigest: %s, ExpiryTimestamp: %d", rr.Owner.Hex(), hex.EncodeToString(rr.RequestDigest[:]), rr.ExpiryTimestamp)
-		allowedRequestsStrs = append(allowedRequestsStrs, allowedReqStr)
+	allowlistedRequest, allowedRequestsStrs, err := r.findAllowlistedItemWithRetry(ctx, req, requestDigest, requestDigestBytes32)
+	if err != nil {
+		return nil, err
 	}
-	r.lggr.Debugw("AllowListBasedAuth loaded allowlisted requests", "method", req.Method, "requestID", req.ID, "allowedRequests", allowedRequestsStrs)
-	allowlistedRequest := r.fetchAllowlistedItem(allowedRequests, requestDigestBytes32)
 	if allowlistedRequest == nil {
 		r.lggr.Debugw("AllowListBasedAuth request digest not allowlisted",
 			"method", req.Method,
@@ -75,6 +78,40 @@ func (r *allowListBasedAuth) AuthorizeRequest(ctx context.Context, req jsonrpc.R
 	}, nil
 }
 
+func (r *allowListBasedAuth) findAllowlistedItemWithRetry(ctx context.Context, req jsonrpc.Request[json.RawMessage], requestDigest string, requestDigestBytes32 [32]byte) (*workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest, []string, error) {
+	for attempt := 0; attempt <= r.retryCount; attempt++ {
+		allowedRequests := r.workflowRegistrySyncer.GetAllowlistedRequests(ctx)
+		allowedRequestsStrs := make([]string, 0, len(allowedRequests))
+		for _, rr := range allowedRequests {
+			allowedReqStr := fmt.Sprintf("AuthorizedOwner: %s, RequestDigest: %s, ExpiryTimestamp: %d", rr.Owner.Hex(), hex.EncodeToString(rr.RequestDigest[:]), rr.ExpiryTimestamp)
+			allowedRequestsStrs = append(allowedRequestsStrs, allowedReqStr)
+		}
+		r.lggr.Debugw("AllowListBasedAuth loaded allowlisted requests", "method", req.Method, "requestID", req.ID, "attempt", attempt+1, "allowedRequests", allowedRequestsStrs)
+
+		allowlistedRequest := r.fetchAllowlistedItem(allowedRequests, requestDigestBytes32)
+		if allowlistedRequest != nil {
+			return allowlistedRequest, allowedRequestsStrs, nil
+		}
+		if attempt == r.retryCount {
+			return nil, allowedRequestsStrs, nil
+		}
+
+		r.lggr.Debugw("AllowListBasedAuth request digest not yet allowlisted, retrying",
+			"method", req.Method,
+			"requestID", req.ID,
+			"digestHexStr", requestDigest,
+			"attempt", attempt+1,
+			"maxAttempts", r.retryCount+1,
+			"retryInterval", r.retryInterval)
+		if err := sleepWithContext(ctx, r.retryInterval); err != nil {
+			r.lggr.Debugw("AllowListBasedAuth retry canceled", "method", req.Method, "requestID", req.ID, "error", err)
+			return nil, nil, err
+		}
+	}
+
+	return nil, nil, nil
+}
+
 func (r *allowListBasedAuth) fetchAllowlistedItem(allowListedRequests []workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest, digest [32]byte) *workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest {
 	for _, item := range allowListedRequests {
 		if item.RequestDigest == digest {
@@ -89,5 +126,19 @@ func NewAllowListBasedAuth(lggr logger.Logger, workflowRegistrySyncer workflowsy
 	return &allowListBasedAuth{
 		workflowRegistrySyncer: workflowRegistrySyncer,
 		lggr:                   logger.Named(lggr, "VaultAllowListBasedAuth"),
+		retryCount:             allowListBasedAuthRetryCount,
+		retryInterval:          allowListBasedAuthRetryInterval,
+	}
+}
+
+func sleepWithContext(ctx context.Context, d time.Duration) error {
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
 	}
 }

core/capabilities/vault/allow_list_based_auth_test.go

@@ -159,6 +159,8 @@ func testAuthForRequests(t *testing.T, allowlistedRequest, notAllowlistedRequest
 
 	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
 	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth.retryCount = 0
+	auth.retryInterval = time.Millisecond
 
 	// Happy path
 	digest, err := allowlistedRequest.Digest()
@@ -203,3 +205,45 @@ func testAuthForRequests(t *testing.T, allowlistedRequest, notAllowlistedRequest
 	require.Nil(t, authResult)
 	require.ErrorContains(t, err, "not allowlisted")
 }
+
+func TestAllowListBasedAuth_RetriesUntilRequestIsAllowlisted(t *testing.T) {
+	lggr := logger.TestLogger(t)
+	owner := common.Address{1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+	params, err := json.Marshal(vaultcommon.ListSecretIdentifiersRequest{
+		Namespace: "b",
+	})
+	require.NoError(t, err)
+
+	req := jsonrpc.Request[json.RawMessage]{
+		ID:     "123",
+		Method: vaulttypes.MethodSecretsList,
+		Params: (*json.RawMessage)(&params),
+	}
+
+	digest, err := req.Digest()
+	require.NoError(t, err)
+	digestBytes, err := hex.DecodeString(digest)
+	require.NoError(t, err)
+	expiry := time.Now().UTC().Unix() + 100
+	allowlisted := []workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest{
+		{
+			RequestDigest:   [32]byte(digestBytes),
+			Owner:           owner,
+			ExpiryTimestamp: uint32(expiry), //nolint:gosec // it is a safe conversion
+		},
+	}
+
+	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
+	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth.retryCount = 1
+	auth.retryInterval = time.Millisecond
+
+	mockSyncer.On("GetAllowlistedRequests", mock.Anything).Return([]workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest{}).Once()
+	mockSyncer.On("GetAllowlistedRequests", mock.Anything).Return(allowlisted).Once()
+
+	authResult, err := auth.AuthorizeRequest(t.Context(), req)
+	require.NoError(t, err)
+	require.Equal(t, owner.Hex(), authResult.AuthorizedOwner())
+	require.Equal(t, expiry, authResult.ExpiresAt)
+}