Add FinalizedStateUnavailable to ClientErrors

Author: Krish-vemula

multinode/config/config.go

@@ -33,13 +33,10 @@ type MultiNode struct {
 	NoNewFinalizedHeadsThreshold *config.Duration
 	FinalityDepth                *uint32
 	FinalityTagEnabled           *bool
-	FinalizedBlockOffset         *uint32
+	FinalizedBlockOffset *uint32
 
 	// Finalized State Availability Check
-	FinalizedStateCheckEnabled          *bool
-	FinalizedStateCheckAddress          *string
 	FinalizedStateCheckFailureThreshold *uint32
-	FinalizedStateUnavailableRegex      *string
 }
 
 func (c *MultiNodeConfig) Enabled() bool {
@@ -100,31 +97,10 @@ func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.Finali
 
 func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset }
 
-func (c *MultiNodeConfig) FinalizedStateCheckEnabled() bool {
-	return c.MultiNode.FinalizedStateCheckEnabled != nil && *c.MultiNode.FinalizedStateCheckEnabled
-}
-
-func (c *MultiNodeConfig) FinalizedStateCheckAddress() string {
-	if c.MultiNode.FinalizedStateCheckAddress == nil {
-		return ""
-	}
-	return *c.MultiNode.FinalizedStateCheckAddress
-}
-
 func (c *MultiNodeConfig) FinalizedStateCheckFailureThreshold() uint32 {
-	if c.MultiNode.FinalizedStateCheckFailureThreshold == nil {
-		return 0
-	}
 	return *c.MultiNode.FinalizedStateCheckFailureThreshold
 }
 
-func (c *MultiNodeConfig) FinalizedStateUnavailableRegex() string {
-	if c.MultiNode.FinalizedStateUnavailableRegex == nil {
-		return ""
-	}
-	return *c.MultiNode.FinalizedStateUnavailableRegex
-}
-
 func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
 	if f.MultiNode.Enabled != nil {
 		c.MultiNode.Enabled = f.MultiNode.Enabled
@@ -183,16 +159,7 @@ func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
 	}
 
 	// Finalized State Availability Check
-	if f.MultiNode.FinalizedStateCheckEnabled != nil {
-		c.MultiNode.FinalizedStateCheckEnabled = f.MultiNode.FinalizedStateCheckEnabled
-	}
-	if f.MultiNode.FinalizedStateCheckAddress != nil {
-		c.MultiNode.FinalizedStateCheckAddress = f.MultiNode.FinalizedStateCheckAddress
-	}
 	if f.MultiNode.FinalizedStateCheckFailureThreshold != nil {
 		c.MultiNode.FinalizedStateCheckFailureThreshold = f.MultiNode.FinalizedStateCheckFailureThreshold
 	}
-	if f.MultiNode.FinalizedStateUnavailableRegex != nil {
-		c.MultiNode.FinalizedStateUnavailableRegex = f.MultiNode.FinalizedStateUnavailableRegex
-	}
 }

multinode/mock_node_metrics_test.go

@@ -27,10 +27,7 @@ type NodeConfig interface {
 	DeathDeclarationDelay() time.Duration
 	NewHeadsPollInterval() time.Duration
 	VerifyChainID() bool
-	FinalizedStateCheckEnabled() bool
 	FinalizedStateCheckFailureThreshold() uint32
-	FinalizedStateCheckAddress() string
-	FinalizedStateUnavailableRegex() string
 }
 
 type ChainConfig interface {
@@ -52,13 +49,15 @@ type nodeMetrics interface {
 	IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string)
 	IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string)
 	IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string)
+	IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx context.Context, nodeName string)
 	RecordNodeClientVersion(ctx context.Context, nodeName string, version string)
 	SetHighestSeenBlock(ctx context.Context, nodeName string, blockNumber int64)
 	SetHighestFinalizedBlock(ctx context.Context, nodeName string, blockNumber int64)
 	IncrementSeenBlocks(ctx context.Context, nodeName string)
 	IncrementPolls(ctx context.Context, nodeName string)
 	IncrementPollsFailed(ctx context.Context, nodeName string)
 	IncrementPollsSuccess(ctx context.Context, nodeName string)
+	IncrementFinalizedStateFailed(ctx context.Context, nodeName string)
 }
 
 type Node[
@@ -277,7 +276,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lgg
 		// The node is already closed, and any subsequent transition is invalid.
 		// To make spotting such transitions a bit easier, return the invalid node state.
 		return nodeStateLen
-	case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing:
+	case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
 	default:
 		panic(fmt.Sprintf("cannot verify node in state %v", st))
 	}

multinode/mock_rpc_client_test.go

@@ -186,7 +186,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) {
 		return
 	}
 	switch n.state {
-	case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing:
+	case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
 		n.state = nodeStateAlive
 	default:
 		panic(transitionFail(n.state, nodeStateAlive))
@@ -368,7 +368,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareFinalizedStateNotAvailable() {
 func (n *node[CHAIN_ID, HEAD, RPC]) transitionToFinalizedStateNotAvailable(fn func()) {
 	ctx, cancel := n.stopCh.NewCtx()
 	defer cancel()
-	n.metrics.IncrementNodeTransitionsToUnreachable(ctx, n.name)
+	n.metrics.IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx, n.name)
 	n.stateMu.Lock()
 	defer n.stateMu.Unlock()
 	if n.state == nodeStateClosed {

multinode/node.go

@@ -2,10 +2,10 @@ package multinode
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"math/big"
-	"regexp"
 	"time"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
@@ -104,10 +104,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() {
 	var pollFailures uint32
 
 	// Finalized state availability check config
-	finalizedStateCheckEnabled := n.nodePoolCfg.FinalizedStateCheckEnabled()
 	finalizedStateCheckFailureThreshold := n.nodePoolCfg.FinalizedStateCheckFailureThreshold()
-	finalizedStateCheckAddress := n.nodePoolCfg.FinalizedStateCheckAddress()
-	finalizedStateUnavailableRegex := n.nodePoolCfg.FinalizedStateUnavailableRegex()
 	var finalizedStateFailures uint32
 
 	for {
@@ -159,38 +156,37 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() {
 				return
 			}
 			// Separate finalized state availability check
-			if finalizedStateCheckEnabled {
-				stateCheckCtx, stateCheckCancel := context.WithTimeout(ctx, pollInterval)
-				stateErr := n.RPC().CheckFinalizedStateAvailability(stateCheckCtx, finalizedStateCheckAddress)
-				stateCheckCancel()
-				if stateErr != nil {
-					if isFinalizedStateUnavailableError(stateErr, finalizedStateUnavailableRegex) {
-						if finalizedStateFailures < math.MaxUint32 {
-							finalizedStateFailures++
-						}
-						lggr.Warnw("Finalized state not available", "err", stateErr, "failures", finalizedStateFailures, "threshold", finalizedStateCheckFailureThreshold)
-						if finalizedStateCheckFailureThreshold > 0 && finalizedStateFailures >= finalizedStateCheckFailureThreshold {
-							lggr.Errorw("RPC node cannot serve finalized state after consecutive failures", "failures", finalizedStateFailures)
-							if n.poolInfoProvider != nil {
-								if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 && !n.isLoadBalancedRPC {
-									lggr.Criticalf("RPC endpoint cannot serve finalized state; %s %s", msgCannotDisable, msgDegradedState)
-									continue
-								}
+			stateCheckCtx, stateCheckCancel := context.WithTimeout(ctx, pollInterval)
+			stateErr := n.RPC().CheckFinalizedStateAvailability(stateCheckCtx)
+			stateCheckCancel()
+			if stateErr != nil {
+				if errors.Is(stateErr, ErrFinalizedStateUnavailable) {
+					if finalizedStateFailures < math.MaxUint32 {
+						n.metrics.IncrementFinalizedStateFailed(ctx, n.name)
+						finalizedStateFailures++
+					}
+					lggr.Warnw("Finalized state not available", "err", stateErr, "failures", finalizedStateFailures, "threshold", finalizedStateCheckFailureThreshold)
+					if finalizedStateFailures >= finalizedStateCheckFailureThreshold {
+						lggr.Errorw("RPC node cannot serve finalized state after consecutive failures", "failures", finalizedStateFailures)
+						if n.poolInfoProvider != nil {
+							if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 && !n.isLoadBalancedRPC {
+								lggr.Criticalf("RPC endpoint cannot serve finalized state; %s %s", msgCannotDisable, msgDegradedState)
+								continue
 							}
-							n.declareFinalizedStateNotAvailable()
-							return
 						}
-					} else {
-						// Treat as RPC reachability error
-						if pollFailures < math.MaxUint32 {
-							n.metrics.IncrementPollsFailed(ctx, n.name)
-							pollFailures++
-						}
-						lggr.Warnw("Finalized state check failed with RPC error", "err", stateErr, "pollFailures", pollFailures)
+						n.declareFinalizedStateNotAvailable()
+						return
 					}
 				} else {
-					finalizedStateFailures = 0
+					// Treat as RPC reachability error
+					if pollFailures < math.MaxUint32 {
+						n.metrics.IncrementPollsFailed(ctx, n.name)
+						pollFailures++
+					}
+					lggr.Warnw("Finalized state check failed with RPC error", "err", stateErr, "pollFailures", pollFailures)
 				}
+			} else {
+				finalizedStateFailures = 0
 			}
 		case bh, open := <-headsSub.Heads:
 			if !open {
@@ -757,43 +753,30 @@ func (n *node[CHAIN_ID, HEAD, RPC]) finalizedStateNotAvailableLoop() {
 		case <-time.After(dialRetryBackoff.Duration()):
 			lggr.Tracew("Trying to re-dial RPC node", "nodeState", n.getCachedState())
 
-			err := n.rpc.Dial(ctx)
-			if err != nil {
-				lggr.Errorw(fmt.Sprintf("Failed to redial RPC node: %v", err), "err", err, "nodeState", n.getCachedState())
+			state := n.createVerifiedConn(ctx, lggr)
+			if state != nodeStateAlive {
+				n.setState(nodeStateFinalizedStateNotAvailable)
 				continue
 			}
 
-			n.setState(nodeStateDialed)
-
-			state := n.verifyConn(ctx, lggr)
-			switch state {
-			case nodeStateUnreachable:
+			stateCheckCtx, stateCheckCancel := context.WithTimeout(ctx, n.nodePoolCfg.PollInterval())
+			stateErr := n.RPC().CheckFinalizedStateAvailability(stateCheckCtx)
+			stateCheckCancel()
+			if stateErr != nil {
+				if errors.Is(stateErr, ErrFinalizedStateUnavailable) {
+					lggr.Warnw("Finalized state still not available", "err", stateErr)
+					n.setState(nodeStateFinalizedStateNotAvailable)
+					continue
+				}
+				lggr.Warnw("Finalized state check failed with RPC error", "err", stateErr)
 				n.setState(nodeStateFinalizedStateNotAvailable)
 				continue
-			case nodeStateAlive:
-				lggr.Infow(fmt.Sprintf("Successfully redialled and verified RPC node %s. Finalized state was unavailable for %s", n.String(), time.Since(unavailableAt)), "nodeState", n.getCachedState())
-				fallthrough
-			default:
-				n.declareState(state)
-				return
 			}
+
+			lggr.Infow(fmt.Sprintf("Successfully redialled and verified RPC node %s. Finalized state was unavailable for %s", n.String(), time.Since(unavailableAt)), "nodeState", n.getCachedState())
+			n.declareState(nodeStateAlive)
+			return
 		}
 	}
 }
 
-// isFinalizedStateUnavailableError checks if the error indicates that the RPC cannot serve
-// historical state (as opposed to an RPC reachability issue).
-// If regexPattern is empty, all errors are treated as state unavailable errors.
-func isFinalizedStateUnavailableError(err error, regexPattern string) bool {
-	if err == nil {
-		return false
-	}
-	if regexPattern == "" {
-		return true
-	}
-	re, compileErr := regexp.Compile(regexPattern)
-	if compileErr != nil {
-		return true
-	}
-	return re.MatchString(err.Error())
-}