add configurable historical balance health check for finalized block … (#352)

* add configurable historical balance health check for finalized block support

* added fix for build issues

* Make CheckFinalizedStateAvailability public for multinode polling, Accept probeAddress parameter, fall back to EVM config if empty.

* Address review feedback for finalized state check

* added default to fallback.toml

* fix: remove HistoricalBalanceCheckEnabled flag, let multinode control polling

* chore: bump chainlink-framework/multinode to PR commit version

* chore: regenerate CONFIG.md with finalized state check fields

* fix: validate FinalizedStateUnavailable config on startup

* Update pkg/client/rpc_client.go

Co-authored-by: amit-momin <108959691+amit-momin@users.noreply.github.com>

* chore: bump chainlink-framework/multinode to merged version

---------

Co-authored-by: amit-momin <108959691+amit-momin@users.noreply.github.com>

Author: Krish-vemula

CONFIG.md

@@ -932,6 +932,8 @@ SyncThreshold = 5 # Default
 LeaseDuration = '0s' # Default
 NodeIsSyncingEnabled = false # Default
 FinalizedBlockPollInterval = '5s' # Default
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default
+FinalizedStateCheckFailureThreshold = 0 # Default
 EnforceRepeatableRead = true # Default
 DeathDeclarationDelay = '1m' # Default
 NewHeadsPollInterval = '0s' # Default
@@ -1010,6 +1012,25 @@ reported based on latest block and finality depth.
 
 Set to 0 to disable.
 
+### HistoricalBalanceCheckAddress
+```toml
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default
+```
+HistoricalBalanceCheckAddress is the probe account used by the historical balance health check.
+The check executes `eth_getBalance` for this address at the latest finalized block.
+Finalized block selection follows chain finality settings:
+- `FinalityTagEnabled = true`: use `finalized` tag
+- `FinalityTagEnabled = false`: use `latest - FinalityDepth`
+The check is only active when `FinalizedStateCheckFailureThreshold > 0`.
+
+### FinalizedStateCheckFailureThreshold
+```toml
+FinalizedStateCheckFailureThreshold = 0 # Default
+```
+FinalizedStateCheckFailureThreshold is the number of consecutive failures of the finalized state availability check
+before the node is marked as FinalizedStateNotAvailable.
+Set to 0 to disable the check.
+
 ### EnforceRepeatableRead
 ```toml
 EnforceRepeatableRead = true # Default
@@ -1075,6 +1096,7 @@ Fatal = '(: |^)fatal' # Example
 ServiceUnavailable = '(: |^)service unavailable' # Example
 TooManyResults = '(: |^)too many results' # Example
 MissingBlocks = '(: |^)invalid block range' # Example
+FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example
 ```
 Errors enable the node to provide custom regex patterns to match against error messages from RPCs.
 
@@ -1174,6 +1196,12 @@ MissingBlocks = '(: |^)invalid block range' # Example
 ```
 MissingBlocks is a regex pattern to match an eth_getLogs error indicating the rpc server is permanently missing some blocks in the requested block range
 
+### FinalizedStateUnavailable
+```toml
+FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example
+```
+FinalizedStateUnavailable is a regex pattern to match errors indicating the RPC cannot serve historical state at the finalized block (e.g., pruned/non-archive node)
+
 ## OCR
 ```toml
 [OCR]

go.mod

@@ -33,8 +33,8 @@ require (
 	github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1
 	github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563
 	github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57
-	github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a
-	github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260326180413-c69f27e37a13
+	github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9
+	github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260401162955-be2bc6b5264b
 	github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396
 	github.com/smartcontractkit/chainlink-protos/svr v1.1.1-0.20260203131522-bb8bc5c423b3
 	github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20250815105909-75499abc4335

go.sum

@@ -662,10 +662,10 @@ github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-202508181755
 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563/go.mod h1:jP5mrOLFEYZZkl7EiCHRRIMSSHCQsYypm1OZSus//iI=
 github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57 h1:sCrr1Oy/JZstf/Oi2cRuU4mDN1BRUKfXP2CKByCMADg=
 github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57/go.mod h1:kGprqyjsz6qFNVszOQoHc24wfvCjyipNZFste/3zcbs=
-github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a h1:pr0VFI7AWlDVJBEkcvzXWd97V8w8QMNjRdfPVa/IQLk=
-github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a/go.mod h1:jo+cUqNcHwN8IF7SInQNXDZ8qzBsyMpnLdYbDswviFc=
-github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260326180413-c69f27e37a13 h1:3KLLkTCIAy9CvT35Ey0k6pcWX/u+qsm3Y/58TI5VSAg=
-github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260326180413-c69f27e37a13/go.mod h1:Y7h84PqCe/Vimf2h1Nc6tMiOJStDbtM33fEUeaaF5xk=
+github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 h1:GK+2aFpW/Z5ZnMGCa9NU6o7LKHQ/9xJVZx2yMAMudnc=
+github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9/go.mod h1:HG/aei0MgBOpsyRLexdKGtOUO8yjSJO3iUu0Uu8KBm4=
+github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260401162955-be2bc6b5264b h1:CNfoAw4HzvaeGwFHavdqU09DCGqVBzwe4dUr82qVZMs=
+github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260401162955-be2bc6b5264b/go.mod h1:n865LsUxibw9oJM0pH74EBiejJ/x/AgIGHaD99D3SDY=
 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396 h1:03tbcwjyIEjvHba1IWOj1sfThwebm2XNzyFHSuZtlWc=
 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396/go.mod h1:Jqt53s27Tr0jDl8mdBXg1xhu6F8Fci8JOuq43tgHOM8=
 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM=

pkg/client/helpers_test.go

@@ -39,6 +39,7 @@ type TestClientErrors struct {
 	serviceUnavailable                string
 	tooManyResults                    string
 	missingBlocks                     string
+	finalizedStateUnavailable         string
 }
 
 func NewTestClientErrors() TestClientErrors {
@@ -83,23 +84,26 @@ func (c *TestClientErrors) L2FeeTooHigh() string            { return c.l2FeeTooH
 func (c *TestClientErrors) L2Full() string                  { return c.l2Full }
 func (c *TestClientErrors) TransactionAlreadyMined() string { return c.transactionAlreadyMined }
 func (c *TestClientErrors) Fatal() string                   { return c.fatal }
-func (c *TestClientErrors) ServiceUnavailable() string      { return c.serviceUnavailable }
-func (c *TestClientErrors) TooManyResults() string          { return c.tooManyResults }
-func (c *TestClientErrors) MissingBlocks() string           { return c.missingBlocks }
+func (c *TestClientErrors) ServiceUnavailable() string       { return c.serviceUnavailable }
+func (c *TestClientErrors) TooManyResults() string           { return c.tooManyResults }
+func (c *TestClientErrors) MissingBlocks() string            { return c.missingBlocks }
+func (c *TestClientErrors) FinalizedStateUnavailable() string { return c.finalizedStateUnavailable }
 
 type TestNodePoolConfig struct {
-	NodePollFailureThreshold          uint32
-	NodePollInterval                  time.Duration
-	NodeSelectionMode                 string
-	NodeSyncThreshold                 uint32
-	NodeLeaseDuration                 time.Duration
-	NodeIsSyncingEnabledVal           bool
-	NodeFinalizedBlockPollInterval    time.Duration
-	NodeErrors                        config.ClientErrors
-	EnforceRepeatableReadVal          bool
-	NodeDeathDeclarationDelay         time.Duration
-	NodeNewHeadsPollInterval          time.Duration
-	ExternalRequestMaxResponseSizeVal uint32
+	NodePollFailureThreshold                uint32
+	NodePollInterval                        time.Duration
+	NodeSelectionMode                       string
+	NodeSyncThreshold                       uint32
+	NodeLeaseDuration                       time.Duration
+	NodeIsSyncingEnabledVal                 bool
+	NodeFinalizedBlockPollInterval          time.Duration
+	HistoricalBalanceCheckAddressVal        string
+	NodeErrors                              config.ClientErrors
+	EnforceRepeatableReadVal                bool
+	NodeDeathDeclarationDelay               time.Duration
+	NodeNewHeadsPollInterval                time.Duration
+	ExternalRequestMaxResponseSizeVal       uint32
+	FinalizedStateCheckFailureThresholdVal  uint32
 }
 
 func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold }
@@ -118,6 +122,10 @@ func (tc TestNodePoolConfig) FinalizedBlockPollInterval() time.Duration {
 	return tc.NodeFinalizedBlockPollInterval
 }
 
+func (tc TestNodePoolConfig) HistoricalBalanceCheckAddress() string {
+	return tc.HistoricalBalanceCheckAddressVal
+}
+
 func (tc TestNodePoolConfig) NewHeadsPollInterval() time.Duration {
 	return tc.NodeNewHeadsPollInterval
 }
@@ -142,6 +150,10 @@ func (tc TestNodePoolConfig) ExternalRequestMaxResponseSize() uint32 {
 	return tc.ExternalRequestMaxResponseSizeVal
 }
 
+func (tc TestNodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 {
+	return tc.FinalizedStateCheckFailureThresholdVal
+}
+
 func NewChainClientWithTestNode(
 	t *testing.T,
 	nodeCfg multinode.NodeConfig,

pkg/client/rpc_client.go

@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 	"sync/atomic"
 	"time"
@@ -105,6 +106,7 @@ type RPCClient struct {
 	finalityTagEnabled             bool
 	finalityDepth                  uint32
 	safeDepth                      uint32
+	historicalBalanceCheckAddress  common.Address
 	externalRequestMaxResponseSize uint32
 
 	beholderMetrics *rpcClientMetrics
@@ -144,6 +146,7 @@ func NewRPCClient(
 		finalityTagEnabled:             supportsFinalityTags,
 		finalityDepth:                  finalityDepth,
 		safeDepth:                      safeDepth,
+		historicalBalanceCheckAddress:  common.HexToAddress(cfg.HistoricalBalanceCheckAddress()),
 		externalRequestMaxResponseSize: externalRequestMaxResponseSize,
 	}
 	r.cfg = cfg
@@ -193,6 +196,51 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro
 	return version, nil
 }
 
+// CheckFinalizedStateAvailability verifies if the RPC can serve historical state at the finalized block.
+// This is used to detect non-archive nodes that cannot serve state queries for older blocks.
+// Returns multinode.ErrFinalizedStateUnavailable if the error matches the FinalizedStateUnavailable pattern.
+// The decision to call this method is made by multinode based on its configuration.
+func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context) error {
+	var blockNumber *big.Int
+	if r.finalityTagEnabled {
+		blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64())
+	} else {
+		latestBlock, err := r.BlockNumber(ctx)
+		if err != nil {
+			return fmt.Errorf("fetching latest block number failed: %w", err)
+		}
+		latest := int64(latestBlock)
+		finalizedHeight := max(int64(0), latest-int64(r.finalityDepth))
+		blockNumber = big.NewInt(finalizedHeight)
+	}
+	_, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber)
+	if err != nil {
+		if r.isFinalizedStateUnavailableError(err) {
+			return fmt.Errorf("%w: %w", multinode.ErrFinalizedStateUnavailable, err)
+		}
+		return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err)
+	}
+	return nil
+}
+
+// isFinalizedStateUnavailableError checks if the error matches the FinalizedStateUnavailable regex pattern.
+func (r *RPCClient) isFinalizedStateUnavailableError(err error) bool {
+	if err == nil {
+		return false
+	}
+	pattern := r.clientErrors.FinalizedStateUnavailable()
+	if pattern == "" {
+		r.rpcLog.Critical("FinalizedStateUnavailable regex pattern is empty; finalized state availability check is effectively disabled")
+		return false
+	}
+	re, compileErr := regexp.Compile(pattern)
+	if compileErr != nil {
+		r.rpcLog.Criticalw("FinalizedStateUnavailable regex pattern is invalid; finalized state availability check is effectively disabled", "pattern", pattern, "err", compileErr)
+		return false
+	}
+	return re.MatchString(err.Error())
+}
+
 func (r *RPCClient) Dial(callerCtx context.Context) error {
 	ctx, cancel, _ := r.AcquireQueryCtx(callerCtx, r.rpcTimeout)
 	defer cancel()

pkg/client/rpc_client_internal_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -436,3 +437,132 @@ func NewTestRPCClient(t *testing.T, opts RPCClientOpts) *RPCClient {
 func ptr[T any](v T) *T {
 	return &v
 }
+
+func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) {
+	t.Parallel()
+	chainID := big.NewInt(1337)
+	probeAddress := "0x0000000000000000000000000000000000000001"
+	expectedAddress := common.HexToAddress(probeAddress).String()
+
+	t.Run("uses finalized tag when enabled", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "eth_getBalance":
+				require.Equal(t, expectedAddress, params.Array()[0].String())
+				require.Equal(t, "finalized", params.Array()[1].String())
+				resp.Result = `"0x0"`
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		err := rpcClient.CheckFinalizedStateAvailability(t.Context())
+		require.NoError(t, err)
+	})
+
+	t.Run("uses latest-finalityDepth when finality tags disabled", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "eth_blockNumber":
+				resp.Result = `"0x14"` // 20
+			case "eth_getBalance":
+				require.Equal(t, expectedAddress, params.Array()[0].String())
+				require.Equal(t, "0x10", params.Array()[1].String()) // 20 - 4
+				resp.Result = `"0x0"`
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+			},
+			FinalityTagsEnabled: false,
+			FinalityDepth:       4,
+			ChainID:             chainID,
+		})
+
+		err := rpcClient.CheckFinalizedStateAvailability(t.Context())
+		require.NoError(t, err)
+	})
+
+	t.Run("returns ErrFinalizedStateUnavailable when error matches regex", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "eth_getBalance":
+				resp.Error.Message = "missing trie node"
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		clientErrors := NewTestClientErrors()
+		clientErrors.finalizedStateUnavailable = "missing trie node"
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+				NodeErrors:                       &clientErrors,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		err := rpcClient.CheckFinalizedStateAvailability(t.Context())
+		require.Error(t, err)
+		require.ErrorIs(t, err, multinode.ErrFinalizedStateUnavailable)
+	})
+
+	t.Run("returns generic error when error does not match regex", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "eth_getBalance":
+				resp.Error.Message = "connection reset"
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		clientErrors := NewTestClientErrors()
+		clientErrors.finalizedStateUnavailable = "missing trie node"
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+				NodeErrors:                       &clientErrors,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		err := rpcClient.CheckFinalizedStateAvailability(t.Context())
+		require.Error(t, err)
+		require.ErrorContains(t, err, "fetching balance")
+		require.NotErrorIs(t, err, multinode.ErrFinalizedStateUnavailable)
+	})
+}

pkg/config/chain_scoped_client_errors.go

@@ -50,3 +50,6 @@ func (c *clientErrorsConfig) ServiceUnavailable() string {
 }
 func (c *clientErrorsConfig) TooManyResults() string { return derefOrDefault(c.c.TooManyResults) }
 func (c *clientErrorsConfig) MissingBlocks() string  { return derefOrDefault(c.c.MissingBlocks) }
+func (c *clientErrorsConfig) FinalizedStateUnavailable() string {
+	return derefOrDefault(c.c.FinalizedStateUnavailable)
+}