Skip to content

Commit 72b9577

Browse files
Krish-vemulaKrish-vemula
committed
Add FinalizedStateUnavailable to ClientErrors
1 parent 6788548 commit 72b9577

11 files changed

Lines changed: 328 additions & 143 deletions

‎metrics/multinode.go‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ var (
9292
Name: "pool_rpc_node_polls_success",
9393
Help: "The total number of successful poll checks for the given RPC node",
9494
}, []string{"network", "chainID", "nodeName"})
95+
promPoolRPCNodeFinalizedStateFailed = promauto.NewCounterVec(prometheus.CounterOpts{
96+
Name: "pool_rpc_node_finalized_state_failed",
97+
Help: "The total number of finalized state availability check failures for the given RPC node",
98+
}, []string{"network", "chainID", "nodeName"})
99+
promPoolRPCNodeTransitionsToFinalizedStateNotAvailable = promauto.NewCounterVec(prometheus.CounterOpts{
100+
Name: "pool_rpc_node_num_transitions_to_finalized_state_not_available",
101+
Help: "Total number of transitions to FinalizedStateNotAvailable",
102+
}, []string{"network", "chainID", "nodeName"})
95103

96104
// Transaction Sender
97105
promMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -120,6 +128,8 @@ type GenericMultiNodeMetrics interface {
120128
IncrementPolls(ctx context.Context, nodeName string)
121129
IncrementPollsFailed(ctx context.Context, nodeName string)
122130
IncrementPollsSuccess(ctx context.Context, nodeName string)
131+
IncrementFinalizedStateFailed(ctx context.Context, nodeName string)
132+
IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx context.Context, nodeName string)
123133
}
124134

125135
var _ GenericMultiNodeMetrics = &multiNodeMetrics{}
@@ -142,10 +152,12 @@ type multiNodeMetrics struct {
142152
highestSeenBlock metric.Int64Gauge
143153
highestFinalizedBlock metric.Int64Gauge
144154
seenBlocks metric.Int64Counter
145-
polls metric.Int64Counter
146-
pollsFailed metric.Int64Counter
147-
pollsSuccess metric.Int64Counter
148-
invariantViolations metric.Int64Counter
155+
polls metric.Int64Counter
156+
pollsFailed metric.Int64Counter
157+
pollsSuccess metric.Int64Counter
158+
finalizedStateFailed metric.Int64Counter
159+
nodeTransitionsToFinalizedStateNotAvailable metric.Int64Counter
160+
invariantViolations metric.Int64Counter
149161
}
150162

151163
func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNodeMetrics, error) {
@@ -239,6 +251,16 @@ func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNod
239251
return nil, fmt.Errorf("failed to register node polls success metric: %w", err)
240252
}
241253

254+
finalizedStateFailed, err := beholder.GetMeter().Int64Counter("pool_rpc_node_finalized_state_failed")
255+
if err != nil {
256+
return nil, fmt.Errorf("failed to register finalized state failed metric: %w", err)
257+
}
258+
259+
nodeTransitionsToFinalizedStateNotAvailable, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_finalized_state_not_available")
260+
if err != nil {
261+
return nil, fmt.Errorf("failed to register node transitions to finalized state not available metric: %w", err)
262+
}
263+
242264
invariantViolations, err := beholder.GetMeter().Int64Counter("multi_node_invariant_violations")
243265
if err != nil {
244266
return nil, fmt.Errorf("failed to register invariant violations metric: %w", err)
@@ -265,6 +287,8 @@ func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNod
265287
polls: polls,
266288
pollsFailed: pollsFailed,
267289
pollsSuccess: pollsSuccess,
290+
finalizedStateFailed: finalizedStateFailed,
291+
nodeTransitionsToFinalizedStateNotAvailable: nodeTransitionsToFinalizedStateNotAvailable,
268292
invariantViolations: invariantViolations,
269293
}, nil
270294
}
@@ -414,6 +438,22 @@ func (m *multiNodeMetrics) IncrementPollsSuccess(ctx context.Context, nodeName s
414438
attribute.String("nodeName", nodeName)))
415439
}
416440

441+
func (m *multiNodeMetrics) IncrementFinalizedStateFailed(ctx context.Context, nodeName string) {
442+
promPoolRPCNodeFinalizedStateFailed.WithLabelValues(m.network, m.chainID, nodeName).Inc()
443+
m.finalizedStateFailed.Add(ctx, 1, metric.WithAttributes(
444+
attribute.String("network", m.network),
445+
attribute.String("chainID", m.chainID),
446+
attribute.String("nodeName", nodeName)))
447+
}
448+
449+
func (m *multiNodeMetrics) IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx context.Context, nodeName string) {
450+
promPoolRPCNodeTransitionsToFinalizedStateNotAvailable.WithLabelValues(m.network, m.chainID, nodeName).Inc()
451+
m.nodeTransitionsToFinalizedStateNotAvailable.Add(ctx, 1, metric.WithAttributes(
452+
attribute.String("network", m.network),
453+
attribute.String("chainID", m.chainID),
454+
attribute.String("nodeName", nodeName)))
455+
}
456+
417457
func (m *multiNodeMetrics) IncrementInvariantViolations(ctx context.Context, invariant string) {
418458
promMultiNodeInvariantViolations.WithLabelValues(m.network, m.chainID, invariant).Inc()
419459
m.invariantViolations.Add(ctx, 1, metric.WithAttributes(

‎multinode/config/config.go‎

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,10 @@ type MultiNode struct {
3333
NoNewFinalizedHeadsThreshold *config.Duration
3434
FinalityDepth *uint32
3535
FinalityTagEnabled *bool
36-
FinalizedBlockOffset *uint32
36+
FinalizedBlockOffset *uint32
3737

3838
// Finalized State Availability Check
39-
FinalizedStateCheckEnabled *bool
40-
FinalizedStateCheckAddress *string
4139
FinalizedStateCheckFailureThreshold *uint32
42-
FinalizedStateUnavailableRegex *string
4340
}
4441

4542
func (c *MultiNodeConfig) Enabled() bool {
@@ -100,31 +97,10 @@ func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.Finali
10097

10198
func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset }
10299

103-
func (c *MultiNodeConfig) FinalizedStateCheckEnabled() bool {
104-
return c.MultiNode.FinalizedStateCheckEnabled != nil && *c.MultiNode.FinalizedStateCheckEnabled
105-
}
106-
107-
func (c *MultiNodeConfig) FinalizedStateCheckAddress() string {
108-
if c.MultiNode.FinalizedStateCheckAddress == nil {
109-
return ""
110-
}
111-
return *c.MultiNode.FinalizedStateCheckAddress
112-
}
113-
114100
func (c *MultiNodeConfig) FinalizedStateCheckFailureThreshold() uint32 {
115-
if c.MultiNode.FinalizedStateCheckFailureThreshold == nil {
116-
return 0
117-
}
118101
return *c.MultiNode.FinalizedStateCheckFailureThreshold
119102
}
120103

121-
func (c *MultiNodeConfig) FinalizedStateUnavailableRegex() string {
122-
if c.MultiNode.FinalizedStateUnavailableRegex == nil {
123-
return ""
124-
}
125-
return *c.MultiNode.FinalizedStateUnavailableRegex
126-
}
127-
128104
func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
129105
if f.MultiNode.Enabled != nil {
130106
c.MultiNode.Enabled = f.MultiNode.Enabled
@@ -183,16 +159,7 @@ func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
183159
}
184160

185161
// Finalized State Availability Check
186-
if f.MultiNode.FinalizedStateCheckEnabled != nil {
187-
c.MultiNode.FinalizedStateCheckEnabled = f.MultiNode.FinalizedStateCheckEnabled
188-
}
189-
if f.MultiNode.FinalizedStateCheckAddress != nil {
190-
c.MultiNode.FinalizedStateCheckAddress = f.MultiNode.FinalizedStateCheckAddress
191-
}
192162
if f.MultiNode.FinalizedStateCheckFailureThreshold != nil {
193163
c.MultiNode.FinalizedStateCheckFailureThreshold = f.MultiNode.FinalizedStateCheckFailureThreshold
194164
}
195-
if f.MultiNode.FinalizedStateUnavailableRegex != nil {
196-
c.MultiNode.FinalizedStateUnavailableRegex = f.MultiNode.FinalizedStateUnavailableRegex
197-
}
198165
}

‎multinode/mock_node_metrics_test.go‎

Lines changed: 68 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎multinode/mock_rpc_client_test.go‎

Lines changed: 10 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎multinode/node.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,7 @@ type NodeConfig interface {
2727
DeathDeclarationDelay() time.Duration
2828
NewHeadsPollInterval() time.Duration
2929
VerifyChainID() bool
30-
FinalizedStateCheckEnabled() bool
3130
FinalizedStateCheckFailureThreshold() uint32
32-
FinalizedStateCheckAddress() string
33-
FinalizedStateUnavailableRegex() string
3431
}
3532

3633
type ChainConfig interface {
@@ -52,13 +49,15 @@ type nodeMetrics interface {
5249
IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string)
5350
IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string)
5451
IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string)
52+
IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx context.Context, nodeName string)
5553
RecordNodeClientVersion(ctx context.Context, nodeName string, version string)
5654
SetHighestSeenBlock(ctx context.Context, nodeName string, blockNumber int64)
5755
SetHighestFinalizedBlock(ctx context.Context, nodeName string, blockNumber int64)
5856
IncrementSeenBlocks(ctx context.Context, nodeName string)
5957
IncrementPolls(ctx context.Context, nodeName string)
6058
IncrementPollsFailed(ctx context.Context, nodeName string)
6159
IncrementPollsSuccess(ctx context.Context, nodeName string)
60+
IncrementFinalizedStateFailed(ctx context.Context, nodeName string)
6261
}
6362

6463
type Node[
@@ -277,7 +276,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lgg
277276
// The node is already closed, and any subsequent transition is invalid.
278277
// To make spotting such transitions a bit easier, return the invalid node state.
279278
return nodeStateLen
280-
case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing:
279+
case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
281280
default:
282281
panic(fmt.Sprintf("cannot verify node in state %v", st))
283282
}

‎multinode/node_fsm.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) {
186186
return
187187
}
188188
switch n.state {
189-
case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing:
189+
case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
190190
n.state = nodeStateAlive
191191
default:
192192
panic(transitionFail(n.state, nodeStateAlive))
@@ -368,7 +368,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareFinalizedStateNotAvailable() {
368368
func (n *node[CHAIN_ID, HEAD, RPC]) transitionToFinalizedStateNotAvailable(fn func()) {
369369
ctx, cancel := n.stopCh.NewCtx()
370370
defer cancel()
371-
n.metrics.IncrementNodeTransitionsToUnreachable(ctx, n.name)
371+
n.metrics.IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx, n.name)
372372
n.stateMu.Lock()
373373
defer n.stateMu.Unlock()
374374
if n.state == nodeStateClosed {

0 commit comments

Comments
 (0)