diff --git a/Makefile b/Makefile index 12aa563..1416abe 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ gomodtidy: gomods .PHONY: mockery mockery: $(mockery) ## Install mockery. - go install github.com/vektra/mockery/v2@v2.46.3 + go install github.com/vektra/mockery/v2@v2.53.0 .PHONY: generate generate: mockery diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index ef19310..305cccf 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -9,8 +9,6 @@ import ( "time" "github.com/jpillora/backoff" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/multierr" "gopkg.in/guregu/null.v4" @@ -43,22 +41,6 @@ const ( hederaChainType = "hedera" ) -var ( - promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_broadcast", - Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - }, - }, []string{"chainID"}) -) - var ErrTxRemoved = errors.New("tx removed") type ProcessUnstartedTxs[ADDR chains.Hashable] func(ctx context.Context, fromAddress ADDR) (retryable bool, err error) @@ -79,6 +61,11 @@ type TransmitChecker[CID chains.ID, ADDR chains.Hashable, THASH, BHASH chains.Ha Check(ctx context.Context, l logger.SugaredLogger, tx types.Tx[CID, ADDR, THASH, BHASH, SEQ, FEE], a types.TxAttempt[CID, ADDR, THASH, BHASH, SEQ, FEE]) error } +type broadcasterMetrics interface { + IncrementNumBroadcastedTxs(ctx context.Context) + RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) +} + // Broadcaster monitors txes for transactions that need to // be broadcast, assigns sequences and ensures that at least one node // somewhere has received the transaction successfully. @@ -106,6 +93,7 @@ type Broadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, T feeConfig types.BroadcasterFeeConfig txConfig types.BroadcasterTransactionsConfig listenerConfig types.BroadcasterListenerConfig + metrics broadcasterMetrics // autoSyncSequence, if set, will cause Broadcaster to fast-forward the sequence // when Start is called @@ -144,6 +132,7 @@ func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable checkerFactory TransmitCheckerFactory[CID, ADDR, THASH, BHASH, SEQ, FEE], autoSyncSequence bool, chainType string, + metrics broadcasterMetrics, ) *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE] { lggr = logger.Named(lggr, "Broadcaster") b := &Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]{ @@ -161,6 +150,7 @@ func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable checkerFactory: checkerFactory, autoSyncSequence: autoSyncSequence, sequenceTracker: sequenceTracker, + metrics: metrics, } b.processUnstartedTxsImpl = b.processUnstartedTxs @@ -532,11 +522,12 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress // In all scenarios, the correct thing to do is assume success for now // and hand off to the confirmer to get the receipt (or mark as // failed). - observeTimeUntilBroadcast(eb.chainID, etx.CreatedAt, time.Now()) + observeTimeUntilBroadcast(ctx, eb.metrics, etx.CreatedAt, time.Now()) err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, types.TxAttemptBroadcast) if err != nil { return err, true } + eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) return err, true @@ -601,6 +592,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress if err != nil { return err, true } + eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) return err, true @@ -755,7 +747,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) saveFatallyError return eb.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, etx) } -func observeTimeUntilBroadcast[CHAIN_ID chains.ID](chainID CHAIN_ID, createdAt, broadcastAt time.Time) { +func observeTimeUntilBroadcast(ctx context.Context, metrics broadcasterMetrics, createdAt, broadcastAt time.Time) { duration := float64(broadcastAt.Sub(createdAt)) - promTimeUntilBroadcast.WithLabelValues(chainID.String()).Observe(duration) + metrics.RecordTimeUntilTxBroadcast(ctx, duration) } diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index 37fbfd5..03ef56c 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -10,8 +10,6 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/multierr" commonhex "github.com/smartcontractkit/chainlink-common/pkg/utils/hex" @@ -33,48 +31,13 @@ const ( processHeadTimeout = 10 * time.Minute ) -var ( - promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_gas_bumps", - Help: "Number of gas bumps", - }, []string{"chainID"}) - - promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_gas_bump_exceeds_limit", - Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", - }, []string{"chainID"}) - promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_confirmed_transactions", - Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", - }, []string{"chainID"}) - promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_confirmed", - Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - float64(5 * time.Minute), - float64(10 * time.Minute), - }, - }, []string{"chainID"}) - promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_blocks_until_tx_confirmed", - Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(1), - float64(5), - float64(10), - float64(20), - float64(50), - float64(100), - }, - }, []string{"chainID"}) -) +type confimerMetrics interface { + IncrementNumGasBumps(ctx context.Context) + IncrementGasBumpExceedsLimit(ctx context.Context) + IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) + RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) + RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) +} // Confirmer is a broad service which performs four different tasks in sequence on every new longest chain // Step 1: Mark that all currently pending transaction attempts were broadcast before this block @@ -95,6 +58,7 @@ type Confirmer[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, THA txConfig types.ConfirmerTransactionsConfig dbConfig types.ConfirmerDatabaseConfig chainID CID + metrics confimerMetrics ks types.KeyStore[ADDR] enabledAddresses []ADDR @@ -127,6 +91,7 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector types.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + metrics confimerMetrics, ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -143,6 +108,7 @@ func NewConfirmer[ mb: mailbox.NewSingle[HEAD](), isReceiptNil: isReceiptNil, stuckTxDetector: stuckTxDetector, + metrics: metrics, } } @@ -368,7 +334,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded return nil } // Add newly confirmed transactions to the prom metric - promNumConfirmedTxs.WithLabelValues(ec.chainID.String()).Add(float64(len(includedTxs))) + ec.metrics.IncrementNumConfirmedTxs(ctx, len(includedTxs)) purgeTxIDs := make([]int64, 0, len(includedTxs)) confirmedTxIDs := make([]int64, 0, len(includedTxs)) @@ -381,7 +347,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded continue } confirmedTxIDs = append(confirmedTxIDs, tx.ID) - observeUntilTxConfirmed(ec.chainID, tx.TxAttempts, head) + observeUntilTxConfirmed(ctx, ec.metrics, tx, head) } // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message if err := ec.txStore.UpdateTxFatalError(ctx, purgeTxIDs, ec.stuckTxDetector.StuckTxFatalError()); err != nil { @@ -667,13 +633,13 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) bumpGas(ctx con // if no error, return attempt // if err, continue below if err == nil { - promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementNumGasBumps(ctx) ec.lggr.Debugw("Rebroadcast bumping fee for tx", append(logFields, "bumpedFee", bumpedFee.String(), "bumpedFeeLimit", bumpedFeeLimit)...) return bumpedAttempt, err } if errors.Is(err, fees.ErrBumpFeeExceedsLimit) { - promGasBumpExceedsLimit.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementGasBumpExceedsLimit(ctx) } return bumpedAttempt, fmt.Errorf("error bumping gas: %w", err) @@ -712,7 +678,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) handleInProgres if err != nil { return fmt.Errorf("could not bump gas for terminally underpriced transaction: %w", err) } - promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementNumGasBumps(ctx) lggr.With( "sendError", sendError, "maxGasPriceConfig", ec.feeConfig.MaxFeePrice(), @@ -853,38 +819,35 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) sendEmptyTransa return txhash, nil } -// observeUntilTxConfirmed observes the promBlocksUntilTxConfirmed metric for each confirmed -// transaction. +// observeUntilTxConfirmed observes the timeUntilTxConfirmed and blocksUntilTxConfirmed metrics for each confirmed transaction. func observeUntilTxConfirmed[ CHAIN_ID chains.ID, ADDR chains.Hashable, TX_HASH, BLOCK_HASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee, -](chainID CHAIN_ID, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { - for _, attempt := range attempts { - // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) - // was created. We want to measure the amount of time taken from when a transaction is created - // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts - // were needed to achieve this. - duration := time.Since(attempt.Tx.CreatedAt) - promTimeUntilTxConfirmed. - WithLabelValues(chainID.String()). - Observe(float64(duration)) - - // Since a tx can have many attempts, we take the number of blocks to confirm as the block number - // of the receipt minus the block number of the first ever broadcast for this transaction. - var minBroadcastBefore int64 - for _, a := range attempt.Tx.TxAttempts { - if b := a.BroadcastBeforeBlockNum; b != nil && *b < minBroadcastBefore { - minBroadcastBefore = *b - } - } - if minBroadcastBefore > 0 { - blocksElapsed := head.BlockNumber() - minBroadcastBefore - promBlocksUntilTxConfirmed. - WithLabelValues(chainID.String()). - Observe(float64(blocksElapsed)) +](ctx context.Context, metrics confimerMetrics, tx *types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { + if tx == nil { + return + } + // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) + // was created. We want to measure the amount of time taken from when a transaction is created + // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts + // were needed to achieve this. + duration := time.Since(tx.CreatedAt) + metrics.RecordTimeUntilTxConfirmed(ctx, float64(duration)) + + // Since a tx can have many attempts, we take the number of blocks to confirm as the current block number + // minus the block number of the first ever broadcast for this transaction. + var minBroadcastBefore int64 + for _, a := range tx.TxAttempts { + if b := a.BroadcastBeforeBlockNum; b != nil && *b < minBroadcastBefore { + minBroadcastBefore = *b } } + + if minBroadcastBefore > 0 { + blocksElapsed := head.BlockNumber() - minBroadcastBefore + metrics.RecordBlocksUntilTxConfirmed(ctx, float64(blocksElapsed)) + } } diff --git a/metrics/go.mod b/metrics/go.mod index 85b9778..52742bd 100644 --- a/metrics/go.mod +++ b/metrics/go.mod @@ -2,15 +2,66 @@ module github.com/smartcontractkit/chainlink-framework/metrics go 1.24.1 -require github.com/prometheus/client_golang v1.22.0 +require ( + github.com/prometheus/client_golang v1.22.0 + github.com/smartcontractkit/chainlink-common v0.7.0 + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/metric v1.35.0 +) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect + github.com/cloudevents/sdk-go/v2 v2.16.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.30.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/metrics/go.sum b/metrics/go.sum index a89f616..8cb99f7 100644 --- a/metrics/go.sum +++ b/metrics/go.sum @@ -1,28 +1,157 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP0qYY6yzdIU9xdrRd/YMP0bNwFjXd0u8= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= +github.com/cloudevents/sdk-go/v2 v2.16.0 h1:wnunjgiLQCfYlyo+E4+mFlZtAh7pKn7vT8MMD3lSwCg= +github.com/cloudevents/sdk-go/v2 v2.16.0/go.mod h1:5YWqklyhDSmGzBK/JENKKXdulbPq0JFf3c/KEnMLqgg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= +github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 h1:UiRNKd1OgqsLbFwE+wkAWTdiAxXtCBqKIHeBIse4FUA= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9/go.mod h1:eqZlW3pJWhjyexnDPrdQxix1pn0wwhI4AO4GKpP/bMI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/logpoller.go b/metrics/logpoller.go index 03b6384..0963a68 100644 --- a/metrics/logpoller.go +++ b/metrics/logpoller.go @@ -1,10 +1,16 @@ package metrics import ( + "context" + "fmt" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" ) type QueryType string @@ -38,21 +44,102 @@ var ( float64(2 * time.Second), float64(5 * time.Second), } - LpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + PromLpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "log_poller_query_duration", Help: "Measures duration of Log Poller's queries fetching logs", Buckets: sqlLatencyBuckets, }, []string{"chainFamily", "chainID", "query", "type"}) - LpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ + PromLpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "log_poller_query_dataset_size", Help: "Measures size of the datasets returned by Log Poller's queries", }, []string{"chainFamily", "chainID", "query", "type"}) - LpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + PromLpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_logs_inserted", Help: "Counter to track number of logs inserted by Log Poller", }, []string{"chainFamily", "chainID"}) - LpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + PromLpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_blocks_inserted", Help: "Counter to track number of blocks inserted by Log Poller", }, []string{"chainFamily", "chainID"}) ) + +type GenericLogPollerMetrics interface { + RecordQueryDuration(ctx context.Context, queryName string, queryType QueryType, duration float64) + RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) + IncrementLogsInserted(ctx context.Context, numLogs int64) + IncrementBlocksInserted(ctx context.Context, numBlocks int64) +} + +var _ GenericLogPollerMetrics = &logPollerMetrics{} + +type logPollerMetrics struct { + chainID string + chainFamily string + queryDuration metric.Float64Histogram + queryDatasetsSize metric.Int64Gauge + logsInserted metric.Int64Counter + blocksInserted metric.Int64Counter +} + +func NewGenericLogPollerMetrics(chainID string, chainFamily string) (GenericLogPollerMetrics, error) { + queryDuration, err := beholder.GetMeter().Float64Histogram("log_poller_query_duration") + if err != nil { + return nil, fmt.Errorf("failed to register logpoller query duration metric: %w", err) + } + + queryDatasetSize, err := beholder.GetMeter().Int64Gauge("log_poller_query_dataset_size") + if err != nil { + return nil, fmt.Errorf("failed to register query dataset size metric: %w", err) + } + + logsInserted, err := beholder.GetMeter().Int64Counter("log_poller_logs_inserted") + if err != nil { + return nil, fmt.Errorf("failed to register logs inserted metric: %w", err) + } + + blocksInserted, err := beholder.GetMeter().Int64Counter("log_poller_blocks_inserted") + if err != nil { + return nil, fmt.Errorf("failed to register blocks inserted metric: %w", err) + } + + return &logPollerMetrics{ + chainID: chainID, + chainFamily: chainFamily, + queryDuration: queryDuration, + queryDatasetsSize: queryDatasetSize, + logsInserted: logsInserted, + blocksInserted: blocksInserted, + }, nil +} + +func (m *logPollerMetrics) RecordQueryDuration(ctx context.Context, queryName string, queryType QueryType, duration float64) { + PromLpQueryDuration.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Observe(duration) + m.queryDuration.Record(ctx, duration, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID), + attribute.String("query", queryName), + attribute.String("type", string(queryType)))) +} + +func (m *logPollerMetrics) RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) { + PromLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Set(float64(size)) + m.queryDatasetsSize.Record(ctx, size, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID), + attribute.String("query", queryName), + attribute.String("type", string(queryType)))) +} + +func (m *logPollerMetrics) IncrementLogsInserted(ctx context.Context, numLogs int64) { + PromLpLogsInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numLogs)) + m.logsInserted.Add(ctx, numLogs, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID))) +} + +func (m *logPollerMetrics) IncrementBlocksInserted(ctx context.Context, numBlocks int64) { + PromLpBlocksInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numBlocks)) + m.blocksInserted.Add(ctx, numBlocks, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID))) +} diff --git a/metrics/logpoller_test.go b/metrics/logpoller_test.go new file mode 100644 index 0000000..2c34bc3 --- /dev/null +++ b/metrics/logpoller_test.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func setupTestLogPollerMetrics(t *testing.T) GenericLogPollerMetrics { + m, err := NewGenericLogPollerMetrics("1", "test-network") + require.NoError(t, err) + return m +} + +func TestLogPollerMetrics_RecordQueryDuration(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.RecordQueryDuration(ctx, "PollLogs", Read, 0.005) // 5ms + + // Collect and make sure at least one sample was recorded + require.Greater(t, testutil.CollectAndCount(PromLpQueryDuration), 0) +} + +func TestLogPollerMetrics_RecordQueryDatasetSize(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.RecordQueryDatasetSize(ctx, "PollLogs", Read, 10) + + require.Equal(t, + float64(10), + testutil.ToFloat64(PromLpQueryDataSets.WithLabelValues("test-network", "1", "PollLogs", "read")), + ) +} + +func TestLogPollerMetrics_IncrementLogsInserted(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.IncrementLogsInserted(ctx, 5) + + require.Equal(t, + float64(5), + testutil.ToFloat64(PromLpLogsInserted.WithLabelValues("test-network", "1")), + ) +} + +func TestLogPollerMetrics_IncrementBlocksInserted(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.IncrementBlocksInserted(ctx, 3) + + require.Equal(t, + float64(3), + testutil.ToFloat64(PromLpBlocksInserted.WithLabelValues("test-network", "1")), + ) +} diff --git a/metrics/multinode.go b/metrics/multinode.go new file mode 100644 index 0000000..5717253 --- /dev/null +++ b/metrics/multinode.go @@ -0,0 +1,280 @@ +package metrics + +import ( + "context" + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" +) + +var ( + // Node States + promMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multi_node_states", + Help: "The number of RPC nodes currently in the given state for the given chain", + }, []string{"network", "chainId", "state"}) + + // Node Verification + promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies", + Help: "The total number of chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_failed", + Help: "The total number of failed chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_success", + Help: "The total number of successful chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + + // Node State Transitions + promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_alive", + Help: "Total number of times node has transitioned to Alive", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_in_sync", + Help: "Total number of times node has transitioned from OutOfSync to Alive", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Help: "Total number of times node has transitioned to OutOfSync", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unreachable", + Help: "Total number of times node has transitioned to Unreachable", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Help: "Total number of times node has transitioned to InvalidChainID", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unusable", + Help: "Total number of times node has transitioned to Unusable", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_syncing", + Help: "Total number of times node has transitioned to Syncing", + }, []string{"network", "chainID", "nodeName"}) + + // Transaction Sender + promMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multi_node_invariant_violations", + Help: "The number of invariant violations", + }, []string{"network", "chainId", "invariant"}) +) + +type GenericMultiNodeMetrics interface { + RecordNodeStates(ctx context.Context, state string, count int64) + IncrementNodeVerifies(ctx context.Context, nodeName string) + IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) + IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) + IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) + IncrementInvariantViolations(ctx context.Context, invariant string) +} + +var _ GenericMultiNodeMetrics = &multiNodeMetrics{} + +type multiNodeMetrics struct { + network string + chainID string + nodeStates metric.Int64Gauge + nodeVerifies metric.Int64Counter + nodeVerifiesFailed metric.Int64Counter + nodeVerifiesSuccess metric.Int64Counter + nodeTransitionsToAlive metric.Int64Counter + nodeTransitionsToInSync metric.Int64Counter + nodeTransitionsToOutOfSync metric.Int64Counter + nodeTransitionsToUnreachable metric.Int64Counter + nodeTransitionsToInvalidChainID metric.Int64Counter + nodeTransitionsToUnusable metric.Int64Counter + nodeTransitionsToSyncing metric.Int64Counter + invariantViolations metric.Int64Counter +} + +func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNodeMetrics, error) { + nodeStates, err := beholder.GetMeter().Int64Gauge("multi_node_states") + if err != nil { + return nil, fmt.Errorf("failed to register multinode states metric: %w", err) + } + + nodeVerifies, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies metric: %w", err) + } + + nodeVerifiesFailed, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies_failed") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies failed metric: %w", err) + } + + nodeVerifiesSuccess, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies_success") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies success metric: %w", err) + } + + nodeTransitionsToAlive, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_alive") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to alive metric: %w", err) + } + + nodeTransitionsToInSync, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_in_sync") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to in sync metric: %w", err) + } + + nodeTransitionsToOutOfSync, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_out_of_sync") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to out of sync metric: %w", err) + } + + nodeTransitionsToUnreachable, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_unreachable") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to unreachable metric: %w", err) + } + + nodeTransitionsToInvalidChainID, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_invalid_chain_id") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to invalid chain id metric: %w", err) + } + + nodeTransitionsToUnusable, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_unusable") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to unusable metric: %w", err) + } + + nodeTransitionsToSyncing, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_syncing") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to syncing metric: %w", err) + } + + invariantViolations, err := beholder.GetMeter().Int64Counter("multi_node_invariant_violations") + if err != nil { + return nil, fmt.Errorf("failed to register invariant violations metric: %w", err) + } + + return &multiNodeMetrics{ + network: network, + chainID: chainID, + nodeStates: nodeStates, + nodeVerifies: nodeVerifies, + nodeVerifiesFailed: nodeVerifiesFailed, + nodeVerifiesSuccess: nodeVerifiesSuccess, + nodeTransitionsToAlive: nodeTransitionsToAlive, + nodeTransitionsToInSync: nodeTransitionsToInSync, + nodeTransitionsToOutOfSync: nodeTransitionsToOutOfSync, + nodeTransitionsToUnreachable: nodeTransitionsToUnreachable, + nodeTransitionsToInvalidChainID: nodeTransitionsToInvalidChainID, + nodeTransitionsToUnusable: nodeTransitionsToUnusable, + nodeTransitionsToSyncing: nodeTransitionsToSyncing, + invariantViolations: invariantViolations, + }, nil +} + +func (m *multiNodeMetrics) RecordNodeStates(ctx context.Context, state string, count int64) { + promMultiNodeRPCNodeStates.WithLabelValues(m.network, m.chainID, state).Set(float64(count)) + m.nodeStates.Record(ctx, count, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("state", state))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifies(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifies.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifies.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifiesFailed.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifiesFailed.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifiesSuccess.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifiesSuccess.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToAlive.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToAlive.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToInSync.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToInSync.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToOutOfSync.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToUnreachable.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToInvalidChainID.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToUnusable.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToUnusable.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToSyncing.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToSyncing.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementInvariantViolations(ctx context.Context, invariant string) { + promMultiNodeInvariantViolations.WithLabelValues(m.network, m.chainID, invariant).Inc() + m.invariantViolations.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("invariant", invariant))) +} diff --git a/metrics/multinode_test.go b/metrics/multinode_test.go new file mode 100644 index 0000000..eecb2ae --- /dev/null +++ b/metrics/multinode_test.go @@ -0,0 +1,129 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func setupTestMultiNodeMetrics(t *testing.T) GenericMultiNodeMetrics { + m, err := NewGenericMultiNodeMetrics("test-network", "1") + require.NoError(t, err) + return m +} + +func TestMultiNodeMetrics_RecordNodeStates(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.RecordNodeStates(ctx, "Alive", 5) + + require.Equal(t, float64(5), + testutil.ToFloat64(promMultiNodeRPCNodeStates.WithLabelValues("test-network", "1", "Alive")), + ) +} + +func TestMultiNodeMetrics_Verifies(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.IncrementNodeVerifies(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifies.WithLabelValues("test-network", "1", "node-1")), + ) + + m.IncrementNodeVerifiesFailed(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifiesFailed.WithLabelValues("test-network", "1", "node-1")), + ) + + m.IncrementNodeVerifiesSuccess(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifiesSuccess.WithLabelValues("test-network", "1", "node-1")), + ) +} + +func TestMultiNodeMetrics_NodeTransitions(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + nodeName := "node-1" + + tests := []struct { + name string + increment func() + promMetric *prometheus.CounterVec + }{ + { + name: "Alive", + increment: func() { + m.IncrementNodeTransitionsToAlive(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToAlive, + }, + { + name: "InSync", + increment: func() { + m.IncrementNodeTransitionsToInSync(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToInSync, + }, + { + name: "OutOfSync", + increment: func() { + m.IncrementNodeTransitionsToOutOfSync(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToOutOfSync, + }, + { + name: "Unreachable", + increment: func() { + m.IncrementNodeTransitionsToUnreachable(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToUnreachable, + }, + { + name: "InvalidChainID", + increment: func() { + m.IncrementNodeTransitionsToInvalidChainID(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToInvalidChainID, + }, + { + name: "Unusable", + increment: func() { + m.IncrementNodeTransitionsToUnusable(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToUnusable, + }, + { + name: "Syncing", + increment: func() { + m.IncrementNodeTransitionsToSyncing(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToSyncing, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.increment() + require.Equal(t, float64(1), + testutil.ToFloat64(tt.promMetric.WithLabelValues("test-network", "1", nodeName)), + ) + }) + } +} + +func TestMultiNodeMetrics_IncrementInvariantViolations(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.IncrementInvariantViolations(ctx, "wrong_nonce") + + require.Equal(t, float64(1), + testutil.ToFloat64(promMultiNodeInvariantViolations.WithLabelValues("test-network", "1", "wrong_nonce")), + ) +} diff --git a/metrics/txm.go b/metrics/txm.go new file mode 100644 index 0000000..ae8be59 --- /dev/null +++ b/metrics/txm.go @@ -0,0 +1,178 @@ +package metrics + +import ( + "context" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" +) + +var ( + promNumBroadcasted = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_broadcasted", + Help: "The number of transactions broadcasted", + }, []string{"chainID"}) + promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_broadcast", + Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + }, + }, []string{"chainID"}) + promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_gas_bumps", + Help: "Number of gas bumps", + }, []string{"chainID"}) + + promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_gas_bump_exceeds_limit", + Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + float64(5 * time.Minute), + float64(10 * time.Minute), + }, + }, []string{"chainID"}) + promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_blocks_until_tx_confirmed", + Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(1), + float64(5), + float64(10), + float64(20), + float64(50), + float64(100), + }, + }, []string{"chainID"}) +) + +type GenericTXMMetrics interface { + IncrementNumBroadcastedTxs(ctx context.Context) + RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) + IncrementNumGasBumps(ctx context.Context) + IncrementGasBumpExceedsLimit(ctx context.Context) + IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) + RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) + RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) +} + +type txmMetrics struct { + chainID string + numBroadcastedTxs metric.Int64Counter + timeUntilBroadcast metric.Float64Histogram + numGasBumps metric.Int64Counter + gasBumpExceedsLimit metric.Int64Counter + numConfirmedTxs metric.Int64Counter + timeUntilTxConfirmed metric.Float64Histogram + blocksUntilTxConfirmed metric.Float64Histogram +} + +func NewGenericTxmMetrics(chainID string) (GenericTXMMetrics, error) { + numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") + if err != nil { + return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) + } + + timeUntilBroadcast, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_broadcast") + if err != nil { + return nil, fmt.Errorf("failed to register time until broadcast metric: %w", err) + } + + numGasBumps, err := beholder.GetMeter().Int64Counter("tx_manager_num_gas_bumps") + if err != nil { + return nil, fmt.Errorf("failed to register number of gas bumps metric: %w", err) + } + + gasBumpExceedsLimit, err := beholder.GetMeter().Int64Counter("tx_manager_gas_bump_exceeds_limit") + if err != nil { + return nil, fmt.Errorf("failed to register gas bump exceeds limit metric: %w", err) + } + + numConfirmedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_confirmed_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register confirmed txs number metric: %w", err) + } + + timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register time until tx confirmed metric: %w", err) + } + + blocksUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_blocks_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register blocks until tx confirmed metric: %w", err) + } + + return &txmMetrics{ + chainID: chainID, + numBroadcastedTxs: numBroadcastedTxs, + timeUntilBroadcast: timeUntilBroadcast, + numGasBumps: numGasBumps, + gasBumpExceedsLimit: gasBumpExceedsLimit, + numConfirmedTxs: numConfirmedTxs, + timeUntilTxConfirmed: timeUntilTxConfirmed, + blocksUntilTxConfirmed: blocksUntilTxConfirmed, + }, nil +} + +func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { + promNumBroadcasted.WithLabelValues(m.chainID).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) { + promTimeUntilBroadcast.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilBroadcast.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementNumGasBumps(ctx context.Context) { + promNumGasBumps.WithLabelValues(m.chainID).Add(float64(1)) + m.numGasBumps.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementGasBumpExceedsLimit(ctx context.Context) { + promGasBumpExceedsLimit.WithLabelValues(m.chainID).Add(float64(1)) + m.gasBumpExceedsLimit.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { + promNumConfirmedTxs.WithLabelValues(m.chainID).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions), metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { + promTimeUntilTxConfirmed.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) { + promBlocksUntilTxConfirmed.WithLabelValues(m.chainID).Observe(blocksElapsed) + m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} diff --git a/multinode/.mockery.yaml b/multinode/.mockery.yaml index 84e8332..91aef66 100644 --- a/multinode/.mockery.yaml +++ b/multinode/.mockery.yaml @@ -18,3 +18,7 @@ packages: Head: PoolChainInfoProvider: Subscription: + multiNodeMetrics: + nodeMetrics: + sendOnlyNodeMetrics: + transactionSenderMetrics: diff --git a/multinode/go.mod b/multinode/go.mod index afb5d43..8d8958c 100644 --- a/multinode/go.mod +++ b/multinode/go.mod @@ -1,40 +1,74 @@ module github.com/smartcontractkit/chainlink-framework/multinode -go 1.23.3 +go 1.24.1 + +toolchain go1.24.2 require ( github.com/jpillora/backoff v1.0.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.7.0 github.com/stretchr/testify v1.10.0 go.uber.org/zap v1.27.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect + github.com/cloudevents/sdk-go/v2 v2.16.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.60.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect github.com/stretchr/objx v0.5.2 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/sys v0.26.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/multinode/go.sum b/multinode/go.sum index 6c3095e..71c59db 100644 --- a/multinode/go.sum +++ b/multinode/go.sum @@ -1,7 +1,13 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP0qYY6yzdIU9xdrRd/YMP0bNwFjXd0u8= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= +github.com/cloudevents/sdk-go/v2 v2.16.0 h1:wnunjgiLQCfYlyo+E4+mFlZtAh7pKn7vT8MMD3lSwCg= +github.com/cloudevents/sdk-go/v2 v2.16.0/go.mod h1:5YWqklyhDSmGzBK/JENKKXdulbPq0JFf3c/KEnMLqgg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -11,77 +17,147 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= -github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= +github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 h1:UiRNKd1OgqsLbFwE+wkAWTdiAxXtCBqKIHeBIse4FUA= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9/go.mod h1:eqZlW3pJWhjyexnDPrdQxix1pn0wwhI4AO4GKpP/bMI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/multinode/mock_head_test.go b/multinode/mock_head_test.go index bd3d414..291a83c 100644 --- a/multinode/mock_head_test.go +++ b/multinode/mock_head_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -21,7 +21,7 @@ func (_m *mockHead) EXPECT() *mockHead_Expecter { return &mockHead_Expecter{mock: &_m.Mock} } -// BlockDifficulty provides a mock function with given fields: +// BlockDifficulty provides a mock function with no fields func (_m *mockHead) BlockDifficulty() *big.Int { ret := _m.Called() @@ -68,7 +68,7 @@ func (_c *mockHead_BlockDifficulty_Call) RunAndReturn(run func() *big.Int) *mock return _c } -// BlockNumber provides a mock function with given fields: +// BlockNumber provides a mock function with no fields func (_m *mockHead) BlockNumber() int64 { ret := _m.Called() @@ -113,7 +113,7 @@ func (_c *mockHead_BlockNumber_Call) RunAndReturn(run func() int64) *mockHead_Bl return _c } -// GetTotalDifficulty provides a mock function with given fields: +// GetTotalDifficulty provides a mock function with no fields func (_m *mockHead) GetTotalDifficulty() *big.Int { ret := _m.Called() @@ -160,7 +160,7 @@ func (_c *mockHead_GetTotalDifficulty_Call) RunAndReturn(run func() *big.Int) *m return _c } -// IsValid provides a mock function with given fields: +// IsValid provides a mock function with no fields func (_m *mockHead) IsValid() bool { ret := _m.Called() diff --git a/multinode/mock_multi_node_metrics_test.go b/multinode/mock_multi_node_metrics_test.go new file mode 100644 index 0000000..0537463 --- /dev/null +++ b/multinode/mock_multi_node_metrics_test.go @@ -0,0 +1,71 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockMultiNodeMetrics is an autogenerated mock type for the multiNodeMetrics type +type mockMultiNodeMetrics struct { + mock.Mock +} + +type mockMultiNodeMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockMultiNodeMetrics) EXPECT() *mockMultiNodeMetrics_Expecter { + return &mockMultiNodeMetrics_Expecter{mock: &_m.Mock} +} + +// RecordNodeStates provides a mock function with given fields: ctx, state, count +func (_m *mockMultiNodeMetrics) RecordNodeStates(ctx context.Context, state string, count int64) { + _m.Called(ctx, state, count) +} + +// mockMultiNodeMetrics_RecordNodeStates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordNodeStates' +type mockMultiNodeMetrics_RecordNodeStates_Call struct { + *mock.Call +} + +// RecordNodeStates is a helper method to define mock.On call +// - ctx context.Context +// - state string +// - count int64 +func (_e *mockMultiNodeMetrics_Expecter) RecordNodeStates(ctx interface{}, state interface{}, count interface{}) *mockMultiNodeMetrics_RecordNodeStates_Call { + return &mockMultiNodeMetrics_RecordNodeStates_Call{Call: _e.mock.On("RecordNodeStates", ctx, state, count)} +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) Run(run func(ctx context.Context, state string, count int64)) *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int64)) + }) + return _c +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) Return() *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Call.Return() + return _c +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) RunAndReturn(run func(context.Context, string, int64)) *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Run(run) + return _c +} + +// newMockMultiNodeMetrics creates a new instance of mockMultiNodeMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockMultiNodeMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockMultiNodeMetrics { + mock := &mockMultiNodeMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/mock_node_metrics_test.go b/multinode/mock_node_metrics_test.go new file mode 100644 index 0000000..fc28931 --- /dev/null +++ b/multinode/mock_node_metrics_test.go @@ -0,0 +1,376 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockNodeMetrics is an autogenerated mock type for the nodeMetrics type +type mockNodeMetrics struct { + mock.Mock +} + +type mockNodeMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockNodeMetrics) EXPECT() *mockNodeMetrics_Expecter { + return &mockNodeMetrics_Expecter{mock: &_m.Mock} +} + +// IncrementNodeTransitionsToAlive provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToAlive_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToAlive' +type mockNodeMetrics_IncrementNodeTransitionsToAlive_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToAlive is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToAlive(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToAlive_Call{Call: _e.mock.On("IncrementNodeTransitionsToAlive", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToInSync provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToInSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToInSync' +type mockNodeMetrics_IncrementNodeTransitionsToInSync_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToInSync is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToInSync(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToInSync_Call{Call: _e.mock.On("IncrementNodeTransitionsToInSync", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToInvalidChainID provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToInvalidChainID' +type mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToInvalidChainID is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToInvalidChainID(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call{Call: _e.mock.On("IncrementNodeTransitionsToInvalidChainID", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToOutOfSync provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToOutOfSync' +type mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToOutOfSync is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToOutOfSync(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call{Call: _e.mock.On("IncrementNodeTransitionsToOutOfSync", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToSyncing provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToSyncing' +type mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToSyncing is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToSyncing(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call{Call: _e.mock.On("IncrementNodeTransitionsToSyncing", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToUnreachable provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToUnreachable' +type mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToUnreachable is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToUnreachable(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call{Call: _e.mock.On("IncrementNodeTransitionsToUnreachable", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToUnusable provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToUnusable' +type mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToUnusable is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToUnusable(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call{Call: _e.mock.On("IncrementNodeTransitionsToUnusable", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifies provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifies(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifies_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifies' +type mockNodeMetrics_IncrementNodeVerifies_Call struct { + *mock.Call +} + +// IncrementNodeVerifies is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifies(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifies_Call { + return &mockNodeMetrics_IncrementNodeVerifies_Call{Call: _e.mock.On("IncrementNodeVerifies", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) Return() *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifiesFailed provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifiesFailed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifiesFailed' +type mockNodeMetrics_IncrementNodeVerifiesFailed_Call struct { + *mock.Call +} + +// IncrementNodeVerifiesFailed is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifiesFailed(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + return &mockNodeMetrics_IncrementNodeVerifiesFailed_Call{Call: _e.mock.On("IncrementNodeVerifiesFailed", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) Return() *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifiesSuccess provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifiesSuccess_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifiesSuccess' +type mockNodeMetrics_IncrementNodeVerifiesSuccess_Call struct { + *mock.Call +} + +// IncrementNodeVerifiesSuccess is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifiesSuccess(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + return &mockNodeMetrics_IncrementNodeVerifiesSuccess_Call{Call: _e.mock.On("IncrementNodeVerifiesSuccess", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) Return() *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Run(run) + return _c +} + +// newMockNodeMetrics creates a new instance of mockNodeMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockNodeMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockNodeMetrics { + mock := &mockNodeMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/mock_node_selector_test.go b/multinode/mock_node_selector_test.go index 6613b51..08e4d76 100644 --- a/multinode/mock_node_selector_test.go +++ b/multinode/mock_node_selector_test.go @@ -1,15 +1,15 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode import mock "github.com/stretchr/testify/mock" // mockNodeSelector is an autogenerated mock type for the NodeSelector type -type mockNodeSelector[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockNodeSelector_Expecter[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -17,7 +17,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) EXPECT() *mockNodeSelector_Expecter[C return &mockNodeSelector_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockNodeSelector[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -36,7 +36,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) Name() string { } // mockNodeSelector_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockNodeSelector_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -62,7 +62,7 @@ func (_c *mockNodeSelector_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() str return _c } -// Select provides a mock function with given fields: +// Select provides a mock function with no fields func (_m *mockNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { ret := _m.Called() @@ -83,7 +83,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { } // mockNodeSelector_Select_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Select' -type mockNodeSelector_Select_Call[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Select_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +111,7 @@ func (_c *mockNodeSelector_Select_Call[CHAIN_ID, RPC]) RunAndReturn(run func() N // newMockNodeSelector creates a new instance of mockNodeSelector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockNodeSelector[CHAIN_ID ID, RPC any](t interface { +func newMockNodeSelector[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockNodeSelector[CHAIN_ID, RPC] { diff --git a/multinode/mock_node_test.go b/multinode/mock_node_test.go index 3924591..917fd63 100644 --- a/multinode/mock_node_test.go +++ b/multinode/mock_node_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -9,11 +9,11 @@ import ( ) // mockNode is an autogenerated mock type for the Node type -type mockNode[CHAIN_ID ID, RPC any] struct { +type mockNode[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockNode_Expecter[CHAIN_ID ID, RPC any] struct { +type mockNode_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -21,7 +21,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) EXPECT() *mockNode_Expecter[CHAIN_ID, RPC] { return &mockNode_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Close() error { ret := _m.Called() @@ -40,7 +40,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Close() error { } // mockNode_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type mockNode_Close_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Close_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -66,7 +66,7 @@ func (_c *mockNode_Close_Call[CHAIN_ID, RPC]) RunAndReturn(run func() error) *mo return _c } -// ConfiguredChainID provides a mock function with given fields: +// ConfiguredChainID provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { ret := _m.Called() @@ -78,14 +78,16 @@ func (_m *mockNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { if rf, ok := ret.Get(0).(func() CHAIN_ID); ok { r0 = rf() } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } return r0 } // mockNode_ConfiguredChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConfiguredChainID' -type mockNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +113,7 @@ func (_c *mockNode_ConfiguredChainID_Call[CHAIN_ID, RPC]) RunAndReturn(run func( return _c } -// HighestUserObservations provides a mock function with given fields: +// HighestUserObservations provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { ret := _m.Called() @@ -130,7 +132,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { } // mockNode_HighestUserObservations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HighestUserObservations' -type mockNode_HighestUserObservations_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_HighestUserObservations_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -156,7 +158,7 @@ func (_c *mockNode_HighestUserObservations_Call[CHAIN_ID, RPC]) RunAndReturn(run return _c } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -175,7 +177,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Name() string { } // mockNode_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockNode_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -201,7 +203,7 @@ func (_c *mockNode_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() string) *mo return _c } -// Order provides a mock function with given fields: +// Order provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Order() int32 { ret := _m.Called() @@ -220,7 +222,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Order() int32 { } // mockNode_Order_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Order' -type mockNode_Order_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Order_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -246,7 +248,7 @@ func (_c *mockNode_Order_Call[CHAIN_ID, RPC]) RunAndReturn(run func() int32) *mo return _c } -// RPC provides a mock function with given fields: +// RPC provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) RPC() RPC { ret := _m.Called() @@ -258,14 +260,16 @@ func (_m *mockNode[CHAIN_ID, RPC]) RPC() RPC { if rf, ok := ret.Get(0).(func() RPC); ok { r0 = rf() } else { - r0 = ret.Get(0).(RPC) + if ret.Get(0) != nil { + r0 = ret.Get(0).(RPC) + } } return r0 } // mockNode_RPC_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RPC' -type mockNode_RPC_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_RPC_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -297,7 +301,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) SetPoolChainInfoProvider(_a0 PoolChainInfoPro } // mockNode_SetPoolChainInfoProvider_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetPoolChainInfoProvider' -type mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -320,7 +324,7 @@ func (_c *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC]) Return() *mockN } func (_c *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC]) RunAndReturn(run func(PoolChainInfoProvider)) *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC] { - _c.Call.Return(run) + _c.Run(run) return _c } @@ -343,7 +347,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Start(_a0 context.Context) error { } // mockNode_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type mockNode_Start_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Start_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -370,7 +374,7 @@ func (_c *mockNode_Start_Call[CHAIN_ID, RPC]) RunAndReturn(run func(context.Cont return _c } -// State provides a mock function with given fields: +// State provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) State() nodeState { ret := _m.Called() @@ -389,7 +393,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) State() nodeState { } // mockNode_State_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'State' -type mockNode_State_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_State_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -415,7 +419,7 @@ func (_c *mockNode_State_Call[CHAIN_ID, RPC]) RunAndReturn(run func() nodeState) return _c } -// StateAndLatest provides a mock function with given fields: +// StateAndLatest provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) StateAndLatest() (nodeState, ChainInfo) { ret := _m.Called() @@ -444,7 +448,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) StateAndLatest() (nodeState, ChainInfo) { } // mockNode_StateAndLatest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StateAndLatest' -type mockNode_StateAndLatest_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_StateAndLatest_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -470,7 +474,7 @@ func (_c *mockNode_StateAndLatest_Call[CHAIN_ID, RPC]) RunAndReturn(run func() ( return _c } -// String provides a mock function with given fields: +// String provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) String() string { ret := _m.Called() @@ -489,7 +493,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) String() string { } // mockNode_String_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'String' -type mockNode_String_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_String_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -515,13 +519,13 @@ func (_c *mockNode_String_Call[CHAIN_ID, RPC]) RunAndReturn(run func() string) * return _c } -// UnsubscribeAllExceptAliveLoop provides a mock function with given fields: +// UnsubscribeAllExceptAliveLoop provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) UnsubscribeAllExceptAliveLoop() { _m.Called() } // mockNode_UnsubscribeAllExceptAliveLoop_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnsubscribeAllExceptAliveLoop' -type mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -543,13 +547,13 @@ func (_c *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC]) Return() * } func (_c *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC]) RunAndReturn(run func()) *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC] { - _c.Call.Return(run) + _c.Run(run) return _c } // newMockNode creates a new instance of mockNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockNode[CHAIN_ID ID, RPC any](t interface { +func newMockNode[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockNode[CHAIN_ID, RPC] { diff --git a/multinode/mock_pool_chain_info_provider_test.go b/multinode/mock_pool_chain_info_provider_test.go index c857ef8..3dcd76c 100644 --- a/multinode/mock_pool_chain_info_provider_test.go +++ b/multinode/mock_pool_chain_info_provider_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -17,7 +17,7 @@ func (_m *mockPoolChainInfoProvider) EXPECT() *mockPoolChainInfoProvider_Expecte return &mockPoolChainInfoProvider_Expecter{mock: &_m.Mock} } -// HighestUserObservations provides a mock function with given fields: +// HighestUserObservations provides a mock function with no fields func (_m *mockPoolChainInfoProvider) HighestUserObservations() ChainInfo { ret := _m.Called() @@ -62,7 +62,7 @@ func (_c *mockPoolChainInfoProvider_HighestUserObservations_Call) RunAndReturn(r return _c } -// LatestChainInfo provides a mock function with given fields: +// LatestChainInfo provides a mock function with no fields func (_m *mockPoolChainInfoProvider) LatestChainInfo() (int, ChainInfo) { ret := _m.Called() diff --git a/multinode/mock_rpc_client_test.go b/multinode/mock_rpc_client_test.go index 0dbdc1a..6b55d4c 100644 --- a/multinode/mock_rpc_client_test.go +++ b/multinode/mock_rpc_client_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -37,7 +37,9 @@ func (_m *mockRPCClient[CHAIN_ID, HEAD]) ChainID(ctx context.Context) (CHAIN_ID, if rf, ok := ret.Get(0).(func(context.Context) CHAIN_ID); ok { r0 = rf(ctx) } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } if rf, ok := ret.Get(1).(func(context.Context) error); ok { @@ -77,7 +79,7 @@ func (_c *mockRPCClient_ChainID_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(cont return _c } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockRPCClient[CHAIN_ID, HEAD]) Close() { _m.Called() } @@ -105,7 +107,7 @@ func (_c *mockRPCClient_Close_Call[CHAIN_ID, HEAD]) Return() *mockRPCClient_Clos } func (_c *mockRPCClient_Close_Call[CHAIN_ID, HEAD]) RunAndReturn(run func()) *mockRPCClient_Close_Call[CHAIN_ID, HEAD] { - _c.Call.Return(run) + _c.Run(run) return _c } @@ -155,7 +157,7 @@ func (_c *mockRPCClient_Dial_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(context return _c } -// GetInterceptedChainInfo provides a mock function with given fields: +// GetInterceptedChainInfo provides a mock function with no fields func (_m *mockRPCClient[CHAIN_ID, HEAD]) GetInterceptedChainInfo() (ChainInfo, ChainInfo) { ret := _m.Called() @@ -488,7 +490,7 @@ func (_c *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD]) Return() *moc } func (_c *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(...Subscription)) *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD] { - _c.Call.Return(run) + _c.Run(run) return _c } diff --git a/multinode/mock_send_only_client_test.go b/multinode/mock_send_only_client_test.go index 5c08506..2360b53 100644 --- a/multinode/mock_send_only_client_test.go +++ b/multinode/mock_send_only_client_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -37,7 +37,9 @@ func (_m *mockSendOnlyClient[CHAIN_ID]) ChainID(_a0 context.Context) (CHAIN_ID, if rf, ok := ret.Get(0).(func(context.Context) CHAIN_ID); ok { r0 = rf(_a0) } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } if rf, ok := ret.Get(1).(func(context.Context) error); ok { @@ -77,7 +79,7 @@ func (_c *mockSendOnlyClient_ChainID_Call[CHAIN_ID]) RunAndReturn(run func(conte return _c } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockSendOnlyClient[CHAIN_ID]) Close() { _m.Called() } @@ -105,7 +107,7 @@ func (_c *mockSendOnlyClient_Close_Call[CHAIN_ID]) Return() *mockSendOnlyClient_ } func (_c *mockSendOnlyClient_Close_Call[CHAIN_ID]) RunAndReturn(run func()) *mockSendOnlyClient_Close_Call[CHAIN_ID] { - _c.Call.Return(run) + _c.Run(run) return _c } diff --git a/multinode/mock_send_only_node_test.go b/multinode/mock_send_only_node_test.go index e76b053..45d55c9 100644 --- a/multinode/mock_send_only_node_test.go +++ b/multinode/mock_send_only_node_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -9,11 +9,11 @@ import ( ) // mockSendOnlyNode is an autogenerated mock type for the SendOnlyNode type -type mockSendOnlyNode[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockSendOnlyNode_Expecter[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -21,7 +21,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) EXPECT() *mockSendOnlyNode_Expecter[C return &mockSendOnlyNode_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Close() error { ret := _m.Called() @@ -40,7 +40,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Close() error { } // mockSendOnlyNode_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type mockSendOnlyNode_Close_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Close_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -66,7 +66,7 @@ func (_c *mockSendOnlyNode_Close_Call[CHAIN_ID, RPC]) RunAndReturn(run func() er return _c } -// ConfiguredChainID provides a mock function with given fields: +// ConfiguredChainID provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { ret := _m.Called() @@ -78,14 +78,16 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { if rf, ok := ret.Get(0).(func() CHAIN_ID); ok { r0 = rf() } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } return r0 } // mockSendOnlyNode_ConfiguredChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConfiguredChainID' -type mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +113,7 @@ func (_c *mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID, RPC]) RunAndReturn(r return _c } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -130,7 +132,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Name() string { } // mockSendOnlyNode_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockSendOnlyNode_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -156,7 +158,7 @@ func (_c *mockSendOnlyNode_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() str return _c } -// RPC provides a mock function with given fields: +// RPC provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { ret := _m.Called() @@ -168,14 +170,16 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { if rf, ok := ret.Get(0).(func() RPC); ok { r0 = rf() } else { - r0 = ret.Get(0).(RPC) + if ret.Get(0) != nil { + r0 = ret.Get(0).(RPC) + } } return r0 } // mockSendOnlyNode_RPC_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RPC' -type mockSendOnlyNode_RPC_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_RPC_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -220,7 +224,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Start(_a0 context.Context) error { } // mockSendOnlyNode_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type mockSendOnlyNode_Start_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Start_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -247,7 +251,7 @@ func (_c *mockSendOnlyNode_Start_Call[CHAIN_ID, RPC]) RunAndReturn(run func(cont return _c } -// State provides a mock function with given fields: +// State provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) State() nodeState { ret := _m.Called() @@ -266,7 +270,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) State() nodeState { } // mockSendOnlyNode_State_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'State' -type mockSendOnlyNode_State_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_State_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -292,7 +296,7 @@ func (_c *mockSendOnlyNode_State_Call[CHAIN_ID, RPC]) RunAndReturn(run func() no return _c } -// String provides a mock function with given fields: +// String provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) String() string { ret := _m.Called() @@ -311,7 +315,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) String() string { } // mockSendOnlyNode_String_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'String' -type mockSendOnlyNode_String_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_String_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -339,7 +343,7 @@ func (_c *mockSendOnlyNode_String_Call[CHAIN_ID, RPC]) RunAndReturn(run func() s // newMockSendOnlyNode creates a new instance of mockSendOnlyNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockSendOnlyNode[CHAIN_ID ID, RPC any](t interface { +func newMockSendOnlyNode[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockSendOnlyNode[CHAIN_ID, RPC] { diff --git a/multinode/mock_subscription_test.go b/multinode/mock_subscription_test.go index ccb9017..1973c92 100644 --- a/multinode/mock_subscription_test.go +++ b/multinode/mock_subscription_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -17,7 +17,7 @@ func (_m *mockSubscription) EXPECT() *mockSubscription_Expecter { return &mockSubscription_Expecter{mock: &_m.Mock} } -// Err provides a mock function with given fields: +// Err provides a mock function with no fields func (_m *mockSubscription) Err() <-chan error { ret := _m.Called() @@ -64,7 +64,7 @@ func (_c *mockSubscription_Err_Call) RunAndReturn(run func() <-chan error) *mock return _c } -// Unsubscribe provides a mock function with given fields: +// Unsubscribe provides a mock function with no fields func (_m *mockSubscription) Unsubscribe() { _m.Called() } @@ -92,7 +92,7 @@ func (_c *mockSubscription_Unsubscribe_Call) Return() *mockSubscription_Unsubscr } func (_c *mockSubscription_Unsubscribe_Call) RunAndReturn(run func()) *mockSubscription_Unsubscribe_Call { - _c.Call.Return(run) + _c.Run(run) return _c } diff --git a/multinode/mock_transaction_sender_metrics_test.go b/multinode/mock_transaction_sender_metrics_test.go new file mode 100644 index 0000000..572e9ea --- /dev/null +++ b/multinode/mock_transaction_sender_metrics_test.go @@ -0,0 +1,70 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockTransactionSenderMetrics is an autogenerated mock type for the transactionSenderMetrics type +type mockTransactionSenderMetrics struct { + mock.Mock +} + +type mockTransactionSenderMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockTransactionSenderMetrics) EXPECT() *mockTransactionSenderMetrics_Expecter { + return &mockTransactionSenderMetrics_Expecter{mock: &_m.Mock} +} + +// IncrementInvariantViolations provides a mock function with given fields: ctx, invariant +func (_m *mockTransactionSenderMetrics) IncrementInvariantViolations(ctx context.Context, invariant string) { + _m.Called(ctx, invariant) +} + +// mockTransactionSenderMetrics_IncrementInvariantViolations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementInvariantViolations' +type mockTransactionSenderMetrics_IncrementInvariantViolations_Call struct { + *mock.Call +} + +// IncrementInvariantViolations is a helper method to define mock.On call +// - ctx context.Context +// - invariant string +func (_e *mockTransactionSenderMetrics_Expecter) IncrementInvariantViolations(ctx interface{}, invariant interface{}) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + return &mockTransactionSenderMetrics_IncrementInvariantViolations_Call{Call: _e.mock.On("IncrementInvariantViolations", ctx, invariant)} +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) Run(run func(ctx context.Context, invariant string)) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) Return() *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Call.Return() + return _c +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) RunAndReturn(run func(context.Context, string)) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Run(run) + return _c +} + +// newMockTransactionSenderMetrics creates a new instance of mockTransactionSenderMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockTransactionSenderMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockTransactionSenderMetrics { + mock := &mockTransactionSenderMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 9c3af89..c350953 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -8,21 +8,15 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) -var ( - // PromMultiNodeRPCNodeStates reports current RPC node state - PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", - Help: "The number of RPC nodes currently in the given state for the given chain", - }, []string{"network", "chainId", "state"}) - ErrNodeError = fmt.Errorf("no live nodes available") -) +var ErrNodeError = fmt.Errorf("no live nodes available") + +type multiNodeMetrics interface { + RecordNodeStates(ctx context.Context, state string, count int64) +} // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. @@ -37,6 +31,7 @@ type MultiNode[ sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC] chainID CHAIN_ID lggr logger.SugaredLogger + metrics multiNodeMetrics selectionMode string nodeSelector NodeSelector[CHAIN_ID, RPC] leaseDuration time.Duration @@ -54,6 +49,7 @@ func NewMultiNode[ RPC any, ]( lggr logger.Logger, + metrics multiNodeMetrics, selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected primaryNodes []Node[CHAIN_ID, RPC], @@ -67,6 +63,7 @@ func NewMultiNode[ // aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) const reportInterval = 6500 * time.Millisecond c := &MultiNode[CHAIN_ID, RPC]{ + metrics: metrics, primaryNodes: primaryNodes, sendOnlyNodes: sendOnlyNodes, chainID: chainID, @@ -361,9 +358,12 @@ func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) { dead++ } } + + ctx, cancel := c.eng.NewCtx() + defer cancel() for _, state := range allNodeStates { - count := counts[state] - PromMultiNodeRPCNodeStates.WithLabelValues(c.chainFamily, c.chainID.String(), state.String()).Set(float64(count)) + count := int64(counts[state]) + c.metrics.RecordNodeStates(ctx, state.String(), count) } total := len(c.primaryNodes) diff --git a/multinode/multi_node_test.go b/multinode/multi_node_test.go index 6c0e659..89b54f1 100644 --- a/multinode/multi_node_test.go +++ b/multinode/multi_node_test.go @@ -41,12 +41,18 @@ func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode { } result := NewMultiNode[ID, multiNodeRPCClient]( - opts.logger, opts.selectionMode, opts.leaseDuration, opts.nodes, opts.sendonlys, opts.chainID, opts.chainFamily, opts.deathDeclarationDelay) + opts.logger, makeMockMultiNodeMetrics(t), opts.selectionMode, opts.leaseDuration, opts.nodes, opts.sendonlys, opts.chainID, opts.chainFamily, opts.deathDeclarationDelay) return testMultiNode{ result, } } +func makeMockMultiNodeMetrics(t *testing.T) *mockMultiNodeMetrics { + mockMetrics := newMockMultiNodeMetrics(t) + mockMetrics.On("RecordNodeStates", mock.Anything, mock.Anything, mock.Anything).Maybe() + return mockMetrics +} + func newHealthyNode(t *testing.T, chainID ID) *mockNode[ID, multiNodeRPCClient] { return newNodeWithState(t, chainID, nodeStateAlive) } diff --git a/multinode/node.go b/multinode/node.go index a457711..825e5b1 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -8,9 +8,6 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -19,21 +16,6 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") -var ( - promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", - Help: "The total number of chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", - Help: "The total number of failed chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", - Help: "The total number of successful chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) -) - type NodeConfig interface { PollFailureThreshold() uint32 PollInterval() time.Duration @@ -55,6 +37,19 @@ type ChainConfig interface { FinalizedBlockOffset() uint32 } +type nodeMetrics interface { + IncrementNodeVerifies(ctx context.Context, nodeName string) + IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) + IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) + IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) +} + type Node[ CHAIN_ID ID, RPC any, @@ -98,6 +93,8 @@ type node[ order int32 chainFamily string + metrics nodeMetrics + ws *url.URL http *url.URL @@ -123,6 +120,7 @@ func NewNode[ nodeCfg NodeConfig, chainCfg ChainConfig, lggr logger.Logger, + metrics nodeMetrics, wsuri *url.URL, httpuri *url.URL, name string, @@ -139,6 +137,7 @@ func NewNode[ n.nodePoolCfg = nodeCfg n.chainCfg = chainCfg n.order = nodeOrder + n.metrics = metrics if wsuri != nil { n.ws = wsuri } @@ -253,9 +252,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) start() { // Not thread-safe // Pure verifyChainID: does not mutate node "state" field. func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lggr logger.Logger) nodeState { - promPoolRPCNodeVerifies.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifies(callerCtx, n.name) promFailed := func() { - promPoolRPCNodeVerifiesFailed.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifiesFailed(callerCtx, n.name) } st := n.getCachedState() @@ -288,7 +287,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lgg return nodeStateInvalidChainID } - promPoolRPCNodeVerifiesSuccess.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifiesSuccess(callerCtx, n.name) return nodeStateAlive } diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index 5b76752..994036f 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -2,41 +2,9 @@ package multinode import ( "fmt" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" ) -var ( - promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", - Help: transitionString(nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", - Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", - Help: transitionString(nodeStateOutOfSync), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", - Help: transitionString(nodeStateUnreachable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", - Help: transitionString(nodeStateInvalidChainID), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", - Help: transitionString(nodeStateUnusable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", - Help: transitionString(nodeStateSyncing), - }, []string{"chainID", "nodeName"}) -) +var () // nodeState represents the current state of the node // Node is a FSM (finite state machine) @@ -207,7 +175,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareAlive() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToAlive(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -233,8 +203,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareInSync() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInSync(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() - promPoolRPCNodeTransitionsToInSync.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToAlive(ctx, n.name) + n.metrics.IncrementNodeTransitionsToInSync(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -260,7 +232,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareOutOfSync(syncIssues syncStatus) { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) { - promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToOutOfSync(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -285,7 +259,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareUnreachable() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToUnreachable(fn func()) { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToUnreachable(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -328,7 +304,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareInvalidChainID() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInvalidChainID(fn func()) { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -353,7 +331,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareSyncing() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { - promPoolRPCNodeTransitionsToSyncing.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToSyncing(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -373,10 +353,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { fn() } -func transitionString(state nodeState) string { - return fmt.Sprintf("Total number of times node has transitioned to %s", state) -} - func transitionFail(from nodeState, to nodeState) string { return fmt.Sprintf("cannot transition from %#v to %#v", from, to) } diff --git a/multinode/node_test.go b/multinode/node_test.go index a6a7f2e..b1ff8f8 100644 --- a/multinode/node_test.go +++ b/multinode/node_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/mock" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-framework/multinode/mocks" @@ -101,10 +103,25 @@ func newTestNode(t *testing.T, opts testNodeOpts) testNode { opts.id = 42 } - nodeI := NewNode[ID, Head, RPCClient[ID, Head]](opts.config, opts.chainConfig, opts.lggr, + nodeI := NewNode[ID, Head, RPCClient[ID, Head]](opts.config, opts.chainConfig, opts.lggr, makeMockNodeMetrics(t), opts.wsuri, opts.httpuri, opts.name, opts.id, opts.chainID, opts.nodeOrder, opts.rpc, opts.chainFamily) return testNode{ nodeI.(*node[ID, Head, RPCClient[ID, Head]]), } } + +func makeMockNodeMetrics(t *testing.T) *mockNodeMetrics { + mockMetrics := newMockNodeMetrics(t) + mockMetrics.On("IncrementNodeVerifies", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeVerifiesFailed", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeVerifiesSuccess", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToAlive", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToInSync", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToOutOfSync", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToUnreachable", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToInvalidChainID", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToUnusable", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToSyncing", mock.Anything, mock.Anything).Maybe() + return mockMetrics +} diff --git a/multinode/send_only_node.go b/multinode/send_only_node.go index 7630362..c61a51c 100644 --- a/multinode/send_only_node.go +++ b/multinode/send_only_node.go @@ -45,6 +45,7 @@ type sendOnlyNode[ ] struct { services.StateMachine + metrics nodeMetrics stateMu sync.RWMutex // protects state* fields state nodeState @@ -63,6 +64,7 @@ func NewSendOnlyNode[ RPC sendOnlyClient[CHAIN_ID], ]( lggr logger.Logger, + metrics nodeMetrics, httpuri url.URL, name string, chainID CHAIN_ID, @@ -74,6 +76,7 @@ func NewSendOnlyNode[ s.log = logger.With(s.log, "nodeTier", "sendonly", ) + s.metrics = metrics s.rpc = rpc s.uri = httpuri s.chainID = chainID @@ -101,7 +104,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { err := s.rpc.Dial(ctx) if err != nil { - promPoolRPCNodeTransitionsToUnusable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnusable(ctx, s.name) s.log.Errorw("Dial failed: SendOnly Node is unusable", "err", err) s.setState(nodeStateUnusable) return @@ -114,13 +117,13 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { } else { chainID, err := s.rpc.ChainID(ctx) if err != nil || chainID.String() != s.chainID.String() { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) if err != nil { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) s.setState(nodeStateUnreachable) } else { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, s.name) s.log.Errorf( "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", chainID.String(), @@ -137,7 +140,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { } } - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToAlive(ctx, s.name) s.setState(nodeStateAlive) s.log.Infow("Sendonly RPC Node is online", "nodeState", s.state) } diff --git a/multinode/send_only_node_lifecycle.go b/multinode/send_only_node_lifecycle.go index fe88465..89b3d12 100644 --- a/multinode/send_only_node_lifecycle.go +++ b/multinode/send_only_node_lifecycle.go @@ -25,7 +25,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { if err != nil { ok := s.IfStarted(func() { if changed := s.setState(nodeStateUnreachable); changed { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) } }) if !ok { @@ -36,7 +36,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { } else if chainID.String() != s.chainID.String() { ok := s.IfStarted(func() { if changed := s.setState(nodeStateInvalidChainID); changed { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, s.name) } }) if !ok { @@ -53,7 +53,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { } ok := s.IfStarted(func() { if changed := s.setState(nodeStateAlive); changed { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToAlive(ctx, s.name) } }) if !ok { diff --git a/multinode/send_only_node_test.go b/multinode/send_only_node_test.go index 646796e..8d01953 100644 --- a/multinode/send_only_node_test.go +++ b/multinode/send_only_node_test.go @@ -28,7 +28,7 @@ func TestNewSendOnlyNode(t *testing.T) { chainID := RandomID() client := newMockSendOnlyClient[ID](t) - node := NewSendOnlyNode(lggr, *u, name, chainID, client) + node := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), *u, name, chainID, client) assert.NotNil(t, node) // Must contain name & url with redacted password @@ -45,7 +45,7 @@ func TestStartSendOnlyNode(t *testing.T) { client.On("Close").Once() expectedError := errors.New("some http error") client.On("Dial", mock.Anything).Return(expectedError).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), RandomID(), client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), RandomID(), client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -61,7 +61,7 @@ func TestStartSendOnlyNode(t *testing.T) { client := newMockSendOnlyClient[ID](t) client.On("Close").Once() client.On("Dial", mock.Anything).Return(nil).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), NewIDFromInt(0), client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), NewIDFromInt(0), client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -82,7 +82,7 @@ func TestStartSendOnlyNode(t *testing.T) { const failuresCount = 2 client.On("ChainID", mock.Anything).Return(RandomID(), expectedError).Times(failuresCount) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), chainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), chainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -105,7 +105,7 @@ func TestStartSendOnlyNode(t *testing.T) { const failuresCount = 2 client.On("ChainID", mock.Anything).Return(rpcChainID, nil).Times(failuresCount) client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), configuredChainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -126,7 +126,7 @@ func TestStartSendOnlyNode(t *testing.T) { client.On("Dial", mock.Anything).Return(nil).Once() configuredChainID := RandomID() client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), configuredChainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index 43e6fda..7be3f23 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -8,21 +8,10 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) -var ( - // PromMultiNodeInvariantViolations reports violation of our assumptions - PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", - Help: "The number of invariant violations", - }, []string{"network", "chainId", "invariant"}) -) - type sendTxResult[RESULT any] struct { res RESULT code SendTxReturnCode @@ -37,11 +26,16 @@ type SendTxRPCClient[TX any, RESULT any] interface { SendTransaction(ctx context.Context, tx TX) (RESULT, SendTxReturnCode, error) } +type transactionSenderMetrics interface { + IncrementInvariantViolations(ctx context.Context, invariant string) +} + func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, multiNode *MultiNode[CHAIN_ID, RPC], + metrics transactionSenderMetrics, classifyErr func(err error) SendTxReturnCode, sendTxSoftTimeout time.Duration, ) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { @@ -52,6 +46,7 @@ func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[T chainID: chainID, chainFamily: chainFamily, lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), + metrics: metrics, multiNode: multiNode, classifyErr: classifyErr, sendTxSoftTimeout: sendTxSoftTimeout, @@ -64,6 +59,7 @@ type TransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, chainID CHAIN_ID chainFamily string lggr logger.SugaredLogger + metrics transactionSenderMetrics multiNode *MultiNode[CHAIN_ID, RPC] classifyErr func(err error) SendTxReturnCode sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation @@ -199,7 +195,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal _, criticalErr := aggregateTxResults(resultsByCode) if criticalErr != nil { txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) - PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() + ctx, cancel := txSender.chStop.NewCtx() + defer cancel() + txSender.metrics.IncrementInvariantViolations(ctx, criticalErr.Error()) } } diff --git a/multinode/transaction_sender_test.go b/multinode/transaction_sender_test.go index 8f88dd1..a78c24e 100644 --- a/multinode/transaction_sender_test.go +++ b/multinode/transaction_sender_test.go @@ -47,13 +47,20 @@ func newTestTransactionSender(t *testing.T, chainID ID, lggr logger.Logger, sendOnlyNodes []SendOnlyNode[ID, TestSendTxRPCClient], ) (*sendTxMultiNode, *TransactionSender[any, any, ID, TestSendTxRPCClient]) { mn := sendTxMultiNode{NewMultiNode[ID, TestSendTxRPCClient]( - lggr, NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} + lggr, makeMockMultiNodeMetrics(t), NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} - txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) + txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, makeMockTxSenderMetrics(t), + func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) servicetest.Run(t, txSender) return &mn, txSender } +func makeMockTxSenderMetrics(t *testing.T) *mockTransactionSenderMetrics { + metrics := newMockTransactionSenderMetrics(t) + metrics.On("IncrementInvariantViolations", mock.Anything, mock.Anything).Maybe() + return metrics +} + func classifySendTxError(_ any, err error) SendTxReturnCode { if err != nil { return Fatal @@ -148,8 +155,9 @@ func TestTransactionSender_SendTransaction(t *testing.T) { chainID := RandomID() mn := sendTxMultiNode{NewMultiNode[ID, TestSendTxRPCClient]( - lggr, NodeSelectionModeRoundRobin, 0, []Node[ID, TestSendTxRPCClient]{mainNode}, nil, chainID, "chainFamily", 0)} - txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) + lggr, makeMockMultiNodeMetrics(t), NodeSelectionModeRoundRobin, 0, []Node[ID, TestSendTxRPCClient]{mainNode}, nil, chainID, "chainFamily", 0)} + txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, makeMockTxSenderMetrics(t), + func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) require.NoError(t, txSender.Start(tests.Context(t))) _, _, err := txSender.SendTransaction(requestContext, nil)