From f23d3673a7f3152542ce2bdf804c4a39c0663b62 Mon Sep 17 00:00:00 2001 From: amit-momin Date: Tue, 1 Apr 2025 17:45:48 -0500 Subject: [PATCH 01/16] Centralized TXM metrics and integrated beholder --- chains/go.mod | 38 ++++++-- chains/go.sum | 90 ++++++++++++++----- chains/txmgr/broadcaster.go | 34 +++---- chains/txmgr/confirmer.go | 74 ++++------------ chains/txmgr/metrics.go | 171 ++++++++++++++++++++++++++++++++++++ 5 files changed, 303 insertions(+), 104 deletions(-) create mode 100644 chains/txmgr/metrics.go diff --git a/chains/go.mod b/chains/go.mod index bf96cc1..3168af0 100644 --- a/chains/go.mod +++ b/chains/go.mod @@ -1,27 +1,34 @@ module github.com/smartcontractkit/chainlink-framework/chains -go 1.23.3 +go 1.24 require ( github.com/google/uuid v1.6.0 github.com/jpillora/backoff v1.0.0 github.com/prometheus/client_golang v1.20.5 github.com/shopspring/decimal v1.4.0 - github.com/smartcontractkit/chainlink-common v0.4.0 + github.com/smartcontractkit/chainlink-common v0.6.0 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/otel/metric v1.31.0 go.uber.org/multierr v1.11.0 gopkg.in/guregu/null.v4 v4.0.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/leodido/go-urn v1.2.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -31,16 +38,33 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 // indirect + github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.29.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/chains/go.sum b/chains/go.sum index 9b260de..38430f2 100644 --- a/chains/go.sum +++ b/chains/go.sum @@ -2,6 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cometbft/cometbft v0.37.5 h1:/U/TlgMh4NdnXNo+YU9T2NMCWyhXNDF34Mx582jlvq0= @@ -10,21 +12,27 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= -github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= -github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= @@ -35,6 +43,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -60,64 +70,104 @@ github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPA github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/smartcontractkit/chainlink-common v0.4.0 h1:GZ9MhHt5QHXSaK/sAZvKDxkEqF4fPiFHWHEPqs/2C2o= -github.com/smartcontractkit/chainlink-common v0.4.0/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ= +github.com/smartcontractkit/chainlink-common v0.6.0 h1:6S9VGl8ObCu9d4zEV/UulKGPL751D0pEmi9CHzPVOaI= +github.com/smartcontractkit/chainlink-common v0.6.0/go.mod h1:ASXpANdCfcKd+LF3Vhz37q4rmJ/XYQKEQ3La1k7idp0= github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 h1:See2isL6KdrTJDlVKWv8qiyYqWhYUcubU2e5yKXV1oY= github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= -github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 h1:IpGoPTXpvllN38kT2z2j13sifJMz4nbHglidvop7mfg= -github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 h1:UiRNKd1OgqsLbFwE+wkAWTdiAxXtCBqKIHeBIse4FUA= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9/go.mod h1:eqZlW3pJWhjyexnDPrdQxix1pn0wwhI4AO4GKpP/bMI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 h1:U2guen0GhqH8o/G2un8f/aG/y++OuW6MyCo6hT9prXk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0/go.mod h1:yeGZANgEcpdx/WK0IvvRFC+2oLiMS2u4L/0Rj2M2Qr0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index ef19310..824e531 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -9,8 +9,6 @@ import ( "time" "github.com/jpillora/backoff" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/multierr" "gopkg.in/guregu/null.v4" @@ -43,22 +41,6 @@ const ( hederaChainType = "hedera" ) -var ( - promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_broadcast", - Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - }, - }, []string{"chainID"}) -) - var ErrTxRemoved = errors.New("tx removed") type ProcessUnstartedTxs[ADDR chains.Hashable] func(ctx context.Context, fromAddress ADDR) (retryable bool, err error) @@ -79,6 +61,11 @@ type TransmitChecker[CID chains.ID, ADDR chains.Hashable, THASH, BHASH chains.Ha Check(ctx context.Context, l logger.SugaredLogger, tx types.Tx[CID, ADDR, THASH, BHASH, SEQ, FEE], a types.TxAttempt[CID, ADDR, THASH, BHASH, SEQ, FEE]) error } +type BroadcasterMetrics interface { + IncrementNumBroadcastedTxs(ctx context.Context) + RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) +} + // Broadcaster monitors txes for transactions that need to // be broadcast, assigns sequences and ensures that at least one node // somewhere has received the transaction successfully. @@ -106,6 +93,7 @@ type Broadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, T feeConfig types.BroadcasterFeeConfig txConfig types.BroadcasterTransactionsConfig listenerConfig types.BroadcasterListenerConfig + metrics BroadcasterMetrics // autoSyncSequence, if set, will cause Broadcaster to fast-forward the sequence // when Start is called @@ -144,6 +132,7 @@ func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable checkerFactory TransmitCheckerFactory[CID, ADDR, THASH, BHASH, SEQ, FEE], autoSyncSequence bool, chainType string, + metrics BroadcasterMetrics, ) *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE] { lggr = logger.Named(lggr, "Broadcaster") b := &Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]{ @@ -161,6 +150,7 @@ func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable checkerFactory: checkerFactory, autoSyncSequence: autoSyncSequence, sequenceTracker: sequenceTracker, + metrics: metrics, } b.processUnstartedTxsImpl = b.processUnstartedTxs @@ -532,11 +522,12 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress // In all scenarios, the correct thing to do is assume success for now // and hand off to the confirmer to get the receipt (or mark as // failed). - observeTimeUntilBroadcast(eb.chainID, etx.CreatedAt, time.Now()) + observeTimeUntilBroadcast(ctx, eb.metrics, etx.CreatedAt, time.Now()) err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, types.TxAttemptBroadcast) if err != nil { return err, true } + eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) return err, true @@ -601,6 +592,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress if err != nil { return err, true } + eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) return err, true @@ -755,7 +747,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) saveFatallyError return eb.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, etx) } -func observeTimeUntilBroadcast[CHAIN_ID chains.ID](chainID CHAIN_ID, createdAt, broadcastAt time.Time) { +func observeTimeUntilBroadcast(ctx context.Context, metrics BroadcasterMetrics, createdAt, broadcastAt time.Time) { duration := float64(broadcastAt.Sub(createdAt)) - promTimeUntilBroadcast.WithLabelValues(chainID.String()).Observe(duration) + metrics.RecordTimeUntilTxBroadcast(ctx, duration) } diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index 37fbfd5..55c9b82 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -10,8 +10,6 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/multierr" commonhex "github.com/smartcontractkit/chainlink-common/pkg/utils/hex" @@ -33,48 +31,13 @@ const ( processHeadTimeout = 10 * time.Minute ) -var ( - promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_gas_bumps", - Help: "Number of gas bumps", - }, []string{"chainID"}) - - promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_gas_bump_exceeds_limit", - Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", - }, []string{"chainID"}) - promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_confirmed_transactions", - Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", - }, []string{"chainID"}) - promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_confirmed", - Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - float64(5 * time.Minute), - float64(10 * time.Minute), - }, - }, []string{"chainID"}) - promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_blocks_until_tx_confirmed", - Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(1), - float64(5), - float64(10), - float64(20), - float64(50), - float64(100), - }, - }, []string{"chainID"}) -) +type ConfimerMetrics interface { + IncrementNumGasBumps(ctx context.Context) + IncrementGasBumpExceedsLimit(ctx context.Context) + IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) + RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) + RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) +} // Confirmer is a broad service which performs four different tasks in sequence on every new longest chain // Step 1: Mark that all currently pending transaction attempts were broadcast before this block @@ -95,6 +58,7 @@ type Confirmer[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, THA txConfig types.ConfirmerTransactionsConfig dbConfig types.ConfirmerDatabaseConfig chainID CID + metrics ConfimerMetrics ks types.KeyStore[ADDR] enabledAddresses []ADDR @@ -127,6 +91,7 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector types.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + metrics ConfimerMetrics, ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -143,6 +108,7 @@ func NewConfirmer[ mb: mailbox.NewSingle[HEAD](), isReceiptNil: isReceiptNil, stuckTxDetector: stuckTxDetector, + metrics: metrics, } } @@ -368,7 +334,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded return nil } // Add newly confirmed transactions to the prom metric - promNumConfirmedTxs.WithLabelValues(ec.chainID.String()).Add(float64(len(includedTxs))) + ec.metrics.IncrementNumConfirmedTxs(ctx, len(includedTxs)) purgeTxIDs := make([]int64, 0, len(includedTxs)) confirmedTxIDs := make([]int64, 0, len(includedTxs)) @@ -381,7 +347,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded continue } confirmedTxIDs = append(confirmedTxIDs, tx.ID) - observeUntilTxConfirmed(ec.chainID, tx.TxAttempts, head) + observeUntilTxConfirmed(ctx, ec.metrics, tx.TxAttempts, head) } // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message if err := ec.txStore.UpdateTxFatalError(ctx, purgeTxIDs, ec.stuckTxDetector.StuckTxFatalError()); err != nil { @@ -667,13 +633,13 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) bumpGas(ctx con // if no error, return attempt // if err, continue below if err == nil { - promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementNumGasBumps(ctx) ec.lggr.Debugw("Rebroadcast bumping fee for tx", append(logFields, "bumpedFee", bumpedFee.String(), "bumpedFeeLimit", bumpedFeeLimit)...) return bumpedAttempt, err } if errors.Is(err, fees.ErrBumpFeeExceedsLimit) { - promGasBumpExceedsLimit.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementGasBumpExceedsLimit(ctx) } return bumpedAttempt, fmt.Errorf("error bumping gas: %w", err) @@ -712,7 +678,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) handleInProgres if err != nil { return fmt.Errorf("could not bump gas for terminally underpriced transaction: %w", err) } - promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + ec.metrics.IncrementNumGasBumps(ctx) lggr.With( "sendError", sendError, "maxGasPriceConfig", ec.feeConfig.MaxFeePrice(), @@ -861,16 +827,14 @@ func observeUntilTxConfirmed[ TX_HASH, BLOCK_HASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee, -](chainID CHAIN_ID, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { +](ctx context.Context, metrics ConfimerMetrics, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { for _, attempt := range attempts { // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) // was created. We want to measure the amount of time taken from when a transaction is created // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts // were needed to achieve this. duration := time.Since(attempt.Tx.CreatedAt) - promTimeUntilTxConfirmed. - WithLabelValues(chainID.String()). - Observe(float64(duration)) + metrics.RecordTimeUntilTxConfirmed(ctx, float64(duration)) // Since a tx can have many attempts, we take the number of blocks to confirm as the block number // of the receipt minus the block number of the first ever broadcast for this transaction. @@ -882,9 +846,7 @@ func observeUntilTxConfirmed[ } if minBroadcastBefore > 0 { blocksElapsed := head.BlockNumber() - minBroadcastBefore - promBlocksUntilTxConfirmed. - WithLabelValues(chainID.String()). - Observe(float64(blocksElapsed)) + metrics.RecordBlocksUntilTxConfirmed(ctx, float64(blocksElapsed)) } } } diff --git a/chains/txmgr/metrics.go b/chains/txmgr/metrics.go new file mode 100644 index 0000000..0b55abc --- /dev/null +++ b/chains/txmgr/metrics.go @@ -0,0 +1,171 @@ +package txmgr + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/metrics" +) + +var ( + promNumBroadcasted = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_broadcasted", + Help: "The number of transactions broadcasted", + }, []string{"chainID"}) + promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_broadcast", + Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + }, + }, []string{"chainID"}) + promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_gas_bumps", + Help: "Number of gas bumps", + }, []string{"chainID"}) + + promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_gas_bump_exceeds_limit", + Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + float64(5 * time.Minute), + float64(10 * time.Minute), + }, + }, []string{"chainID"}) + promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_blocks_until_tx_confirmed", + Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(1), + float64(5), + float64(10), + float64(20), + float64(50), + float64(100), + }, + }, []string{"chainID"}) +) + +type txmMetrics struct { + metrics.Labeler + chainID *big.Int + numBroadcastedTxs metric.Int64Counter + timeUntilBroadcast metric.Float64Histogram + numGasBumps metric.Int64Counter + gasBumpExceedsLimit metric.Int64Counter + numConfirmedTxs metric.Int64Counter + timeUntilTxConfirmed metric.Float64Histogram + blocksUntilTxConfirmed metric.Float64Histogram +} + +func NewGenericTxmMetrics(chainID *big.Int) (*txmMetrics, error) { + numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") + if err != nil { + return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) + } + + timeUntilBroadcast, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_broadcast") + if err != nil { + return nil, fmt.Errorf("failed to register time until broadcast metric: %w", err) + } + + numGasBumps, err := beholder.GetMeter().Int64Counter("tx_manager_num_gas_bumps") + if err != nil { + return nil, fmt.Errorf("failed to register number of gas bumps metric: %w", err) + } + + gasBumpExceedsLimit, err := beholder.GetMeter().Int64Counter("tx_manager_gas_bump_exceeds_limit") + if err != nil { + return nil, fmt.Errorf("failed to register gas bump exceeds limit metric: %w", err) + } + + numConfirmedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_confirmed_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register confirmed txs number metric: %w", err) + } + + timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register time until tx confirmed metric: %w", err) + } + + blocksUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_blocks_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register blocks until tx confirmed metric: %w", err) + } + + return &txmMetrics{ + chainID: chainID, + Labeler: metrics.NewLabeler().With("chainID", chainID.String()), + numBroadcastedTxs: numBroadcastedTxs, + timeUntilBroadcast: timeUntilBroadcast, + numGasBumps: numGasBumps, + gasBumpExceedsLimit: gasBumpExceedsLimit, + numConfirmedTxs: numConfirmedTxs, + timeUntilTxConfirmed: timeUntilTxConfirmed, + blocksUntilTxConfirmed: blocksUntilTxConfirmed, + }, nil +} + +func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { + promNumBroadcasted.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1) +} + +func (m *txmMetrics) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) { + promTimeUntilBroadcast.WithLabelValues(m.chainID.String()).Observe(duration) + m.timeUntilBroadcast.Record(ctx, duration) +} + +func (m *txmMetrics) IncrementNumGasBumps(ctx context.Context) { + promNumGasBumps.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numGasBumps.Add(ctx, 1) +} + +func (m *txmMetrics) IncrementGasBumpExceedsLimit(ctx context.Context) { + promGasBumpExceedsLimit.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.gasBumpExceedsLimit.Add(ctx, 1) +} + +func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { + promNumConfirmedTxs.WithLabelValues(m.chainID.String()).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions)) +} + +func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { + promTimeUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration) +} + +func (m *txmMetrics) RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) { + promBlocksUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(blocksElapsed) + m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed) +} From 2868c2d56b61b6ba2e7446e39368f49f3b5df1b4 Mon Sep 17 00:00:00 2001 From: amit-momin Date: Wed, 2 Apr 2025 10:51:40 -0500 Subject: [PATCH 02/16] Renamed types and added metric attributes --- chains/txmgr/broadcaster.go | 8 +++---- chains/txmgr/confirmer.go | 8 +++---- chains/txmgr/metrics.go | 47 +++++++++++++++++++++---------------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 824e531..305cccf 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -61,7 +61,7 @@ type TransmitChecker[CID chains.ID, ADDR chains.Hashable, THASH, BHASH chains.Ha Check(ctx context.Context, l logger.SugaredLogger, tx types.Tx[CID, ADDR, THASH, BHASH, SEQ, FEE], a types.TxAttempt[CID, ADDR, THASH, BHASH, SEQ, FEE]) error } -type BroadcasterMetrics interface { +type broadcasterMetrics interface { IncrementNumBroadcastedTxs(ctx context.Context) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) } @@ -93,7 +93,7 @@ type Broadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, T feeConfig types.BroadcasterFeeConfig txConfig types.BroadcasterTransactionsConfig listenerConfig types.BroadcasterListenerConfig - metrics BroadcasterMetrics + metrics broadcasterMetrics // autoSyncSequence, if set, will cause Broadcaster to fast-forward the sequence // when Start is called @@ -132,7 +132,7 @@ func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable checkerFactory TransmitCheckerFactory[CID, ADDR, THASH, BHASH, SEQ, FEE], autoSyncSequence bool, chainType string, - metrics BroadcasterMetrics, + metrics broadcasterMetrics, ) *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE] { lggr = logger.Named(lggr, "Broadcaster") b := &Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]{ @@ -747,7 +747,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) saveFatallyError return eb.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, etx) } -func observeTimeUntilBroadcast(ctx context.Context, metrics BroadcasterMetrics, createdAt, broadcastAt time.Time) { +func observeTimeUntilBroadcast(ctx context.Context, metrics broadcasterMetrics, createdAt, broadcastAt time.Time) { duration := float64(broadcastAt.Sub(createdAt)) metrics.RecordTimeUntilTxBroadcast(ctx, duration) } diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index 55c9b82..2fae96a 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -31,7 +31,7 @@ const ( processHeadTimeout = 10 * time.Minute ) -type ConfimerMetrics interface { +type confimerMetrics interface { IncrementNumGasBumps(ctx context.Context) IncrementGasBumpExceedsLimit(ctx context.Context) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) @@ -58,7 +58,7 @@ type Confirmer[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, THA txConfig types.ConfirmerTransactionsConfig dbConfig types.ConfirmerDatabaseConfig chainID CID - metrics ConfimerMetrics + metrics confimerMetrics ks types.KeyStore[ADDR] enabledAddresses []ADDR @@ -91,7 +91,7 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector types.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - metrics ConfimerMetrics, + metrics confimerMetrics, ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -827,7 +827,7 @@ func observeUntilTxConfirmed[ TX_HASH, BLOCK_HASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee, -](ctx context.Context, metrics ConfimerMetrics, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { +](ctx context.Context, metrics confimerMetrics, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { for _, attempt := range attempts { // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) // was created. We want to measure the amount of time taken from when a transaction is created diff --git a/chains/txmgr/metrics.go b/chains/txmgr/metrics.go index 0b55abc..5a2ec61 100644 --- a/chains/txmgr/metrics.go +++ b/chains/txmgr/metrics.go @@ -3,15 +3,14 @@ package txmgr import ( "context" "fmt" - "math/big" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink-common/pkg/metrics" ) var ( @@ -74,9 +73,18 @@ var ( }, []string{"chainID"}) ) +type GenericTXMMetrics interface { + IncrementNumBroadcastedTxs(ctx context.Context) + RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) + IncrementNumGasBumps(ctx context.Context) + IncrementGasBumpExceedsLimit(ctx context.Context) + IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) + RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) + RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) +} + type txmMetrics struct { - metrics.Labeler - chainID *big.Int + chainID string numBroadcastedTxs metric.Int64Counter timeUntilBroadcast metric.Float64Histogram numGasBumps metric.Int64Counter @@ -86,7 +94,7 @@ type txmMetrics struct { blocksUntilTxConfirmed metric.Float64Histogram } -func NewGenericTxmMetrics(chainID *big.Int) (*txmMetrics, error) { +func NewGenericTxmMetrics(chainID string) (*txmMetrics, error) { numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") if err != nil { return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) @@ -124,7 +132,6 @@ func NewGenericTxmMetrics(chainID *big.Int) (*txmMetrics, error) { return &txmMetrics{ chainID: chainID, - Labeler: metrics.NewLabeler().With("chainID", chainID.String()), numBroadcastedTxs: numBroadcastedTxs, timeUntilBroadcast: timeUntilBroadcast, numGasBumps: numGasBumps, @@ -136,36 +143,36 @@ func NewGenericTxmMetrics(chainID *big.Int) (*txmMetrics, error) { } func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { - promNumBroadcasted.WithLabelValues(m.chainID.String()).Add(float64(1)) - m.numBroadcastedTxs.Add(ctx, 1) + promNumBroadcasted.WithLabelValues(m.chainID).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) { - promTimeUntilBroadcast.WithLabelValues(m.chainID.String()).Observe(duration) - m.timeUntilBroadcast.Record(ctx, duration) + promTimeUntilBroadcast.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilBroadcast.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) IncrementNumGasBumps(ctx context.Context) { - promNumGasBumps.WithLabelValues(m.chainID.String()).Add(float64(1)) - m.numGasBumps.Add(ctx, 1) + promNumGasBumps.WithLabelValues(m.chainID).Add(float64(1)) + m.numGasBumps.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) IncrementGasBumpExceedsLimit(ctx context.Context) { - promGasBumpExceedsLimit.WithLabelValues(m.chainID.String()).Add(float64(1)) - m.gasBumpExceedsLimit.Add(ctx, 1) + promGasBumpExceedsLimit.WithLabelValues(m.chainID).Add(float64(1)) + m.gasBumpExceedsLimit.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { - promNumConfirmedTxs.WithLabelValues(m.chainID.String()).Add(float64(confirmedTransactions)) - m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions)) + promNumConfirmedTxs.WithLabelValues(m.chainID).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions), metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { - promTimeUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(duration) - m.timeUntilTxConfirmed.Record(ctx, duration) + promTimeUntilTxConfirmed.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) } func (m *txmMetrics) RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) { - promBlocksUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(blocksElapsed) - m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed) + promBlocksUntilTxConfirmed.WithLabelValues(m.chainID).Observe(blocksElapsed) + m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed, metric.WithAttributes(attribute.String("chainID", m.chainID))) } From 89bf2a124462c1ad1e7d22a00ecaf59102f047dd Mon Sep 17 00:00:00 2001 From: amit-momin Date: Fri, 11 Apr 2025 11:42:06 -0500 Subject: [PATCH 03/16] Tidied go mod --- chains/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/go.mod b/chains/go.mod index 3168af0..933cf08 100644 --- a/chains/go.mod +++ b/chains/go.mod @@ -10,6 +10,7 @@ require ( github.com/smartcontractkit/chainlink-common v0.6.0 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/metric v1.31.0 go.uber.org/multierr v1.11.0 gopkg.in/guregu/null.v4 v4.0.0 @@ -39,7 +40,6 @@ require ( github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 // indirect From 8aa57dd65b3d12c5a6272d1e8d23ef0a84d980e3 Mon Sep 17 00:00:00 2001 From: amit-momin Date: Fri, 11 Apr 2025 16:45:44 -0500 Subject: [PATCH 04/16] Simplified time and block till confirmation observations --- chains/txmgr/confirmer.go | 47 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index 2fae96a..03ef56c 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -347,7 +347,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded continue } confirmedTxIDs = append(confirmedTxIDs, tx.ID) - observeUntilTxConfirmed(ctx, ec.metrics, tx.TxAttempts, head) + observeUntilTxConfirmed(ctx, ec.metrics, tx, head) } // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message if err := ec.txStore.UpdateTxFatalError(ctx, purgeTxIDs, ec.stuckTxDetector.StuckTxFatalError()); err != nil { @@ -819,34 +819,35 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) sendEmptyTransa return txhash, nil } -// observeUntilTxConfirmed observes the promBlocksUntilTxConfirmed metric for each confirmed -// transaction. +// observeUntilTxConfirmed observes the timeUntilTxConfirmed and blocksUntilTxConfirmed metrics for each confirmed transaction. func observeUntilTxConfirmed[ CHAIN_ID chains.ID, ADDR chains.Hashable, TX_HASH, BLOCK_HASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee, -](ctx context.Context, metrics confimerMetrics, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { - for _, attempt := range attempts { - // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) - // was created. We want to measure the amount of time taken from when a transaction is created - // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts - // were needed to achieve this. - duration := time.Since(attempt.Tx.CreatedAt) - metrics.RecordTimeUntilTxConfirmed(ctx, float64(duration)) - - // Since a tx can have many attempts, we take the number of blocks to confirm as the block number - // of the receipt minus the block number of the first ever broadcast for this transaction. - var minBroadcastBefore int64 - for _, a := range attempt.Tx.TxAttempts { - if b := a.BroadcastBeforeBlockNum; b != nil && *b < minBroadcastBefore { - minBroadcastBefore = *b - } - } - if minBroadcastBefore > 0 { - blocksElapsed := head.BlockNumber() - minBroadcastBefore - metrics.RecordBlocksUntilTxConfirmed(ctx, float64(blocksElapsed)) +](ctx context.Context, metrics confimerMetrics, tx *types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { + if tx == nil { + return + } + // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) + // was created. We want to measure the amount of time taken from when a transaction is created + // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts + // were needed to achieve this. + duration := time.Since(tx.CreatedAt) + metrics.RecordTimeUntilTxConfirmed(ctx, float64(duration)) + + // Since a tx can have many attempts, we take the number of blocks to confirm as the current block number + // minus the block number of the first ever broadcast for this transaction. + var minBroadcastBefore int64 + for _, a := range tx.TxAttempts { + if b := a.BroadcastBeforeBlockNum; b != nil && *b < minBroadcastBefore { + minBroadcastBefore = *b } } + + if minBroadcastBefore > 0 { + blocksElapsed := head.BlockNumber() - minBroadcastBefore + metrics.RecordBlocksUntilTxConfirmed(ctx, float64(blocksElapsed)) + } } From 3d8034a47bfe9638e240266348ebd9202ad9c3ef Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 10:47:37 -0400 Subject: [PATCH 05/16] Add beholder metrics --- chains/txmgr/metrics.go | 178 ---------------------------------------- metrics/go.mod | 56 +++++++++++-- metrics/go.sum | 139 ++++++++++++++++++++++++++++--- metrics/logpoller.go | 19 +++++ metrics/txm.go | 177 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 376 insertions(+), 193 deletions(-) delete mode 100644 chains/txmgr/metrics.go diff --git a/chains/txmgr/metrics.go b/chains/txmgr/metrics.go deleted file mode 100644 index 5a2ec61..0000000 --- a/chains/txmgr/metrics.go +++ /dev/null @@ -1,178 +0,0 @@ -package txmgr - -import ( - "context" - "fmt" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" - - "github.com/smartcontractkit/chainlink-common/pkg/beholder" -) - -var ( - promNumBroadcasted = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_broadcasted", - Help: "The number of transactions broadcasted", - }, []string{"chainID"}) - promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_broadcast", - Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - }, - }, []string{"chainID"}) - promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_gas_bumps", - Help: "Number of gas bumps", - }, []string{"chainID"}) - - promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_gas_bump_exceeds_limit", - Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", - }, []string{"chainID"}) - promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_confirmed_transactions", - Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", - }, []string{"chainID"}) - promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_time_until_tx_confirmed", - Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(500 * time.Millisecond), - float64(time.Second), - float64(5 * time.Second), - float64(15 * time.Second), - float64(30 * time.Second), - float64(time.Minute), - float64(2 * time.Minute), - float64(5 * time.Minute), - float64(10 * time.Minute), - }, - }, []string{"chainID"}) - promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tx_manager_blocks_until_tx_confirmed", - Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", - Buckets: []float64{ - float64(1), - float64(5), - float64(10), - float64(20), - float64(50), - float64(100), - }, - }, []string{"chainID"}) -) - -type GenericTXMMetrics interface { - IncrementNumBroadcastedTxs(ctx context.Context) - RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) - IncrementNumGasBumps(ctx context.Context) - IncrementGasBumpExceedsLimit(ctx context.Context) - IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) - RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) - RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) -} - -type txmMetrics struct { - chainID string - numBroadcastedTxs metric.Int64Counter - timeUntilBroadcast metric.Float64Histogram - numGasBumps metric.Int64Counter - gasBumpExceedsLimit metric.Int64Counter - numConfirmedTxs metric.Int64Counter - timeUntilTxConfirmed metric.Float64Histogram - blocksUntilTxConfirmed metric.Float64Histogram -} - -func NewGenericTxmMetrics(chainID string) (*txmMetrics, error) { - numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") - if err != nil { - return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) - } - - timeUntilBroadcast, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_broadcast") - if err != nil { - return nil, fmt.Errorf("failed to register time until broadcast metric: %w", err) - } - - numGasBumps, err := beholder.GetMeter().Int64Counter("tx_manager_num_gas_bumps") - if err != nil { - return nil, fmt.Errorf("failed to register number of gas bumps metric: %w", err) - } - - gasBumpExceedsLimit, err := beholder.GetMeter().Int64Counter("tx_manager_gas_bump_exceeds_limit") - if err != nil { - return nil, fmt.Errorf("failed to register gas bump exceeds limit metric: %w", err) - } - - numConfirmedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_confirmed_transactions") - if err != nil { - return nil, fmt.Errorf("failed to register confirmed txs number metric: %w", err) - } - - timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_confirmed") - if err != nil { - return nil, fmt.Errorf("failed to register time until tx confirmed metric: %w", err) - } - - blocksUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_blocks_until_tx_confirmed") - if err != nil { - return nil, fmt.Errorf("failed to register blocks until tx confirmed metric: %w", err) - } - - return &txmMetrics{ - chainID: chainID, - numBroadcastedTxs: numBroadcastedTxs, - timeUntilBroadcast: timeUntilBroadcast, - numGasBumps: numGasBumps, - gasBumpExceedsLimit: gasBumpExceedsLimit, - numConfirmedTxs: numConfirmedTxs, - timeUntilTxConfirmed: timeUntilTxConfirmed, - blocksUntilTxConfirmed: blocksUntilTxConfirmed, - }, nil -} - -func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { - promNumBroadcasted.WithLabelValues(m.chainID).Add(float64(1)) - m.numBroadcastedTxs.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) { - promTimeUntilBroadcast.WithLabelValues(m.chainID).Observe(duration) - m.timeUntilBroadcast.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) IncrementNumGasBumps(ctx context.Context) { - promNumGasBumps.WithLabelValues(m.chainID).Add(float64(1)) - m.numGasBumps.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) IncrementGasBumpExceedsLimit(ctx context.Context) { - promGasBumpExceedsLimit.WithLabelValues(m.chainID).Add(float64(1)) - m.gasBumpExceedsLimit.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { - promNumConfirmedTxs.WithLabelValues(m.chainID).Add(float64(confirmedTransactions)) - m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions), metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { - promTimeUntilTxConfirmed.WithLabelValues(m.chainID).Observe(duration) - m.timeUntilTxConfirmed.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} - -func (m *txmMetrics) RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) { - promBlocksUntilTxConfirmed.WithLabelValues(m.chainID).Observe(blocksElapsed) - m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed, metric.WithAttributes(attribute.String("chainID", m.chainID))) -} diff --git a/metrics/go.mod b/metrics/go.mod index 85b9778..c0a0b52 100644 --- a/metrics/go.mod +++ b/metrics/go.mod @@ -2,15 +2,61 @@ module github.com/smartcontractkit/chainlink-framework/metrics go 1.24.1 -require github.com/prometheus/client_golang v1.22.0 +require ( + github.com/prometheus/client_golang v1.22.0 + github.com/smartcontractkit/chainlink-common v0.7.0 + go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/metric v1.35.0 +) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect + github.com/cloudevents/sdk-go/v2 v2.16.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.30.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/metrics/go.sum b/metrics/go.sum index a89f616..13563ef 100644 --- a/metrics/go.sum +++ b/metrics/go.sum @@ -1,28 +1,147 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP0qYY6yzdIU9xdrRd/YMP0bNwFjXd0u8= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= +github.com/cloudevents/sdk-go/v2 v2.16.0 h1:wnunjgiLQCfYlyo+E4+mFlZtAh7pKn7vT8MMD3lSwCg= +github.com/cloudevents/sdk-go/v2 v2.16.0/go.mod h1:5YWqklyhDSmGzBK/JENKKXdulbPq0JFf3c/KEnMLqgg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= +github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 h1:UiRNKd1OgqsLbFwE+wkAWTdiAxXtCBqKIHeBIse4FUA= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9/go.mod h1:eqZlW3pJWhjyexnDPrdQxix1pn0wwhI4AO4GKpP/bMI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/logpoller.go b/metrics/logpoller.go index 03b6384..924318d 100644 --- a/metrics/logpoller.go +++ b/metrics/logpoller.go @@ -1,8 +1,11 @@ package metrics import ( + "context" "time" + "go.opentelemetry.io/otel/metric" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -56,3 +59,19 @@ var ( Help: "Counter to track number of blocks inserted by Log Poller", }, []string{"chainFamily", "chainID"}) ) + +type GenericLogPollerORMMetrics interface { + RecordQueryDuration(ctx context.Context, query QueryType, duration float64) + RecordQueryDataSetsSize(ctx context.Context, query QueryType, size int) + IncrementLogsInserted(ctx context.Context, numLogs int) + IncrementBlocksInserted(ctx context.Context, numBlocks int) +} + +type logPollerORMMetrics struct { + chainID string + chainFamily string + queryDuration metric.Float64Histogram + queryDataSets metric.Int64Gauge + logsInserted metric.Int64Counter + blocksInserted metric.Int64Counter +} diff --git a/metrics/txm.go b/metrics/txm.go index 1abe097..be0a68f 100644 --- a/metrics/txm.go +++ b/metrics/txm.go @@ -1 +1,178 @@ package metrics + +import ( + "context" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" +) + +var ( + promNumBroadcasted = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_broadcasted", + Help: "The number of transactions broadcasted", + }, []string{"chainID"}) + promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_broadcast", + Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + }, + }, []string{"chainID"}) + promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_gas_bumps", + Help: "Number of gas bumps", + }, []string{"chainID"}) + + promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_gas_bump_exceeds_limit", + Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + float64(5 * time.Minute), + float64(10 * time.Minute), + }, + }, []string{"chainID"}) + promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_blocks_until_tx_confirmed", + Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(1), + float64(5), + float64(10), + float64(20), + float64(50), + float64(100), + }, + }, []string{"chainID"}) +) + +type GenericTXMMetrics interface { + IncrementNumBroadcastedTxs(ctx context.Context) + RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) + IncrementNumGasBumps(ctx context.Context) + IncrementGasBumpExceedsLimit(ctx context.Context) + IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) + RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) + RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) +} + +type txmMetrics struct { + chainID string + numBroadcastedTxs metric.Int64Counter + timeUntilBroadcast metric.Float64Histogram + numGasBumps metric.Int64Counter + gasBumpExceedsLimit metric.Int64Counter + numConfirmedTxs metric.Int64Counter + timeUntilTxConfirmed metric.Float64Histogram + blocksUntilTxConfirmed metric.Float64Histogram +} + +func NewGenericTxmMetrics(chainID string) (*txmMetrics, error) { + numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") + if err != nil { + return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) + } + + timeUntilBroadcast, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_broadcast") + if err != nil { + return nil, fmt.Errorf("failed to register time until broadcast metric: %w", err) + } + + numGasBumps, err := beholder.GetMeter().Int64Counter("tx_manager_num_gas_bumps") + if err != nil { + return nil, fmt.Errorf("failed to register number of gas bumps metric: %w", err) + } + + gasBumpExceedsLimit, err := beholder.GetMeter().Int64Counter("tx_manager_gas_bump_exceeds_limit") + if err != nil { + return nil, fmt.Errorf("failed to register gas bump exceeds limit metric: %w", err) + } + + numConfirmedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_confirmed_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register confirmed txs number metric: %w", err) + } + + timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_time_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register time until tx confirmed metric: %w", err) + } + + blocksUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("tx_manager_blocks_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register blocks until tx confirmed metric: %w", err) + } + + return &txmMetrics{ + chainID: chainID, + numBroadcastedTxs: numBroadcastedTxs, + timeUntilBroadcast: timeUntilBroadcast, + numGasBumps: numGasBumps, + gasBumpExceedsLimit: gasBumpExceedsLimit, + numConfirmedTxs: numConfirmedTxs, + timeUntilTxConfirmed: timeUntilTxConfirmed, + blocksUntilTxConfirmed: blocksUntilTxConfirmed, + }, nil +} + +func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { + promNumBroadcasted.WithLabelValues(m.chainID).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordTimeUntilTxBroadcast(ctx context.Context, duration float64) { + promTimeUntilBroadcast.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilBroadcast.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementNumGasBumps(ctx context.Context) { + promNumGasBumps.WithLabelValues(m.chainID).Add(float64(1)) + m.numGasBumps.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementGasBumpExceedsLimit(ctx context.Context) { + promGasBumpExceedsLimit.WithLabelValues(m.chainID).Add(float64(1)) + m.gasBumpExceedsLimit.Add(ctx, 1, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { + promNumConfirmedTxs.WithLabelValues(m.chainID).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions), metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { + promTimeUntilTxConfirmed.WithLabelValues(m.chainID).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} + +func (m *txmMetrics) RecordBlocksUntilTxConfirmed(ctx context.Context, blocksElapsed float64) { + promBlocksUntilTxConfirmed.WithLabelValues(m.chainID).Observe(blocksElapsed) + m.blocksUntilTxConfirmed.Record(ctx, blocksElapsed, metric.WithAttributes(attribute.String("chainID", m.chainID))) +} From e2685f8677f7d9f8a8c9d3076f7d932fd8a6816c Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 12:31:46 -0400 Subject: [PATCH 06/16] Add Beholder Metrics --- metrics/logpoller.go | 104 +++++++++++++--- metrics/multinode.go | 257 ++++++++++++++++++++++++++++++++++++++++ metrics/txm.go | 2 +- multinode/multi_node.go | 12 +- multinode/node.go | 13 +- multinode/node_fsm.go | 34 +----- 6 files changed, 347 insertions(+), 75 deletions(-) create mode 100644 metrics/multinode.go diff --git a/metrics/logpoller.go b/metrics/logpoller.go index 924318d..88a01d3 100644 --- a/metrics/logpoller.go +++ b/metrics/logpoller.go @@ -2,12 +2,15 @@ package metrics import ( "context" + "fmt" "time" - "go.opentelemetry.io/otel/metric" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" ) type QueryType string @@ -41,37 +44,102 @@ var ( float64(2 * time.Second), float64(5 * time.Second), } - LpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + promLpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "log_poller_query_duration", Help: "Measures duration of Log Poller's queries fetching logs", Buckets: sqlLatencyBuckets, }, []string{"chainFamily", "chainID", "query", "type"}) - LpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ + promLpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "log_poller_query_dataset_size", Help: "Measures size of the datasets returned by Log Poller's queries", }, []string{"chainFamily", "chainID", "query", "type"}) - LpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + promLpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_logs_inserted", Help: "Counter to track number of logs inserted by Log Poller", }, []string{"chainFamily", "chainID"}) - LpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + promLpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_blocks_inserted", Help: "Counter to track number of blocks inserted by Log Poller", }, []string{"chainFamily", "chainID"}) ) -type GenericLogPollerORMMetrics interface { - RecordQueryDuration(ctx context.Context, query QueryType, duration float64) - RecordQueryDataSetsSize(ctx context.Context, query QueryType, size int) - IncrementLogsInserted(ctx context.Context, numLogs int) - IncrementBlocksInserted(ctx context.Context, numBlocks int) +type GenericLogPollerMetrics interface { + RecordQueryDuration(ctx context.Context, queryName string, queryType QueryType, duration float64) + RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) + IncrementLogsInserted(ctx context.Context, numLogs int64) + IncrementBlocksInserted(ctx context.Context, numBlocks int64) +} + +var _ GenericLogPollerMetrics = &logPollerMetrics{} + +type logPollerMetrics struct { + chainID string + chainFamily string + queryDuration metric.Float64Histogram + queryDatasetsSize metric.Int64Gauge + logsInserted metric.Int64Counter + blocksInserted metric.Int64Counter +} + +func NewGenericLogPollerMetrics(chainID string, chainFamily string) (GenericLogPollerMetrics, error) { + queryDuration, err := beholder.GetMeter().Float64Histogram("log_poller_query_duration") + if err != nil { + return nil, fmt.Errorf("failed to register logpoller query duration metric: %w", err) + } + + queryDatasetSize, err := beholder.GetMeter().Int64Gauge("log_poller_query_dataset_size") + if err != nil { + return nil, fmt.Errorf("failed to register query dataset size metric: %w", err) + } + + logsInserted, err := beholder.GetMeter().Int64Counter("log_poller_logs_inserted") + if err != nil { + return nil, fmt.Errorf("failed to register logs inserted metric: %w", err) + } + + blocksInserted, err := beholder.GetMeter().Int64Counter("log_poller_blocks_inserted") + if err != nil { + return nil, fmt.Errorf("failed to register blocks inserted metric: %w", err) + } + + return &logPollerMetrics{ + chainID: chainID, + chainFamily: chainFamily, + queryDuration: queryDuration, + queryDatasetsSize: queryDatasetSize, + logsInserted: logsInserted, + blocksInserted: blocksInserted, + }, nil +} + +func (m *logPollerMetrics) RecordQueryDuration(ctx context.Context, queryName string, queryType QueryType, duration float64) { + promLpQueryDuration.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Observe(duration) + m.queryDuration.Record(ctx, duration, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID), + attribute.String("query", queryName), + attribute.String("type", string(queryType)))) +} + +func (m *logPollerMetrics) RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) { + promLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Add(float64(size)) + m.queryDatasetsSize.Record(ctx, size, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID), + attribute.String("query", queryName), + attribute.String("type", string(queryType)))) +} + +func (m *logPollerMetrics) IncrementLogsInserted(ctx context.Context, numLogs int64) { + promLpLogsInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numLogs)) + m.logsInserted.Add(ctx, numLogs, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID))) } -type logPollerORMMetrics struct { - chainID string - chainFamily string - queryDuration metric.Float64Histogram - queryDataSets metric.Int64Gauge - logsInserted metric.Int64Counter - blocksInserted metric.Int64Counter +func (m *logPollerMetrics) IncrementBlocksInserted(ctx context.Context, numBlocks int64) { + promLpBlocksInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numBlocks)) + m.blocksInserted.Add(ctx, numBlocks, metric.WithAttributes( + attribute.String("chainFamily", m.chainFamily), + attribute.String("chainID", m.chainID))) } diff --git a/metrics/multinode.go b/metrics/multinode.go new file mode 100644 index 0000000..d9ca0e6 --- /dev/null +++ b/metrics/multinode.go @@ -0,0 +1,257 @@ +package metrics + +import ( + "context" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + // Node States + promMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multi_node_states", + Help: "The number of RPC nodes currently in the given state for the given chain", + }, []string{"network", "chainId", "state"}) + + // Node Verification + promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies", + Help: "The total number of chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_failed", + Help: "The total number of failed chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_success", + Help: "The total number of successful chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + + // TODO: Should these all have network as well? + // Node State Transitions + promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_alive", + Help: "Total number of times node has transitioned to Alive", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_in_sync", + Help: "Total number of times node has transitioned from OutOfSync to Alive", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Help: "Total number of times node has transitioned to OutOfSync", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unreachable", + Help: "Total number of times node has transitioned to Unreachable", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Help: "Total number of times node has transitioned to InvalidChainID", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unusable", + Help: "Total number of times node has transitioned to Unusable", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_syncing", + Help: "Total number of times node has transitioned to Syncing", + }, []string{"chainID", "nodeName"}) +) + +type GenericMultiNodeMetrics interface { + RecordNodeStates(ctx context.Context, state string, count int64) + IncrementNodeVerifies(ctx context.Context, nodeName string) + IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) + IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) + IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) +} + +var _ GenericMultiNodeMetrics = &multiNodeMetrics{} + +type multiNodeMetrics struct { + network string + chainID string + nodeStates metric.Int64Gauge + nodeVerifies metric.Int64Counter + nodeVerifiesFailed metric.Int64Counter + nodeVerifiesSuccess metric.Int64Counter + nodeTransitionsToAlive metric.Int64Counter + nodeTransitionsToInSync metric.Int64Counter + nodeTransitionsToOutOfSync metric.Int64Counter + nodeTransitionsToUnreachable metric.Int64Counter + nodeTransitionsToInvalidChainID metric.Int64Counter + nodeTransitionsToUnusable metric.Int64Counter + nodeTransitionsToSyncing metric.Int64Counter +} + +func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNodeMetrics, error) { + nodeStates, err := beholder.GetMeter().Int64Gauge("multi_node_states") + if err != nil { + return nil, fmt.Errorf("failed to register multinode states metric: %w", err) + } + + nodeVerifies, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies metric: %w", err) + } + + nodeVerifiesFailed, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies_failed") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies failed metric: %w", err) + } + + nodeVerifiesSuccess, err := beholder.GetMeter().Int64Counter("pool_rpc_node_verifies_success") + if err != nil { + return nil, fmt.Errorf("failed to register node verifies success metric: %w", err) + } + + nodeTransitionsToAlive, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_alive") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to alive metric: %w", err) + } + + nodeTransitionsToInSync, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_in_sync") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to in sync metric: %w", err) + } + + nodeTransitionsToOutOfSync, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_out_of_sync") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to out of sync metric: %w", err) + } + + nodeTransitionsToUnreachable, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_unreachable") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to unreachable metric: %w", err) + } + + nodeTransitionsToInvalidChainID, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_invalid_chain_id") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to invalid chain id metric: %w", err) + } + + nodeTransitionsToUnusable, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_unusable") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to unusable metric: %w", err) + } + + nodeTransitionsToSyncing, err := beholder.GetMeter().Int64Counter("pool_rpc_node_num_transitions_to_syncing") + if err != nil { + return nil, fmt.Errorf("failed to register node transitions to syncing metric: %w", err) + } + + return &multiNodeMetrics{ + network: network, + chainID: chainID, + nodeStates: nodeStates, + nodeVerifies: nodeVerifies, + nodeVerifiesFailed: nodeVerifiesFailed, + nodeVerifiesSuccess: nodeVerifiesSuccess, + nodeTransitionsToAlive: nodeTransitionsToAlive, + nodeTransitionsToInSync: nodeTransitionsToInSync, + nodeTransitionsToOutOfSync: nodeTransitionsToOutOfSync, + nodeTransitionsToUnreachable: nodeTransitionsToUnreachable, + nodeTransitionsToInvalidChainID: nodeTransitionsToInvalidChainID, + nodeTransitionsToUnusable: nodeTransitionsToUnusable, + nodeTransitionsToSyncing: nodeTransitionsToSyncing, + }, nil +} + +func (m *multiNodeMetrics) RecordNodeStates(ctx context.Context, state string, count int64) { + promMultiNodeRPCNodeStates.WithLabelValues(m.network, m.chainID, state).Set(float64(count)) + m.nodeStates.Record(ctx, count, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("state", state))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifies(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifies.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifies.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifiesFailed.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifiesFailed.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) { + promPoolRPCNodeVerifiesSuccess.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeVerifiesSuccess.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToAlive.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToAlive.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToInSync.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToInSync.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToOutOfSync.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToUnreachable.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToInvalidChainID.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToUnusable.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToUnusable.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} + +func (m *multiNodeMetrics) IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) { + promPoolRPCNodeTransitionsToSyncing.WithLabelValues(m.network, m.chainID, nodeName).Inc() + m.nodeTransitionsToSyncing.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("nodeName", nodeName))) +} diff --git a/metrics/txm.go b/metrics/txm.go index be0a68f..ae8be59 100644 --- a/metrics/txm.go +++ b/metrics/txm.go @@ -94,7 +94,7 @@ type txmMetrics struct { blocksUntilTxConfirmed metric.Float64Histogram } -func NewGenericTxmMetrics(chainID string) (*txmMetrics, error) { +func NewGenericTxmMetrics(chainID string) (GenericTXMMetrics, error) { numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("tx_manager_num_broadcasted") if err != nil { return nil, fmt.Errorf("failed to register broadcasted txs number metric: %w", err) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 9c3af89..b90b4f9 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -8,21 +8,11 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) -var ( - // PromMultiNodeRPCNodeStates reports current RPC node state - PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", - Help: "The number of RPC nodes currently in the given state for the given chain", - }, []string{"network", "chainId", "state"}) - ErrNodeError = fmt.Errorf("no live nodes available") -) +var ErrNodeError = fmt.Errorf("no live nodes available") // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. diff --git a/multinode/node.go b/multinode/node.go index a457711..2982616 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -20,18 +20,7 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") var ( - promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", - Help: "The total number of chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", - Help: "The total number of failed chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", - Help: "The total number of successful chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) +) ) type NodeConfig interface { diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index 5b76752..4b4bb42 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -2,41 +2,9 @@ package multinode import ( "fmt" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" ) -var ( - promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", - Help: transitionString(nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", - Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", - Help: transitionString(nodeStateOutOfSync), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", - Help: transitionString(nodeStateUnreachable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", - Help: transitionString(nodeStateInvalidChainID), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", - Help: transitionString(nodeStateUnusable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", - Help: transitionString(nodeStateSyncing), - }, []string{"chainID", "nodeName"}) -) +var () // nodeState represents the current state of the node // Node is a FSM (finite state machine) From 05e081bdb655cca9cca4a08bfb00dc5e8404ef07 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 13:16:13 -0400 Subject: [PATCH 07/16] Add metrics --- metrics/multinode.go | 26 ++++++- multinode/go.mod | 63 +++++++++++++---- multinode/go.sum | 152 ++++++++++++++++++++++++++++++---------- multinode/multi_node.go | 11 ++- multinode/node.go | 18 +++-- multinode/node_fsm.go | 30 +++++--- 6 files changed, 225 insertions(+), 75 deletions(-) diff --git a/metrics/multinode.go b/metrics/multinode.go index d9ca0e6..1103b6b 100644 --- a/metrics/multinode.go +++ b/metrics/multinode.go @@ -3,11 +3,13 @@ package metrics import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/beholder" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" ) var ( @@ -61,6 +63,12 @@ var ( Name: "pool_rpc_node_num_transitions_to_syncing", Help: "Total number of times node has transitioned to Syncing", }, []string{"chainID", "nodeName"}) + + // Transaction Sender + promMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multi_node_invariant_violations", + Help: "The number of invariant violations", + }, []string{"network", "chainId", "invariant"}) ) type GenericMultiNodeMetrics interface { @@ -75,6 +83,7 @@ type GenericMultiNodeMetrics interface { IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) + IncrementInvariantViolations(ctx context.Context, invariant string) } var _ GenericMultiNodeMetrics = &multiNodeMetrics{} @@ -93,6 +102,7 @@ type multiNodeMetrics struct { nodeTransitionsToInvalidChainID metric.Int64Counter nodeTransitionsToUnusable metric.Int64Counter nodeTransitionsToSyncing metric.Int64Counter + invariantViolations metric.Int64Counter } func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNodeMetrics, error) { @@ -151,6 +161,11 @@ func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNod return nil, fmt.Errorf("failed to register node transitions to syncing metric: %w", err) } + invariantViolations, err := beholder.GetMeter().Int64Counter("multi_node_invariant_violations") + if err != nil { + return nil, fmt.Errorf("failed to register invariant violations metric: %w", err) + } + return &multiNodeMetrics{ network: network, chainID: chainID, @@ -165,6 +180,7 @@ func NewGenericMultiNodeMetrics(network string, chainID string) (GenericMultiNod nodeTransitionsToInvalidChainID: nodeTransitionsToInvalidChainID, nodeTransitionsToUnusable: nodeTransitionsToUnusable, nodeTransitionsToSyncing: nodeTransitionsToSyncing, + invariantViolations: invariantViolations, }, nil } @@ -255,3 +271,11 @@ func (m *multiNodeMetrics) IncrementNodeTransitionsToSyncing(ctx context.Context attribute.String("chainID", m.chainID), attribute.String("nodeName", nodeName))) } + +func (m *multiNodeMetrics) IncrementInvariantViolations(ctx context.Context, invariant string) { + promMultiNodeInvariantViolations.WithLabelValues(m.network, m.chainID, invariant).Inc() + m.invariantViolations.Add(ctx, 1, metric.WithAttributes( + attribute.String("network", m.network), + attribute.String("chainID", m.chainID), + attribute.String("invariant", invariant))) +} diff --git a/multinode/go.mod b/multinode/go.mod index afb5d43..46accbb 100644 --- a/multinode/go.mod +++ b/multinode/go.mod @@ -1,40 +1,75 @@ module github.com/smartcontractkit/chainlink-framework/multinode -go 1.23.3 +go 1.24.1 + +toolchain go1.24.2 require ( github.com/jpillora/backoff v1.0.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.7.0 + github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7 github.com/stretchr/testify v1.10.0 go.uber.org/zap v1.27.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect + github.com/cloudevents/sdk-go/v2 v2.16.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.60.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect github.com/stretchr/objx v0.5.2 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.6.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/sys v0.26.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/multinode/go.sum b/multinode/go.sum index 6c3095e..5381803 100644 --- a/multinode/go.sum +++ b/multinode/go.sum @@ -1,7 +1,13 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP0qYY6yzdIU9xdrRd/YMP0bNwFjXd0u8= +github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= +github.com/cloudevents/sdk-go/v2 v2.16.0 h1:wnunjgiLQCfYlyo+E4+mFlZtAh7pKn7vT8MMD3lSwCg= +github.com/cloudevents/sdk-go/v2 v2.16.0/go.mod h1:5YWqklyhDSmGzBK/JENKKXdulbPq0JFf3c/KEnMLqgg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -11,77 +17,149 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= -github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= +github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7 h1:s/wTwHj1C2IsW05vziADSHsrqTY2d33VRFZFsumQM9s= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7/go.mod h1:1D0TYbsfFFNVkPs5QAr5xV6aCpcGukQ+jgwh5G91Fu0= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= +github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9 h1:UiRNKd1OgqsLbFwE+wkAWTdiAxXtCBqKIHeBIse4FUA= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240823153156-2a54df7bffb9/go.mod h1:eqZlW3pJWhjyexnDPrdQxix1pn0wwhI4AO4GKpP/bMI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= +go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= +go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/multinode/multi_node.go b/multinode/multi_node.go index b90b4f9..4286c62 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/metrics" ) var ErrNodeError = fmt.Errorf("no live nodes available") @@ -27,6 +28,7 @@ type MultiNode[ sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC] chainID CHAIN_ID lggr logger.SugaredLogger + metrics metrics.GenericMultiNodeMetrics selectionMode string nodeSelector NodeSelector[CHAIN_ID, RPC] leaseDuration time.Duration @@ -44,6 +46,7 @@ func NewMultiNode[ RPC any, ]( lggr logger.Logger, + metrics metrics.GenericMultiNodeMetrics, selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected primaryNodes []Node[CHAIN_ID, RPC], @@ -57,6 +60,7 @@ func NewMultiNode[ // aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) const reportInterval = 6500 * time.Millisecond c := &MultiNode[CHAIN_ID, RPC]{ + metrics: metrics, primaryNodes: primaryNodes, sendOnlyNodes: sendOnlyNodes, chainID: chainID, @@ -351,9 +355,12 @@ func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) { dead++ } } + + ctx, cancel := c.eng.NewCtx() + defer cancel() for _, state := range allNodeStates { - count := counts[state] - PromMultiNodeRPCNodeStates.WithLabelValues(c.chainFamily, c.chainID.String(), state.String()).Set(float64(count)) + count := int64(counts[state]) + c.metrics.RecordNodeStates(ctx, state.String(), count) } total := len(c.primaryNodes) diff --git a/multinode/node.go b/multinode/node.go index 2982616..db0547a 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -4,13 +4,11 @@ import ( "context" "errors" "fmt" + "github.com/smartcontractkit/chainlink-framework/metrics" "net/url" "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -19,10 +17,6 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") -var ( -) -) - type NodeConfig interface { PollFailureThreshold() uint32 PollInterval() time.Duration @@ -87,6 +81,8 @@ type node[ order int32 chainFamily string + metrics metrics.GenericMultiNodeMetrics + ws *url.URL http *url.URL @@ -112,6 +108,7 @@ func NewNode[ nodeCfg NodeConfig, chainCfg ChainConfig, lggr logger.Logger, + metrics metrics.GenericMultiNodeMetrics, wsuri *url.URL, httpuri *url.URL, name string, @@ -128,6 +125,7 @@ func NewNode[ n.nodePoolCfg = nodeCfg n.chainCfg = chainCfg n.order = nodeOrder + n.metrics = metrics if wsuri != nil { n.ws = wsuri } @@ -242,9 +240,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) start() { // Not thread-safe // Pure verifyChainID: does not mutate node "state" field. func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lggr logger.Logger) nodeState { - promPoolRPCNodeVerifies.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifies(callerCtx, n.name) promFailed := func() { - promPoolRPCNodeVerifiesFailed.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifiesFailed(callerCtx, n.name) } st := n.getCachedState() @@ -277,7 +275,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lgg return nodeStateInvalidChainID } - promPoolRPCNodeVerifiesSuccess.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + n.metrics.IncrementNodeVerifiesSuccess(callerCtx, n.name) return nodeStateAlive } diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index 4b4bb42..994036f 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -175,7 +175,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareAlive() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToAlive(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -201,8 +203,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareInSync() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInSync(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() - promPoolRPCNodeTransitionsToInSync.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToAlive(ctx, n.name) + n.metrics.IncrementNodeTransitionsToInSync(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -228,7 +232,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareOutOfSync(syncIssues syncStatus) { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) { - promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToOutOfSync(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -253,7 +259,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareUnreachable() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToUnreachable(fn func()) { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToUnreachable(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -296,7 +304,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareInvalidChainID() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInvalidChainID(fn func()) { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -321,7 +331,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareSyncing() { } func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { - promPoolRPCNodeTransitionsToSyncing.WithLabelValues(n.chainID.String(), n.name).Inc() + ctx, cancel := n.stopCh.NewCtx() + defer cancel() + n.metrics.IncrementNodeTransitionsToSyncing(ctx, n.name) n.stateMu.Lock() defer n.stateMu.Unlock() if n.state == nodeStateClosed { @@ -341,10 +353,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { fn() } -func transitionString(state nodeState) string { - return fmt.Sprintf("Total number of times node has transitioned to %s", state) -} - func transitionFail(from nodeState, to nodeState) string { return fmt.Sprintf("cannot transition from %#v to %#v", from, to) } From d6803ac510d72da042c4272cbfed9648116ef662 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 13:24:57 -0400 Subject: [PATCH 08/16] Create generic metrics --- multinode/go.mod | 1 - multinode/go.sum | 2 -- multinode/multi_node.go | 9 ++++++--- multinode/node.go | 18 +++++++++++++++--- multinode/transaction_sender.go | 22 ++++++++++------------ 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/multinode/go.mod b/multinode/go.mod index 46accbb..8d8958c 100644 --- a/multinode/go.mod +++ b/multinode/go.mod @@ -10,7 +10,6 @@ require ( github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 github.com/smartcontractkit/chainlink-common v0.7.0 - github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7 github.com/stretchr/testify v1.10.0 go.uber.org/zap v1.27.0 ) diff --git a/multinode/go.sum b/multinode/go.sum index 5381803..71c59db 100644 --- a/multinode/go.sum +++ b/multinode/go.sum @@ -72,8 +72,6 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7 h1:s/wTwHj1C2IsW05vziADSHsrqTY2d33VRFZFsumQM9s= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250425163146-e2685f8677f7/go.mod h1:1D0TYbsfFFNVkPs5QAr5xV6aCpcGukQ+jgwh5G91Fu0= github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298/go.mod h1:Mb7+/LC4edz7HyHxX4QkE42pSuov4AV68+AxBXAap0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 4286c62..c350953 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -10,11 +10,14 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-framework/metrics" ) var ErrNodeError = fmt.Errorf("no live nodes available") +type multiNodeMetrics interface { + RecordNodeStates(ctx context.Context, state string, count int64) +} + // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. type MultiNode[ @@ -28,7 +31,7 @@ type MultiNode[ sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC] chainID CHAIN_ID lggr logger.SugaredLogger - metrics metrics.GenericMultiNodeMetrics + metrics multiNodeMetrics selectionMode string nodeSelector NodeSelector[CHAIN_ID, RPC] leaseDuration time.Duration @@ -46,7 +49,7 @@ func NewMultiNode[ RPC any, ]( lggr logger.Logger, - metrics metrics.GenericMultiNodeMetrics, + metrics multiNodeMetrics, selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected primaryNodes []Node[CHAIN_ID, RPC], diff --git a/multinode/node.go b/multinode/node.go index db0547a..825e5b1 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/smartcontractkit/chainlink-framework/metrics" "net/url" "sync" "time" @@ -38,6 +37,19 @@ type ChainConfig interface { FinalizedBlockOffset() uint32 } +type nodeMetrics interface { + IncrementNodeVerifies(ctx context.Context, nodeName string) + IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) + IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) + IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) + IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) + IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) +} + type Node[ CHAIN_ID ID, RPC any, @@ -81,7 +93,7 @@ type node[ order int32 chainFamily string - metrics metrics.GenericMultiNodeMetrics + metrics nodeMetrics ws *url.URL http *url.URL @@ -108,7 +120,7 @@ func NewNode[ nodeCfg NodeConfig, chainCfg ChainConfig, lggr logger.Logger, - metrics metrics.GenericMultiNodeMetrics, + metrics nodeMetrics, wsuri *url.URL, httpuri *url.URL, name string, diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index 43e6fda..7be3f23 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -8,21 +8,10 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) -var ( - // PromMultiNodeInvariantViolations reports violation of our assumptions - PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", - Help: "The number of invariant violations", - }, []string{"network", "chainId", "invariant"}) -) - type sendTxResult[RESULT any] struct { res RESULT code SendTxReturnCode @@ -37,11 +26,16 @@ type SendTxRPCClient[TX any, RESULT any] interface { SendTransaction(ctx context.Context, tx TX) (RESULT, SendTxReturnCode, error) } +type transactionSenderMetrics interface { + IncrementInvariantViolations(ctx context.Context, invariant string) +} + func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, multiNode *MultiNode[CHAIN_ID, RPC], + metrics transactionSenderMetrics, classifyErr func(err error) SendTxReturnCode, sendTxSoftTimeout time.Duration, ) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { @@ -52,6 +46,7 @@ func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[T chainID: chainID, chainFamily: chainFamily, lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), + metrics: metrics, multiNode: multiNode, classifyErr: classifyErr, sendTxSoftTimeout: sendTxSoftTimeout, @@ -64,6 +59,7 @@ type TransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, chainID CHAIN_ID chainFamily string lggr logger.SugaredLogger + metrics transactionSenderMetrics multiNode *MultiNode[CHAIN_ID, RPC] classifyErr func(err error) SendTxReturnCode sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation @@ -199,7 +195,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal _, criticalErr := aggregateTxResults(resultsByCode) if criticalErr != nil { txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) - PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() + ctx, cancel := txSender.chStop.NewCtx() + defer cancel() + txSender.metrics.IncrementInvariantViolations(ctx, criticalErr.Error()) } } From ec2fb6ab36a5e69b161895f855044b9d51e2af27 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 13:32:42 -0400 Subject: [PATCH 09/16] Update send_only_node.go --- multinode/send_only_node.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/multinode/send_only_node.go b/multinode/send_only_node.go index 7630362..c61a51c 100644 --- a/multinode/send_only_node.go +++ b/multinode/send_only_node.go @@ -45,6 +45,7 @@ type sendOnlyNode[ ] struct { services.StateMachine + metrics nodeMetrics stateMu sync.RWMutex // protects state* fields state nodeState @@ -63,6 +64,7 @@ func NewSendOnlyNode[ RPC sendOnlyClient[CHAIN_ID], ]( lggr logger.Logger, + metrics nodeMetrics, httpuri url.URL, name string, chainID CHAIN_ID, @@ -74,6 +76,7 @@ func NewSendOnlyNode[ s.log = logger.With(s.log, "nodeTier", "sendonly", ) + s.metrics = metrics s.rpc = rpc s.uri = httpuri s.chainID = chainID @@ -101,7 +104,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { err := s.rpc.Dial(ctx) if err != nil { - promPoolRPCNodeTransitionsToUnusable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnusable(ctx, s.name) s.log.Errorw("Dial failed: SendOnly Node is unusable", "err", err) s.setState(nodeStateUnusable) return @@ -114,13 +117,13 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { } else { chainID, err := s.rpc.ChainID(ctx) if err != nil || chainID.String() != s.chainID.String() { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) if err != nil { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) s.setState(nodeStateUnreachable) } else { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, s.name) s.log.Errorf( "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", chainID.String(), @@ -137,7 +140,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start() { } } - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToAlive(ctx, s.name) s.setState(nodeStateAlive) s.log.Infow("Sendonly RPC Node is online", "nodeState", s.state) } From 19afa9b0b9b02dd45eab770b77a7f0da12a07d08 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Apr 2025 13:35:21 -0400 Subject: [PATCH 10/16] Update send_only_node_lifecycle.go --- multinode/send_only_node_lifecycle.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multinode/send_only_node_lifecycle.go b/multinode/send_only_node_lifecycle.go index fe88465..89b3d12 100644 --- a/multinode/send_only_node_lifecycle.go +++ b/multinode/send_only_node_lifecycle.go @@ -25,7 +25,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { if err != nil { ok := s.IfStarted(func() { if changed := s.setState(nodeStateUnreachable); changed { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToUnreachable(ctx, s.name) } }) if !ok { @@ -36,7 +36,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { } else if chainID.String() != s.chainID.String() { ok := s.IfStarted(func() { if changed := s.setState(nodeStateInvalidChainID); changed { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToInvalidChainID(ctx, s.name) } }) if !ok { @@ -53,7 +53,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { } ok := s.IfStarted(func() { if changed := s.setState(nodeStateAlive); changed { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + s.metrics.IncrementNodeTransitionsToAlive(ctx, s.name) } }) if !ok { From 933e3172bbe75af92ee0e29f52d41e5ba1e88bf3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 07:59:22 -0400 Subject: [PATCH 11/16] Export logpoller metrics --- metrics/logpoller.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/metrics/logpoller.go b/metrics/logpoller.go index 88a01d3..d2d7950 100644 --- a/metrics/logpoller.go +++ b/metrics/logpoller.go @@ -44,20 +44,20 @@ var ( float64(2 * time.Second), float64(5 * time.Second), } - promLpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + PromLpQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "log_poller_query_duration", Help: "Measures duration of Log Poller's queries fetching logs", Buckets: sqlLatencyBuckets, }, []string{"chainFamily", "chainID", "query", "type"}) - promLpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ + PromLpQueryDataSets = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "log_poller_query_dataset_size", Help: "Measures size of the datasets returned by Log Poller's queries", }, []string{"chainFamily", "chainID", "query", "type"}) - promLpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + PromLpLogsInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_logs_inserted", Help: "Counter to track number of logs inserted by Log Poller", }, []string{"chainFamily", "chainID"}) - promLpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ + PromLpBlocksInserted = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "log_poller_blocks_inserted", Help: "Counter to track number of blocks inserted by Log Poller", }, []string{"chainFamily", "chainID"}) @@ -113,7 +113,7 @@ func NewGenericLogPollerMetrics(chainID string, chainFamily string) (GenericLogP } func (m *logPollerMetrics) RecordQueryDuration(ctx context.Context, queryName string, queryType QueryType, duration float64) { - promLpQueryDuration.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Observe(duration) + PromLpQueryDuration.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Observe(duration) m.queryDuration.Record(ctx, duration, metric.WithAttributes( attribute.String("chainFamily", m.chainFamily), attribute.String("chainID", m.chainID), @@ -122,7 +122,7 @@ func (m *logPollerMetrics) RecordQueryDuration(ctx context.Context, queryName st } func (m *logPollerMetrics) RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) { - promLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Add(float64(size)) + PromLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Add(float64(size)) m.queryDatasetsSize.Record(ctx, size, metric.WithAttributes( attribute.String("chainFamily", m.chainFamily), attribute.String("chainID", m.chainID), @@ -131,14 +131,14 @@ func (m *logPollerMetrics) RecordQueryDatasetSize(ctx context.Context, queryName } func (m *logPollerMetrics) IncrementLogsInserted(ctx context.Context, numLogs int64) { - promLpLogsInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numLogs)) + PromLpLogsInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numLogs)) m.logsInserted.Add(ctx, numLogs, metric.WithAttributes( attribute.String("chainFamily", m.chainFamily), attribute.String("chainID", m.chainID))) } func (m *logPollerMetrics) IncrementBlocksInserted(ctx context.Context, numBlocks int64) { - promLpBlocksInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numBlocks)) + PromLpBlocksInserted.WithLabelValues(m.chainFamily, m.chainID).Add(float64(numBlocks)) m.blocksInserted.Add(ctx, numBlocks, metric.WithAttributes( attribute.String("chainFamily", m.chainFamily), attribute.String("chainID", m.chainID))) From 1c6ca7dda2253d6f9c60294e3c1a4cb59601ec11 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 09:00:26 -0400 Subject: [PATCH 12/16] Set dataset size --- metrics/logpoller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/logpoller.go b/metrics/logpoller.go index d2d7950..0963a68 100644 --- a/metrics/logpoller.go +++ b/metrics/logpoller.go @@ -122,7 +122,7 @@ func (m *logPollerMetrics) RecordQueryDuration(ctx context.Context, queryName st } func (m *logPollerMetrics) RecordQueryDatasetSize(ctx context.Context, queryName string, queryType QueryType, size int64) { - PromLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Add(float64(size)) + PromLpQueryDataSets.WithLabelValues(m.chainFamily, m.chainID, queryName, string(queryType)).Set(float64(size)) m.queryDatasetsSize.Record(ctx, size, metric.WithAttributes( attribute.String("chainFamily", m.chainFamily), attribute.String("chainID", m.chainID), From c3486bf525fa63fa3ec7fd698dd672580e36573d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 09:17:13 -0400 Subject: [PATCH 13/16] Bump mockery --- Makefile | 2 +- multinode/mock_head_test.go | 10 +-- multinode/mock_node_selector_test.go | 16 ++--- multinode/mock_node_test.go | 64 ++++++++++--------- .../mock_pool_chain_info_provider_test.go | 6 +- multinode/mock_rpc_client_test.go | 14 ++-- multinode/mock_send_only_client_test.go | 10 +-- multinode/mock_send_only_node_test.go | 42 ++++++------ multinode/mock_subscription_test.go | 8 +-- 9 files changed, 92 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 12aa563..1416abe 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ gomodtidy: gomods .PHONY: mockery mockery: $(mockery) ## Install mockery. - go install github.com/vektra/mockery/v2@v2.46.3 + go install github.com/vektra/mockery/v2@v2.53.0 .PHONY: generate generate: mockery diff --git a/multinode/mock_head_test.go b/multinode/mock_head_test.go index bd3d414..291a83c 100644 --- a/multinode/mock_head_test.go +++ b/multinode/mock_head_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -21,7 +21,7 @@ func (_m *mockHead) EXPECT() *mockHead_Expecter { return &mockHead_Expecter{mock: &_m.Mock} } -// BlockDifficulty provides a mock function with given fields: +// BlockDifficulty provides a mock function with no fields func (_m *mockHead) BlockDifficulty() *big.Int { ret := _m.Called() @@ -68,7 +68,7 @@ func (_c *mockHead_BlockDifficulty_Call) RunAndReturn(run func() *big.Int) *mock return _c } -// BlockNumber provides a mock function with given fields: +// BlockNumber provides a mock function with no fields func (_m *mockHead) BlockNumber() int64 { ret := _m.Called() @@ -113,7 +113,7 @@ func (_c *mockHead_BlockNumber_Call) RunAndReturn(run func() int64) *mockHead_Bl return _c } -// GetTotalDifficulty provides a mock function with given fields: +// GetTotalDifficulty provides a mock function with no fields func (_m *mockHead) GetTotalDifficulty() *big.Int { ret := _m.Called() @@ -160,7 +160,7 @@ func (_c *mockHead_GetTotalDifficulty_Call) RunAndReturn(run func() *big.Int) *m return _c } -// IsValid provides a mock function with given fields: +// IsValid provides a mock function with no fields func (_m *mockHead) IsValid() bool { ret := _m.Called() diff --git a/multinode/mock_node_selector_test.go b/multinode/mock_node_selector_test.go index 6613b51..08e4d76 100644 --- a/multinode/mock_node_selector_test.go +++ b/multinode/mock_node_selector_test.go @@ -1,15 +1,15 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode import mock "github.com/stretchr/testify/mock" // mockNodeSelector is an autogenerated mock type for the NodeSelector type -type mockNodeSelector[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockNodeSelector_Expecter[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -17,7 +17,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) EXPECT() *mockNodeSelector_Expecter[C return &mockNodeSelector_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockNodeSelector[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -36,7 +36,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) Name() string { } // mockNodeSelector_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockNodeSelector_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -62,7 +62,7 @@ func (_c *mockNodeSelector_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() str return _c } -// Select provides a mock function with given fields: +// Select provides a mock function with no fields func (_m *mockNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { ret := _m.Called() @@ -83,7 +83,7 @@ func (_m *mockNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { } // mockNodeSelector_Select_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Select' -type mockNodeSelector_Select_Call[CHAIN_ID ID, RPC any] struct { +type mockNodeSelector_Select_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +111,7 @@ func (_c *mockNodeSelector_Select_Call[CHAIN_ID, RPC]) RunAndReturn(run func() N // newMockNodeSelector creates a new instance of mockNodeSelector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockNodeSelector[CHAIN_ID ID, RPC any](t interface { +func newMockNodeSelector[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockNodeSelector[CHAIN_ID, RPC] { diff --git a/multinode/mock_node_test.go b/multinode/mock_node_test.go index 3924591..917fd63 100644 --- a/multinode/mock_node_test.go +++ b/multinode/mock_node_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -9,11 +9,11 @@ import ( ) // mockNode is an autogenerated mock type for the Node type -type mockNode[CHAIN_ID ID, RPC any] struct { +type mockNode[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockNode_Expecter[CHAIN_ID ID, RPC any] struct { +type mockNode_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -21,7 +21,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) EXPECT() *mockNode_Expecter[CHAIN_ID, RPC] { return &mockNode_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Close() error { ret := _m.Called() @@ -40,7 +40,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Close() error { } // mockNode_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type mockNode_Close_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Close_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -66,7 +66,7 @@ func (_c *mockNode_Close_Call[CHAIN_ID, RPC]) RunAndReturn(run func() error) *mo return _c } -// ConfiguredChainID provides a mock function with given fields: +// ConfiguredChainID provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { ret := _m.Called() @@ -78,14 +78,16 @@ func (_m *mockNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { if rf, ok := ret.Get(0).(func() CHAIN_ID); ok { r0 = rf() } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } return r0 } // mockNode_ConfiguredChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConfiguredChainID' -type mockNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +113,7 @@ func (_c *mockNode_ConfiguredChainID_Call[CHAIN_ID, RPC]) RunAndReturn(run func( return _c } -// HighestUserObservations provides a mock function with given fields: +// HighestUserObservations provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { ret := _m.Called() @@ -130,7 +132,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { } // mockNode_HighestUserObservations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HighestUserObservations' -type mockNode_HighestUserObservations_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_HighestUserObservations_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -156,7 +158,7 @@ func (_c *mockNode_HighestUserObservations_Call[CHAIN_ID, RPC]) RunAndReturn(run return _c } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -175,7 +177,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Name() string { } // mockNode_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockNode_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -201,7 +203,7 @@ func (_c *mockNode_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() string) *mo return _c } -// Order provides a mock function with given fields: +// Order provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) Order() int32 { ret := _m.Called() @@ -220,7 +222,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Order() int32 { } // mockNode_Order_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Order' -type mockNode_Order_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Order_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -246,7 +248,7 @@ func (_c *mockNode_Order_Call[CHAIN_ID, RPC]) RunAndReturn(run func() int32) *mo return _c } -// RPC provides a mock function with given fields: +// RPC provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) RPC() RPC { ret := _m.Called() @@ -258,14 +260,16 @@ func (_m *mockNode[CHAIN_ID, RPC]) RPC() RPC { if rf, ok := ret.Get(0).(func() RPC); ok { r0 = rf() } else { - r0 = ret.Get(0).(RPC) + if ret.Get(0) != nil { + r0 = ret.Get(0).(RPC) + } } return r0 } // mockNode_RPC_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RPC' -type mockNode_RPC_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_RPC_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -297,7 +301,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) SetPoolChainInfoProvider(_a0 PoolChainInfoPro } // mockNode_SetPoolChainInfoProvider_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetPoolChainInfoProvider' -type mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -320,7 +324,7 @@ func (_c *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC]) Return() *mockN } func (_c *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC]) RunAndReturn(run func(PoolChainInfoProvider)) *mockNode_SetPoolChainInfoProvider_Call[CHAIN_ID, RPC] { - _c.Call.Return(run) + _c.Run(run) return _c } @@ -343,7 +347,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) Start(_a0 context.Context) error { } // mockNode_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type mockNode_Start_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_Start_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -370,7 +374,7 @@ func (_c *mockNode_Start_Call[CHAIN_ID, RPC]) RunAndReturn(run func(context.Cont return _c } -// State provides a mock function with given fields: +// State provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) State() nodeState { ret := _m.Called() @@ -389,7 +393,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) State() nodeState { } // mockNode_State_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'State' -type mockNode_State_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_State_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -415,7 +419,7 @@ func (_c *mockNode_State_Call[CHAIN_ID, RPC]) RunAndReturn(run func() nodeState) return _c } -// StateAndLatest provides a mock function with given fields: +// StateAndLatest provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) StateAndLatest() (nodeState, ChainInfo) { ret := _m.Called() @@ -444,7 +448,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) StateAndLatest() (nodeState, ChainInfo) { } // mockNode_StateAndLatest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StateAndLatest' -type mockNode_StateAndLatest_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_StateAndLatest_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -470,7 +474,7 @@ func (_c *mockNode_StateAndLatest_Call[CHAIN_ID, RPC]) RunAndReturn(run func() ( return _c } -// String provides a mock function with given fields: +// String provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) String() string { ret := _m.Called() @@ -489,7 +493,7 @@ func (_m *mockNode[CHAIN_ID, RPC]) String() string { } // mockNode_String_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'String' -type mockNode_String_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_String_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -515,13 +519,13 @@ func (_c *mockNode_String_Call[CHAIN_ID, RPC]) RunAndReturn(run func() string) * return _c } -// UnsubscribeAllExceptAliveLoop provides a mock function with given fields: +// UnsubscribeAllExceptAliveLoop provides a mock function with no fields func (_m *mockNode[CHAIN_ID, RPC]) UnsubscribeAllExceptAliveLoop() { _m.Called() } // mockNode_UnsubscribeAllExceptAliveLoop_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnsubscribeAllExceptAliveLoop' -type mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID ID, RPC any] struct { +type mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -543,13 +547,13 @@ func (_c *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC]) Return() * } func (_c *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC]) RunAndReturn(run func()) *mockNode_UnsubscribeAllExceptAliveLoop_Call[CHAIN_ID, RPC] { - _c.Call.Return(run) + _c.Run(run) return _c } // newMockNode creates a new instance of mockNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockNode[CHAIN_ID ID, RPC any](t interface { +func newMockNode[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockNode[CHAIN_ID, RPC] { diff --git a/multinode/mock_pool_chain_info_provider_test.go b/multinode/mock_pool_chain_info_provider_test.go index c857ef8..3dcd76c 100644 --- a/multinode/mock_pool_chain_info_provider_test.go +++ b/multinode/mock_pool_chain_info_provider_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -17,7 +17,7 @@ func (_m *mockPoolChainInfoProvider) EXPECT() *mockPoolChainInfoProvider_Expecte return &mockPoolChainInfoProvider_Expecter{mock: &_m.Mock} } -// HighestUserObservations provides a mock function with given fields: +// HighestUserObservations provides a mock function with no fields func (_m *mockPoolChainInfoProvider) HighestUserObservations() ChainInfo { ret := _m.Called() @@ -62,7 +62,7 @@ func (_c *mockPoolChainInfoProvider_HighestUserObservations_Call) RunAndReturn(r return _c } -// LatestChainInfo provides a mock function with given fields: +// LatestChainInfo provides a mock function with no fields func (_m *mockPoolChainInfoProvider) LatestChainInfo() (int, ChainInfo) { ret := _m.Called() diff --git a/multinode/mock_rpc_client_test.go b/multinode/mock_rpc_client_test.go index 0dbdc1a..6b55d4c 100644 --- a/multinode/mock_rpc_client_test.go +++ b/multinode/mock_rpc_client_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -37,7 +37,9 @@ func (_m *mockRPCClient[CHAIN_ID, HEAD]) ChainID(ctx context.Context) (CHAIN_ID, if rf, ok := ret.Get(0).(func(context.Context) CHAIN_ID); ok { r0 = rf(ctx) } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } if rf, ok := ret.Get(1).(func(context.Context) error); ok { @@ -77,7 +79,7 @@ func (_c *mockRPCClient_ChainID_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(cont return _c } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockRPCClient[CHAIN_ID, HEAD]) Close() { _m.Called() } @@ -105,7 +107,7 @@ func (_c *mockRPCClient_Close_Call[CHAIN_ID, HEAD]) Return() *mockRPCClient_Clos } func (_c *mockRPCClient_Close_Call[CHAIN_ID, HEAD]) RunAndReturn(run func()) *mockRPCClient_Close_Call[CHAIN_ID, HEAD] { - _c.Call.Return(run) + _c.Run(run) return _c } @@ -155,7 +157,7 @@ func (_c *mockRPCClient_Dial_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(context return _c } -// GetInterceptedChainInfo provides a mock function with given fields: +// GetInterceptedChainInfo provides a mock function with no fields func (_m *mockRPCClient[CHAIN_ID, HEAD]) GetInterceptedChainInfo() (ChainInfo, ChainInfo) { ret := _m.Called() @@ -488,7 +490,7 @@ func (_c *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD]) Return() *moc } func (_c *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD]) RunAndReturn(run func(...Subscription)) *mockRPCClient_UnsubscribeAllExcept_Call[CHAIN_ID, HEAD] { - _c.Call.Return(run) + _c.Run(run) return _c } diff --git a/multinode/mock_send_only_client_test.go b/multinode/mock_send_only_client_test.go index 5c08506..2360b53 100644 --- a/multinode/mock_send_only_client_test.go +++ b/multinode/mock_send_only_client_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -37,7 +37,9 @@ func (_m *mockSendOnlyClient[CHAIN_ID]) ChainID(_a0 context.Context) (CHAIN_ID, if rf, ok := ret.Get(0).(func(context.Context) CHAIN_ID); ok { r0 = rf(_a0) } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } if rf, ok := ret.Get(1).(func(context.Context) error); ok { @@ -77,7 +79,7 @@ func (_c *mockSendOnlyClient_ChainID_Call[CHAIN_ID]) RunAndReturn(run func(conte return _c } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockSendOnlyClient[CHAIN_ID]) Close() { _m.Called() } @@ -105,7 +107,7 @@ func (_c *mockSendOnlyClient_Close_Call[CHAIN_ID]) Return() *mockSendOnlyClient_ } func (_c *mockSendOnlyClient_Close_Call[CHAIN_ID]) RunAndReturn(run func()) *mockSendOnlyClient_Close_Call[CHAIN_ID] { - _c.Call.Return(run) + _c.Run(run) return _c } diff --git a/multinode/mock_send_only_node_test.go b/multinode/mock_send_only_node_test.go index e76b053..45d55c9 100644 --- a/multinode/mock_send_only_node_test.go +++ b/multinode/mock_send_only_node_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -9,11 +9,11 @@ import ( ) // mockSendOnlyNode is an autogenerated mock type for the SendOnlyNode type -type mockSendOnlyNode[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode[CHAIN_ID ID, RPC interface{}] struct { mock.Mock } -type mockSendOnlyNode_Expecter[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Expecter[CHAIN_ID ID, RPC interface{}] struct { mock *mock.Mock } @@ -21,7 +21,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) EXPECT() *mockSendOnlyNode_Expecter[C return &mockSendOnlyNode_Expecter[CHAIN_ID, RPC]{mock: &_m.Mock} } -// Close provides a mock function with given fields: +// Close provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Close() error { ret := _m.Called() @@ -40,7 +40,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Close() error { } // mockSendOnlyNode_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type mockSendOnlyNode_Close_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Close_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -66,7 +66,7 @@ func (_c *mockSendOnlyNode_Close_Call[CHAIN_ID, RPC]) RunAndReturn(run func() er return _c } -// ConfiguredChainID provides a mock function with given fields: +// ConfiguredChainID provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { ret := _m.Called() @@ -78,14 +78,16 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { if rf, ok := ret.Get(0).(func() CHAIN_ID); ok { r0 = rf() } else { - r0 = ret.Get(0).(CHAIN_ID) + if ret.Get(0) != nil { + r0 = ret.Get(0).(CHAIN_ID) + } } return r0 } // mockSendOnlyNode_ConfiguredChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConfiguredChainID' -type mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -111,7 +113,7 @@ func (_c *mockSendOnlyNode_ConfiguredChainID_Call[CHAIN_ID, RPC]) RunAndReturn(r return _c } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Name() string { ret := _m.Called() @@ -130,7 +132,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Name() string { } // mockSendOnlyNode_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' -type mockSendOnlyNode_Name_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Name_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -156,7 +158,7 @@ func (_c *mockSendOnlyNode_Name_Call[CHAIN_ID, RPC]) RunAndReturn(run func() str return _c } -// RPC provides a mock function with given fields: +// RPC provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { ret := _m.Called() @@ -168,14 +170,16 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { if rf, ok := ret.Get(0).(func() RPC); ok { r0 = rf() } else { - r0 = ret.Get(0).(RPC) + if ret.Get(0) != nil { + r0 = ret.Get(0).(RPC) + } } return r0 } // mockSendOnlyNode_RPC_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RPC' -type mockSendOnlyNode_RPC_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_RPC_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -220,7 +224,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) Start(_a0 context.Context) error { } // mockSendOnlyNode_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type mockSendOnlyNode_Start_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_Start_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -247,7 +251,7 @@ func (_c *mockSendOnlyNode_Start_Call[CHAIN_ID, RPC]) RunAndReturn(run func(cont return _c } -// State provides a mock function with given fields: +// State provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) State() nodeState { ret := _m.Called() @@ -266,7 +270,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) State() nodeState { } // mockSendOnlyNode_State_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'State' -type mockSendOnlyNode_State_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_State_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -292,7 +296,7 @@ func (_c *mockSendOnlyNode_State_Call[CHAIN_ID, RPC]) RunAndReturn(run func() no return _c } -// String provides a mock function with given fields: +// String provides a mock function with no fields func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) String() string { ret := _m.Called() @@ -311,7 +315,7 @@ func (_m *mockSendOnlyNode[CHAIN_ID, RPC]) String() string { } // mockSendOnlyNode_String_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'String' -type mockSendOnlyNode_String_Call[CHAIN_ID ID, RPC any] struct { +type mockSendOnlyNode_String_Call[CHAIN_ID ID, RPC interface{}] struct { *mock.Call } @@ -339,7 +343,7 @@ func (_c *mockSendOnlyNode_String_Call[CHAIN_ID, RPC]) RunAndReturn(run func() s // newMockSendOnlyNode creates a new instance of mockSendOnlyNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockSendOnlyNode[CHAIN_ID ID, RPC any](t interface { +func newMockSendOnlyNode[CHAIN_ID ID, RPC interface{}](t interface { mock.TestingT Cleanup(func()) }) *mockSendOnlyNode[CHAIN_ID, RPC] { diff --git a/multinode/mock_subscription_test.go b/multinode/mock_subscription_test.go index ccb9017..1973c92 100644 --- a/multinode/mock_subscription_test.go +++ b/multinode/mock_subscription_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. +// Code generated by mockery v2.53.0. DO NOT EDIT. package multinode @@ -17,7 +17,7 @@ func (_m *mockSubscription) EXPECT() *mockSubscription_Expecter { return &mockSubscription_Expecter{mock: &_m.Mock} } -// Err provides a mock function with given fields: +// Err provides a mock function with no fields func (_m *mockSubscription) Err() <-chan error { ret := _m.Called() @@ -64,7 +64,7 @@ func (_c *mockSubscription_Err_Call) RunAndReturn(run func() <-chan error) *mock return _c } -// Unsubscribe provides a mock function with given fields: +// Unsubscribe provides a mock function with no fields func (_m *mockSubscription) Unsubscribe() { _m.Called() } @@ -92,7 +92,7 @@ func (_c *mockSubscription_Unsubscribe_Call) Return() *mockSubscription_Unsubscr } func (_c *mockSubscription_Unsubscribe_Call) RunAndReturn(run func()) *mockSubscription_Unsubscribe_Call { - _c.Call.Return(run) + _c.Run(run) return _c } From 32c9d4d555e1c27bf8adfbbf0706a35f445fe6f3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 10:03:24 -0400 Subject: [PATCH 14/16] Add network to node fsm metrics --- metrics/multinode.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/metrics/multinode.go b/metrics/multinode.go index 1103b6b..5717253 100644 --- a/metrics/multinode.go +++ b/metrics/multinode.go @@ -33,36 +33,35 @@ var ( Help: "The total number of successful chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) - // TODO: Should these all have network as well? // Node State Transitions promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_alive", Help: "Total number of times node has transitioned to Alive", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_in_sync", Help: "Total number of times node has transitioned from OutOfSync to Alive", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_out_of_sync", Help: "Total number of times node has transitioned to OutOfSync", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_unreachable", Help: "Total number of times node has transitioned to Unreachable", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", Help: "Total number of times node has transitioned to InvalidChainID", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_unusable", Help: "Total number of times node has transitioned to Unusable", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_syncing", Help: "Total number of times node has transitioned to Syncing", - }, []string{"chainID", "nodeName"}) + }, []string{"network", "chainID", "nodeName"}) // Transaction Sender promMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ From bbff8b5641b72245f1fc033860c6e7e600980035 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 10:42:24 -0400 Subject: [PATCH 15/16] Mock test metrics --- multinode/.mockery.yaml | 4 + multinode/mock_multi_node_metrics_test.go | 71 ++++ multinode/mock_node_metrics_test.go | 376 ++++++++++++++++++ .../mock_transaction_sender_metrics_test.go | 70 ++++ multinode/multi_node_test.go | 8 +- multinode/node_test.go | 19 +- multinode/send_only_node_test.go | 12 +- multinode/transaction_sender_test.go | 16 +- 8 files changed, 564 insertions(+), 12 deletions(-) create mode 100644 multinode/mock_multi_node_metrics_test.go create mode 100644 multinode/mock_node_metrics_test.go create mode 100644 multinode/mock_transaction_sender_metrics_test.go diff --git a/multinode/.mockery.yaml b/multinode/.mockery.yaml index 84e8332..91aef66 100644 --- a/multinode/.mockery.yaml +++ b/multinode/.mockery.yaml @@ -18,3 +18,7 @@ packages: Head: PoolChainInfoProvider: Subscription: + multiNodeMetrics: + nodeMetrics: + sendOnlyNodeMetrics: + transactionSenderMetrics: diff --git a/multinode/mock_multi_node_metrics_test.go b/multinode/mock_multi_node_metrics_test.go new file mode 100644 index 0000000..0537463 --- /dev/null +++ b/multinode/mock_multi_node_metrics_test.go @@ -0,0 +1,71 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockMultiNodeMetrics is an autogenerated mock type for the multiNodeMetrics type +type mockMultiNodeMetrics struct { + mock.Mock +} + +type mockMultiNodeMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockMultiNodeMetrics) EXPECT() *mockMultiNodeMetrics_Expecter { + return &mockMultiNodeMetrics_Expecter{mock: &_m.Mock} +} + +// RecordNodeStates provides a mock function with given fields: ctx, state, count +func (_m *mockMultiNodeMetrics) RecordNodeStates(ctx context.Context, state string, count int64) { + _m.Called(ctx, state, count) +} + +// mockMultiNodeMetrics_RecordNodeStates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordNodeStates' +type mockMultiNodeMetrics_RecordNodeStates_Call struct { + *mock.Call +} + +// RecordNodeStates is a helper method to define mock.On call +// - ctx context.Context +// - state string +// - count int64 +func (_e *mockMultiNodeMetrics_Expecter) RecordNodeStates(ctx interface{}, state interface{}, count interface{}) *mockMultiNodeMetrics_RecordNodeStates_Call { + return &mockMultiNodeMetrics_RecordNodeStates_Call{Call: _e.mock.On("RecordNodeStates", ctx, state, count)} +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) Run(run func(ctx context.Context, state string, count int64)) *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int64)) + }) + return _c +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) Return() *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Call.Return() + return _c +} + +func (_c *mockMultiNodeMetrics_RecordNodeStates_Call) RunAndReturn(run func(context.Context, string, int64)) *mockMultiNodeMetrics_RecordNodeStates_Call { + _c.Run(run) + return _c +} + +// newMockMultiNodeMetrics creates a new instance of mockMultiNodeMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockMultiNodeMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockMultiNodeMetrics { + mock := &mockMultiNodeMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/mock_node_metrics_test.go b/multinode/mock_node_metrics_test.go new file mode 100644 index 0000000..fc28931 --- /dev/null +++ b/multinode/mock_node_metrics_test.go @@ -0,0 +1,376 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockNodeMetrics is an autogenerated mock type for the nodeMetrics type +type mockNodeMetrics struct { + mock.Mock +} + +type mockNodeMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockNodeMetrics) EXPECT() *mockNodeMetrics_Expecter { + return &mockNodeMetrics_Expecter{mock: &_m.Mock} +} + +// IncrementNodeTransitionsToAlive provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToAlive(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToAlive_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToAlive' +type mockNodeMetrics_IncrementNodeTransitionsToAlive_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToAlive is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToAlive(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToAlive_Call{Call: _e.mock.On("IncrementNodeTransitionsToAlive", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToAlive_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToInSync provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToInSync(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToInSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToInSync' +type mockNodeMetrics_IncrementNodeTransitionsToInSync_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToInSync is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToInSync(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToInSync_Call{Call: _e.mock.On("IncrementNodeTransitionsToInSync", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToInSync_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToInvalidChainID provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToInvalidChainID' +type mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToInvalidChainID is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToInvalidChainID(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call{Call: _e.mock.On("IncrementNodeTransitionsToInvalidChainID", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToInvalidChainID_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToOutOfSync provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToOutOfSync(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToOutOfSync' +type mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToOutOfSync is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToOutOfSync(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call{Call: _e.mock.On("IncrementNodeTransitionsToOutOfSync", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToOutOfSync_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToSyncing provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToSyncing' +type mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToSyncing is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToSyncing(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call{Call: _e.mock.On("IncrementNodeTransitionsToSyncing", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToSyncing_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToUnreachable provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToUnreachable(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToUnreachable' +type mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToUnreachable is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToUnreachable(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call{Call: _e.mock.On("IncrementNodeTransitionsToUnreachable", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToUnreachable_Call { + _c.Run(run) + return _c +} + +// IncrementNodeTransitionsToUnusable provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeTransitionsToUnusable' +type mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call struct { + *mock.Call +} + +// IncrementNodeTransitionsToUnusable is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeTransitionsToUnusable(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + return &mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call{Call: _e.mock.On("IncrementNodeTransitionsToUnusable", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) Return() *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeTransitionsToUnusable_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifies provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifies(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifies_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifies' +type mockNodeMetrics_IncrementNodeVerifies_Call struct { + *mock.Call +} + +// IncrementNodeVerifies is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifies(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifies_Call { + return &mockNodeMetrics_IncrementNodeVerifies_Call{Call: _e.mock.On("IncrementNodeVerifies", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) Return() *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifies_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifies_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifiesFailed provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifiesFailed(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifiesFailed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifiesFailed' +type mockNodeMetrics_IncrementNodeVerifiesFailed_Call struct { + *mock.Call +} + +// IncrementNodeVerifiesFailed is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifiesFailed(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + return &mockNodeMetrics_IncrementNodeVerifiesFailed_Call{Call: _e.mock.On("IncrementNodeVerifiesFailed", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) Return() *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesFailed_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifiesFailed_Call { + _c.Run(run) + return _c +} + +// IncrementNodeVerifiesSuccess provides a mock function with given fields: ctx, nodeName +func (_m *mockNodeMetrics) IncrementNodeVerifiesSuccess(ctx context.Context, nodeName string) { + _m.Called(ctx, nodeName) +} + +// mockNodeMetrics_IncrementNodeVerifiesSuccess_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementNodeVerifiesSuccess' +type mockNodeMetrics_IncrementNodeVerifiesSuccess_Call struct { + *mock.Call +} + +// IncrementNodeVerifiesSuccess is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *mockNodeMetrics_Expecter) IncrementNodeVerifiesSuccess(ctx interface{}, nodeName interface{}) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + return &mockNodeMetrics_IncrementNodeVerifiesSuccess_Call{Call: _e.mock.On("IncrementNodeVerifiesSuccess", ctx, nodeName)} +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) Run(run func(ctx context.Context, nodeName string)) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) Return() *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Call.Return() + return _c +} + +func (_c *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call) RunAndReturn(run func(context.Context, string)) *mockNodeMetrics_IncrementNodeVerifiesSuccess_Call { + _c.Run(run) + return _c +} + +// newMockNodeMetrics creates a new instance of mockNodeMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockNodeMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockNodeMetrics { + mock := &mockNodeMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/mock_transaction_sender_metrics_test.go b/multinode/mock_transaction_sender_metrics_test.go new file mode 100644 index 0000000..572e9ea --- /dev/null +++ b/multinode/mock_transaction_sender_metrics_test.go @@ -0,0 +1,70 @@ +// Code generated by mockery v2.53.0. DO NOT EDIT. + +package multinode + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockTransactionSenderMetrics is an autogenerated mock type for the transactionSenderMetrics type +type mockTransactionSenderMetrics struct { + mock.Mock +} + +type mockTransactionSenderMetrics_Expecter struct { + mock *mock.Mock +} + +func (_m *mockTransactionSenderMetrics) EXPECT() *mockTransactionSenderMetrics_Expecter { + return &mockTransactionSenderMetrics_Expecter{mock: &_m.Mock} +} + +// IncrementInvariantViolations provides a mock function with given fields: ctx, invariant +func (_m *mockTransactionSenderMetrics) IncrementInvariantViolations(ctx context.Context, invariant string) { + _m.Called(ctx, invariant) +} + +// mockTransactionSenderMetrics_IncrementInvariantViolations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementInvariantViolations' +type mockTransactionSenderMetrics_IncrementInvariantViolations_Call struct { + *mock.Call +} + +// IncrementInvariantViolations is a helper method to define mock.On call +// - ctx context.Context +// - invariant string +func (_e *mockTransactionSenderMetrics_Expecter) IncrementInvariantViolations(ctx interface{}, invariant interface{}) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + return &mockTransactionSenderMetrics_IncrementInvariantViolations_Call{Call: _e.mock.On("IncrementInvariantViolations", ctx, invariant)} +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) Run(run func(ctx context.Context, invariant string)) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) Return() *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Call.Return() + return _c +} + +func (_c *mockTransactionSenderMetrics_IncrementInvariantViolations_Call) RunAndReturn(run func(context.Context, string)) *mockTransactionSenderMetrics_IncrementInvariantViolations_Call { + _c.Run(run) + return _c +} + +// newMockTransactionSenderMetrics creates a new instance of mockTransactionSenderMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockTransactionSenderMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *mockTransactionSenderMetrics { + mock := &mockTransactionSenderMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/multinode/multi_node_test.go b/multinode/multi_node_test.go index 6c0e659..89b54f1 100644 --- a/multinode/multi_node_test.go +++ b/multinode/multi_node_test.go @@ -41,12 +41,18 @@ func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode { } result := NewMultiNode[ID, multiNodeRPCClient]( - opts.logger, opts.selectionMode, opts.leaseDuration, opts.nodes, opts.sendonlys, opts.chainID, opts.chainFamily, opts.deathDeclarationDelay) + opts.logger, makeMockMultiNodeMetrics(t), opts.selectionMode, opts.leaseDuration, opts.nodes, opts.sendonlys, opts.chainID, opts.chainFamily, opts.deathDeclarationDelay) return testMultiNode{ result, } } +func makeMockMultiNodeMetrics(t *testing.T) *mockMultiNodeMetrics { + mockMetrics := newMockMultiNodeMetrics(t) + mockMetrics.On("RecordNodeStates", mock.Anything, mock.Anything, mock.Anything).Maybe() + return mockMetrics +} + func newHealthyNode(t *testing.T, chainID ID) *mockNode[ID, multiNodeRPCClient] { return newNodeWithState(t, chainID, nodeStateAlive) } diff --git a/multinode/node_test.go b/multinode/node_test.go index a6a7f2e..b1ff8f8 100644 --- a/multinode/node_test.go +++ b/multinode/node_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/mock" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-framework/multinode/mocks" @@ -101,10 +103,25 @@ func newTestNode(t *testing.T, opts testNodeOpts) testNode { opts.id = 42 } - nodeI := NewNode[ID, Head, RPCClient[ID, Head]](opts.config, opts.chainConfig, opts.lggr, + nodeI := NewNode[ID, Head, RPCClient[ID, Head]](opts.config, opts.chainConfig, opts.lggr, makeMockNodeMetrics(t), opts.wsuri, opts.httpuri, opts.name, opts.id, opts.chainID, opts.nodeOrder, opts.rpc, opts.chainFamily) return testNode{ nodeI.(*node[ID, Head, RPCClient[ID, Head]]), } } + +func makeMockNodeMetrics(t *testing.T) *mockNodeMetrics { + mockMetrics := newMockNodeMetrics(t) + mockMetrics.On("IncrementNodeVerifies", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeVerifiesFailed", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeVerifiesSuccess", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToAlive", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToInSync", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToOutOfSync", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToUnreachable", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToInvalidChainID", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToUnusable", mock.Anything, mock.Anything).Maybe() + mockMetrics.On("IncrementNodeTransitionsToSyncing", mock.Anything, mock.Anything).Maybe() + return mockMetrics +} diff --git a/multinode/send_only_node_test.go b/multinode/send_only_node_test.go index 646796e..8d01953 100644 --- a/multinode/send_only_node_test.go +++ b/multinode/send_only_node_test.go @@ -28,7 +28,7 @@ func TestNewSendOnlyNode(t *testing.T) { chainID := RandomID() client := newMockSendOnlyClient[ID](t) - node := NewSendOnlyNode(lggr, *u, name, chainID, client) + node := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), *u, name, chainID, client) assert.NotNil(t, node) // Must contain name & url with redacted password @@ -45,7 +45,7 @@ func TestStartSendOnlyNode(t *testing.T) { client.On("Close").Once() expectedError := errors.New("some http error") client.On("Dial", mock.Anything).Return(expectedError).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), RandomID(), client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), RandomID(), client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -61,7 +61,7 @@ func TestStartSendOnlyNode(t *testing.T) { client := newMockSendOnlyClient[ID](t) client.On("Close").Once() client.On("Dial", mock.Anything).Return(nil).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), NewIDFromInt(0), client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), NewIDFromInt(0), client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -82,7 +82,7 @@ func TestStartSendOnlyNode(t *testing.T) { const failuresCount = 2 client.On("ChainID", mock.Anything).Return(RandomID(), expectedError).Times(failuresCount) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), chainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), chainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -105,7 +105,7 @@ func TestStartSendOnlyNode(t *testing.T) { const failuresCount = 2 client.On("ChainID", mock.Anything).Return(rpcChainID, nil).Times(failuresCount) client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), configuredChainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) @@ -126,7 +126,7 @@ func TestStartSendOnlyNode(t *testing.T) { client.On("Dial", mock.Anything).Return(nil).Once() configuredChainID := RandomID() client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + s := NewSendOnlyNode(lggr, makeMockNodeMetrics(t), url.URL{}, t.Name(), configuredChainID, client) defer func() { assert.NoError(t, s.Close()) }() err := s.Start(tests.Context(t)) diff --git a/multinode/transaction_sender_test.go b/multinode/transaction_sender_test.go index 8f88dd1..a78c24e 100644 --- a/multinode/transaction_sender_test.go +++ b/multinode/transaction_sender_test.go @@ -47,13 +47,20 @@ func newTestTransactionSender(t *testing.T, chainID ID, lggr logger.Logger, sendOnlyNodes []SendOnlyNode[ID, TestSendTxRPCClient], ) (*sendTxMultiNode, *TransactionSender[any, any, ID, TestSendTxRPCClient]) { mn := sendTxMultiNode{NewMultiNode[ID, TestSendTxRPCClient]( - lggr, NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} + lggr, makeMockMultiNodeMetrics(t), NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} - txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) + txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, makeMockTxSenderMetrics(t), + func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) servicetest.Run(t, txSender) return &mn, txSender } +func makeMockTxSenderMetrics(t *testing.T) *mockTransactionSenderMetrics { + metrics := newMockTransactionSenderMetrics(t) + metrics.On("IncrementInvariantViolations", mock.Anything, mock.Anything).Maybe() + return metrics +} + func classifySendTxError(_ any, err error) SendTxReturnCode { if err != nil { return Fatal @@ -148,8 +155,9 @@ func TestTransactionSender_SendTransaction(t *testing.T) { chainID := RandomID() mn := sendTxMultiNode{NewMultiNode[ID, TestSendTxRPCClient]( - lggr, NodeSelectionModeRoundRobin, 0, []Node[ID, TestSendTxRPCClient]{mainNode}, nil, chainID, "chainFamily", 0)} - txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) + lggr, makeMockMultiNodeMetrics(t), NodeSelectionModeRoundRobin, 0, []Node[ID, TestSendTxRPCClient]{mainNode}, nil, chainID, "chainFamily", 0)} + txSender := NewTransactionSender[any, any, ID, TestSendTxRPCClient](lggr, chainID, mn.chainFamily, mn.MultiNode, makeMockTxSenderMetrics(t), + func(err error) SendTxReturnCode { return 0 }, tests.TestInterval) require.NoError(t, txSender.Start(tests.Context(t))) _, _, err := txSender.SendTransaction(requestContext, nil) From bc2836f39f5e44910e70fe0d59acd10842765290 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Apr 2025 11:06:11 -0400 Subject: [PATCH 16/16] Add test coverage --- metrics/go.mod | 5 ++ metrics/go.sum | 10 +++ metrics/logpoller_test.go | 61 ++++++++++++++++++ metrics/multinode_test.go | 129 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+) create mode 100644 metrics/logpoller_test.go create mode 100644 metrics/multinode_test.go diff --git a/metrics/go.mod b/metrics/go.mod index c0a0b52..52742bd 100644 --- a/metrics/go.mod +++ b/metrics/go.mod @@ -5,6 +5,7 @@ go 1.24.1 require ( github.com/prometheus/client_golang v1.22.0 github.com/smartcontractkit/chainlink-common v0.7.0 + github.com/stretchr/testify v1.10.0 go.opentelemetry.io/otel v1.35.0 go.opentelemetry.io/otel/metric v1.35.0 ) @@ -15,6 +16,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect github.com/cloudevents/sdk-go/v2 v2.16.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.13.0 // indirect @@ -23,11 +25,13 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.63.0 // indirect github.com/prometheus/procfs v0.16.0 // indirect @@ -59,4 +63,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect google.golang.org/grpc v1.71.0 // indirect google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/metrics/go.sum b/metrics/go.sum index 13563ef..8cb99f7 100644 --- a/metrics/go.sum +++ b/metrics/go.sum @@ -36,6 +36,12 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0Ntos github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -58,6 +64,8 @@ github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/smartcontractkit/chainlink-common v0.7.0 h1:QThOrHKn+du8CTmzJPCha0Nwnvw0tonIEAQca+dnmE0= github.com/smartcontractkit/chainlink-common v0.7.0/go.mod h1:pptbsF6z90IGCewkCgDMBxNYjfSOyW9X9l2jzYyQgmk= github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 h1:PKiqnVOTChlH4a4ljJKL3OKGRgYfIpJS4YD1daAIKks= @@ -142,6 +150,8 @@ google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/logpoller_test.go b/metrics/logpoller_test.go new file mode 100644 index 0000000..2c34bc3 --- /dev/null +++ b/metrics/logpoller_test.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func setupTestLogPollerMetrics(t *testing.T) GenericLogPollerMetrics { + m, err := NewGenericLogPollerMetrics("1", "test-network") + require.NoError(t, err) + return m +} + +func TestLogPollerMetrics_RecordQueryDuration(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.RecordQueryDuration(ctx, "PollLogs", Read, 0.005) // 5ms + + // Collect and make sure at least one sample was recorded + require.Greater(t, testutil.CollectAndCount(PromLpQueryDuration), 0) +} + +func TestLogPollerMetrics_RecordQueryDatasetSize(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.RecordQueryDatasetSize(ctx, "PollLogs", Read, 10) + + require.Equal(t, + float64(10), + testutil.ToFloat64(PromLpQueryDataSets.WithLabelValues("test-network", "1", "PollLogs", "read")), + ) +} + +func TestLogPollerMetrics_IncrementLogsInserted(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.IncrementLogsInserted(ctx, 5) + + require.Equal(t, + float64(5), + testutil.ToFloat64(PromLpLogsInserted.WithLabelValues("test-network", "1")), + ) +} + +func TestLogPollerMetrics_IncrementBlocksInserted(t *testing.T) { + m := setupTestLogPollerMetrics(t) + ctx := context.Background() + + m.IncrementBlocksInserted(ctx, 3) + + require.Equal(t, + float64(3), + testutil.ToFloat64(PromLpBlocksInserted.WithLabelValues("test-network", "1")), + ) +} diff --git a/metrics/multinode_test.go b/metrics/multinode_test.go new file mode 100644 index 0000000..eecb2ae --- /dev/null +++ b/metrics/multinode_test.go @@ -0,0 +1,129 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func setupTestMultiNodeMetrics(t *testing.T) GenericMultiNodeMetrics { + m, err := NewGenericMultiNodeMetrics("test-network", "1") + require.NoError(t, err) + return m +} + +func TestMultiNodeMetrics_RecordNodeStates(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.RecordNodeStates(ctx, "Alive", 5) + + require.Equal(t, float64(5), + testutil.ToFloat64(promMultiNodeRPCNodeStates.WithLabelValues("test-network", "1", "Alive")), + ) +} + +func TestMultiNodeMetrics_Verifies(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.IncrementNodeVerifies(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifies.WithLabelValues("test-network", "1", "node-1")), + ) + + m.IncrementNodeVerifiesFailed(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifiesFailed.WithLabelValues("test-network", "1", "node-1")), + ) + + m.IncrementNodeVerifiesSuccess(ctx, "node-1") + require.Equal(t, float64(1), + testutil.ToFloat64(promPoolRPCNodeVerifiesSuccess.WithLabelValues("test-network", "1", "node-1")), + ) +} + +func TestMultiNodeMetrics_NodeTransitions(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + nodeName := "node-1" + + tests := []struct { + name string + increment func() + promMetric *prometheus.CounterVec + }{ + { + name: "Alive", + increment: func() { + m.IncrementNodeTransitionsToAlive(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToAlive, + }, + { + name: "InSync", + increment: func() { + m.IncrementNodeTransitionsToInSync(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToInSync, + }, + { + name: "OutOfSync", + increment: func() { + m.IncrementNodeTransitionsToOutOfSync(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToOutOfSync, + }, + { + name: "Unreachable", + increment: func() { + m.IncrementNodeTransitionsToUnreachable(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToUnreachable, + }, + { + name: "InvalidChainID", + increment: func() { + m.IncrementNodeTransitionsToInvalidChainID(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToInvalidChainID, + }, + { + name: "Unusable", + increment: func() { + m.IncrementNodeTransitionsToUnusable(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToUnusable, + }, + { + name: "Syncing", + increment: func() { + m.IncrementNodeTransitionsToSyncing(ctx, nodeName) + }, + promMetric: promPoolRPCNodeTransitionsToSyncing, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.increment() + require.Equal(t, float64(1), + testutil.ToFloat64(tt.promMetric.WithLabelValues("test-network", "1", nodeName)), + ) + }) + } +} + +func TestMultiNodeMetrics_IncrementInvariantViolations(t *testing.T) { + m := setupTestMultiNodeMetrics(t) + ctx := context.Background() + + m.IncrementInvariantViolations(ctx, "wrong_nonce") + + require.Equal(t, float64(1), + testutil.ToFloat64(promMultiNodeInvariantViolations.WithLabelValues("test-network", "1", "wrong_nonce")), + ) +}