Skip to content

Commit 2191d79

Browse files
cll-ggcll-gg
authored
OEV-832 Ingest block_history_estimator and fee_history_estimator metrics into Beholder (#393)
1 parent d8b68e7 commit 2191d79

5 files changed

Lines changed: 335 additions & 134 deletions

‎pkg/gas/block_history_estimator.go‎

Lines changed: 31 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ import (
1313
"github.com/ethereum/go-ethereum/common"
1414
"github.com/ethereum/go-ethereum/common/hexutil"
1515
"github.com/ethereum/go-ethereum/rpc"
16-
"github.com/prometheus/client_golang/prometheus"
17-
"github.com/prometheus/client_golang/prometheus/promauto"
1816

1917
"github.com/smartcontractkit/chainlink-common/pkg/logger"
2018
"github.com/smartcontractkit/chainlink-common/pkg/services"
@@ -35,45 +33,6 @@ import (
3533
// block the application from starting.
3634
var MaxStartTime = 10 * time.Second
3735

38-
var (
39-
promBlockHistoryEstimatorAllGasPricePercentiles = promauto.NewGaugeVec(prometheus.GaugeOpts{
40-
Name: "gas_updater_all_gas_price_percentiles",
41-
Help: "Gas price at given percentile",
42-
},
43-
[]string{"percentile", "evmChainID"},
44-
)
45-
promBlockHistoryEstimatorAllTipCapPercentiles = promauto.NewGaugeVec(prometheus.GaugeOpts{
46-
Name: "gas_updater_all_tip_cap_percentiles",
47-
Help: "Tip cap at given percentile",
48-
},
49-
[]string{"percentile", "evmChainID"},
50-
)
51-
promBlockHistoryEstimatorSetGasPrice = promauto.NewGaugeVec(prometheus.GaugeOpts{
52-
Name: "gas_updater_set_gas_price",
53-
Help: "Gas updater set gas price (in Wei)",
54-
},
55-
[]string{"percentile", "evmChainID"},
56-
)
57-
promBlockHistoryEstimatorSetTipCap = promauto.NewGaugeVec(prometheus.GaugeOpts{
58-
Name: "gas_updater_set_tip_cap",
59-
Help: "Gas updater set gas tip cap (in Wei)",
60-
},
61-
[]string{"percentile", "evmChainID"},
62-
)
63-
promBlockHistoryEstimatorCurrentBaseFee = promauto.NewGaugeVec(prometheus.GaugeOpts{
64-
Name: "gas_updater_current_base_fee",
65-
Help: "Gas updater current block base fee in Wei",
66-
},
67-
[]string{"evmChainID"},
68-
)
69-
promBlockHistoryEstimatorConnectivityFailureCount = promauto.NewCounterVec(prometheus.CounterOpts{
70-
Name: "block_history_estimator_connectivity_failure_count",
71-
Help: "Counter is incremented every time a gas bump is prevented due to a detected network propagation/connectivity issue",
72-
},
73-
[]string{"evmChainID", "mode"},
74-
)
75-
)
76-
7736
const BumpingHaltedLabel = "Tx gas bumping halted since price exceeds current block prices by significant margin; tx will continue to be rebroadcasted but your node, RPC, or the chain might be experiencing connectivity issues; please investigate and fix ASAP"
7837

7938
var _ EvmEstimator = &BlockHistoryEstimator{}
@@ -116,13 +75,16 @@ type BlockHistoryEstimator struct {
11675
logger logger.SugaredLogger
11776

11877
l1Oracle rollups.L1Oracle
78+
79+
metrics *blockHistoryEstimatorMetrics
11980
}
12081

12182
// NewBlockHistoryEstimator returns a new BlockHistoryEstimator that listens
12283
// for new heads and updates the base gas price dynamically based on the
12384
// configured percentile of gas prices in that block
12485
func NewBlockHistoryEstimator(lggr logger.Logger, ethClient FeeEstimatorClient, chaintype chaintype.ChainType, eCfg BlockHistoryEstimatorConfig, bhCfg evmconfig.BlockHistory, chainID *big.Int, l1Oracle rollups.L1Oracle) EvmEstimator {
125-
return &BlockHistoryEstimator{
86+
l := logger.Sugared(logger.Named(lggr, "BlockHistoryEstimator"))
87+
b := &BlockHistoryEstimator{
12688
ethClient: ethClient,
12789
chainID: chainID,
12890
chaintype: chaintype,
@@ -134,26 +96,29 @@ func NewBlockHistoryEstimator(lggr logger.Logger, ethClient FeeEstimatorClient,
13496
mb: mailbox.NewSingle[*evmtypes.Head](),
13597
wg: new(sync.WaitGroup),
13698
stopCh: make(chan struct{}),
137-
logger: logger.Sugared(logger.Named(lggr, "BlockHistoryEstimator")),
99+
logger: l,
138100
l1Oracle: l1Oracle,
101+
metrics: newBlockHistoryEstimatorMetrics(l, chainID),
139102
}
103+
return b
140104
}
141105

142106
// OnNewLongestChain recalculates and sets global gas price if a sampled new head comes
143107
// in and we are not currently fetching
144-
func (b *BlockHistoryEstimator) OnNewLongestChain(_ context.Context, head *evmtypes.Head) {
108+
func (b *BlockHistoryEstimator) OnNewLongestChain(ctx context.Context, head *evmtypes.Head) {
145109
// set latest base fee here to avoid potential lag introduced by block delay
146110
// it is really important that base fee be as up-to-date as possible
147-
b.setLatest(head)
111+
b.setLatest(ctx, head)
148112
b.mb.Deliver(head)
149113
}
150114

151115
// setLatest assumes that head won't be mutated
152-
func (b *BlockHistoryEstimator) setLatest(head *evmtypes.Head) {
116+
func (b *BlockHistoryEstimator) setLatest(ctx context.Context, head *evmtypes.Head) {
153117
// Non-eip1559 blocks don't include base fee
154118
if baseFee := head.BaseFeePerGas; baseFee != nil {
155-
promBlockHistoryEstimatorCurrentBaseFee.WithLabelValues(b.chainID.String()).Set(float64(baseFee.Int64()))
119+
b.metrics.RecordCurrentBaseFee(ctx, float64(baseFee.Int64()))
156120
}
121+
157122
b.logger.Debugw("Set latest block", "blockNum", head.Number, "blockHash", head.Hash, "baseFee", head.BaseFeePerGas, "baseFeeWei", head.BaseFeePerGas.ToInt())
158123
b.latestMu.Lock()
159124
defer b.latestMu.Unlock()
@@ -197,7 +162,7 @@ func (b *BlockHistoryEstimator) Start(ctx context.Context) error {
197162
b.logger.Warnw("initial check for latest head failed, head was unexpectedly nil")
198163
} else {
199164
b.logger.Debugw("Got latest head", "number", latestHead.Number, "blockHash", latestHead.Hash.Hex())
200-
b.setLatest(latestHead)
165+
b.setLatest(fetchCtx, latestHead)
201166
b.FetchBlocksAndRecalculate(fetchCtx, latestHead)
202167
}
203168

@@ -318,13 +283,13 @@ func (b *BlockHistoryEstimator) setMaxPercentileTipCap(tipCap *assets.Wei) {
318283
b.maxPercentileTipCap = tipCap
319284
}
320285

321-
func (b *BlockHistoryEstimator) BumpLegacyGas(_ context.Context, originalGasPrice *assets.Wei, gasLimit uint64, maxGasPriceWei *assets.Wei, attempts []EvmPriorAttempt) (bumpedGasPrice *assets.Wei, chainSpecificGasLimit uint64, err error) {
286+
func (b *BlockHistoryEstimator) BumpLegacyGas(ctx context.Context, originalGasPrice *assets.Wei, gasLimit uint64, maxGasPriceWei *assets.Wei, attempts []EvmPriorAttempt) (bumpedGasPrice *assets.Wei, chainSpecificGasLimit uint64, err error) {
322287
if b.bhConfig.CheckInclusionBlocks() > 0 {
323288
if err = b.haltBumping(attempts); err != nil {
324289
if errors.Is(err, fees.ErrConnectivity) {
325290
b.logger.Criticalw(BumpingHaltedLabel, "err", err)
326291
b.SvcErrBuffer.Append(err)
327-
promBlockHistoryEstimatorConnectivityFailureCount.WithLabelValues(b.chainID.String(), "legacy").Inc()
292+
b.metrics.RecordConnectivityFailure(ctx, "legacy")
328293
}
329294
return nil, 0, err
330295
}
@@ -498,13 +463,14 @@ func calcFeeCap(latestAvailableBaseFeePerGas *assets.Wei, bufferBlocks int, tipC
498463
return feeCap
499464
}
500465

501-
func (b *BlockHistoryEstimator) BumpDynamicFee(_ context.Context, originalFee DynamicFee, maxGasPriceWei *assets.Wei, attempts []EvmPriorAttempt) (bumped DynamicFee, err error) {
466+
func (b *BlockHistoryEstimator) BumpDynamicFee(ctx context.Context, originalFee DynamicFee, maxGasPriceWei *assets.Wei, attempts []EvmPriorAttempt) (bumped DynamicFee, err error) {
502467
if b.bhConfig.CheckInclusionBlocks() > 0 {
503468
if err = b.haltBumping(attempts); err != nil {
504469
if errors.Is(err, fees.ErrConnectivity) {
505470
b.logger.Criticalw(BumpingHaltedLabel, "err", err)
506471
b.SvcErrBuffer.Append(err)
507-
promBlockHistoryEstimatorConnectivityFailureCount.WithLabelValues(b.chainID.String(), "eip1559").Inc()
472+
473+
b.metrics.RecordConnectivityFailure(ctx, "eip1559")
508474
}
509475
return bumped, err
510476
}
@@ -539,11 +505,11 @@ func (b *BlockHistoryEstimator) FetchBlocksAndRecalculate(ctx context.Context, h
539505
return
540506
}
541507
b.initialFetch.Store(true)
542-
b.Recalculate(head)
508+
b.Recalculate(ctx, head)
543509
}
544510

545511
// Recalculate adds the given heads to the history and recalculates gas price.
546-
func (b *BlockHistoryEstimator) Recalculate(head *evmtypes.Head) {
512+
func (b *BlockHistoryEstimator) Recalculate(ctx context.Context, head *evmtypes.Head) {
547513
lggr := b.logger.With("head", head)
548514

549515
blockHistory := b.getBlocks()
@@ -553,13 +519,13 @@ func (b *BlockHistoryEstimator) Recalculate(head *evmtypes.Head) {
553519
}
554520

555521
// Calculate and set the TransactionPercentile gas price and tip cap to use during gas estimation
556-
b.calculateGasPriceTipCap(lggr, blockHistory, head)
522+
b.calculateGasPriceTipCap(ctx, lggr, blockHistory, head)
557523

558524
// Calculate and set the CheckInclusionPercentile gas price and tip cap to halt excessive bumping
559525
b.calculateMaxPercentileGasPriceTipCap(lggr, blockHistory, head)
560526
}
561527

562-
func (b *BlockHistoryEstimator) calculateGasPriceTipCap(lggr logger.SugaredLogger, blockHistory []evmtypes.Block, head *evmtypes.Head) {
528+
func (b *BlockHistoryEstimator) calculateGasPriceTipCap(ctx context.Context, lggr logger.SugaredLogger, blockHistory []evmtypes.Block, head *evmtypes.Head) {
563529
percentile := int(b.bhConfig.TransactionPercentile())
564530
eip1559 := b.eConfig.EIP1559DynamicFees()
565531

@@ -571,12 +537,14 @@ func (b *BlockHistoryEstimator) calculateGasPriceTipCap(lggr logger.SugaredLogge
571537
func(gasPrices []*assets.Wei) {
572538
for i := 0; i <= 100; i += 5 {
573539
jdx := ((len(gasPrices) - 1) * i) / 100
574-
promBlockHistoryEstimatorAllGasPricePercentiles.WithLabelValues(fmt.Sprintf("%v%%", i), b.chainID.String()).Set(float64(gasPrices[jdx].Int64()))
540+
percentileLabel := fmt.Sprintf("%v%%", i)
541+
b.metrics.RecordAllGasPricePercentile(ctx, percentileLabel, float64(gasPrices[jdx].Int64()))
575542
}
576543
}, func(tipCaps []*assets.Wei) {
577544
for i := 0; i <= 100; i += 5 {
578545
jdx := ((len(tipCaps) - 1) * i) / 100
579-
promBlockHistoryEstimatorAllTipCapPercentiles.WithLabelValues(fmt.Sprintf("%v%%", i), b.chainID.String()).Set(float64(tipCaps[jdx].Int64()))
546+
percentileLabel := fmt.Sprintf("%v%%", i)
547+
b.metrics.RecordAllTipCapPercentile(ctx, percentileLabel, float64(tipCaps[jdx].Int64()))
580548
}
581549
})
582550
if err != nil {
@@ -602,7 +570,9 @@ func (b *BlockHistoryEstimator) calculateGasPriceTipCap(lggr logger.SugaredLogge
602570
"priceBlocks", numsForPrice,
603571
}
604572
b.setPercentileGasPrice(percentileGasPrice)
605-
promBlockHistoryEstimatorSetGasPrice.WithLabelValues(fmt.Sprintf("%v%%", percentile), b.chainID.String()).Set(float64(percentileGasPrice.Int64()))
573+
574+
percentileLabel := fmt.Sprintf("%v%%", percentile)
575+
b.metrics.RecordSetGasPrice(ctx, percentileLabel, float64(percentileGasPrice.Int64()))
606576

607577
if !eip1559 {
608578
lggr.Debugw(fmt.Sprintf("Setting new default GasPrice: %v Gwei", gasPriceGwei), lggrFields...)
@@ -616,7 +586,8 @@ func (b *BlockHistoryEstimator) calculateGasPriceTipCap(lggr logger.SugaredLogge
616586
}...)
617587
lggr.Debugw(fmt.Sprintf("Setting new default prices, GasPrice: %v Gwei, TipCap: %v Gwei", gasPriceGwei, tipCapGwei), lggrFields...)
618588
b.setPercentileTipCap(percentileTipCap)
619-
promBlockHistoryEstimatorSetTipCap.WithLabelValues(fmt.Sprintf("%v%%", percentile), b.chainID.String()).Set(float64(percentileTipCap.Int64()))
589+
590+
b.metrics.RecordSetTipCap(ctx, percentileLabel, float64(percentileTipCap.Int64()))
620591
}
621592

622593
func (b *BlockHistoryEstimator) calculateMaxPercentileGasPriceTipCap(lggr logger.SugaredLogger, blockHistory []evmtypes.Block, head *evmtypes.Head) {

‎pkg/gas/block_history_estimator_metrics.go‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package gas
2+
3+
import (
4+
"context"
5+
"math/big"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
"github.com/prometheus/client_golang/prometheus/promauto"
9+
"go.opentelemetry.io/otel/attribute"
10+
"go.opentelemetry.io/otel/metric"
11+
12+
"github.com/smartcontractkit/chainlink-common/pkg/beholder"
13+
"github.com/smartcontractkit/chainlink-common/pkg/logger"
14+
)
15+
16+
var (
17+
promBlockHistoryEstimatorAllGasPricePercentiles = promauto.NewGaugeVec(prometheus.GaugeOpts{
18+
Name: "gas_updater_all_gas_price_percentiles",
19+
Help: "Gas price at given percentile",
20+
}, []string{"percentile", "evmChainID"})
21+
promBlockHistoryEstimatorAllTipCapPercentiles = promauto.NewGaugeVec(prometheus.GaugeOpts{
22+
Name: "gas_updater_all_tip_cap_percentiles",
23+
Help: "Tip cap at given percentile",
24+
}, []string{"percentile", "evmChainID"})
25+
promBlockHistoryEstimatorSetGasPrice = promauto.NewGaugeVec(prometheus.GaugeOpts{
26+
Name: "gas_updater_set_gas_price",
27+
Help: "Gas updater set gas price (in Wei)",
28+
}, []string{"percentile", "evmChainID"})
29+
promBlockHistoryEstimatorSetTipCap = promauto.NewGaugeVec(prometheus.GaugeOpts{
30+
Name: "gas_updater_set_tip_cap",
31+
Help: "Gas updater set gas tip cap (in Wei)",
32+
}, []string{"percentile", "evmChainID"})
33+
promBlockHistoryEstimatorCurrentBaseFee = promauto.NewGaugeVec(prometheus.GaugeOpts{
34+
Name: "gas_updater_current_base_fee",
35+
Help: "Gas updater current block base fee in Wei",
36+
}, []string{"evmChainID"})
37+
promBlockHistoryEstimatorConnectivityFailureCount = promauto.NewCounterVec(prometheus.CounterOpts{
38+
Name: "block_history_estimator_connectivity_failure_count",
39+
Help: "Counter is incremented every time a gas bump is prevented due to a detected network propagation/connectivity issue",
40+
}, []string{"evmChainID", "mode"})
41+
)
42+
43+
// blockHistoryEstimatorMetrics dual-writes to Prometheus and optional Beholder OTel instruments.
44+
// All Record* methods are safe to call when Beholder instruments are nil (no-op for OTel side).
45+
type blockHistoryEstimatorMetrics struct {
46+
chainID string
47+
48+
gasPriceGauge metric.Float64Gauge
49+
tipCapGauge metric.Float64Gauge
50+
allGasPricePercentilesGauge metric.Float64Gauge
51+
allTipCapPercentilesGauge metric.Float64Gauge
52+
currentBaseFeeGauge metric.Float64Gauge
53+
connectivityFailureCountCounter metric.Int64Counter
54+
}
55+
56+
func newBlockHistoryEstimatorMetrics(lggr logger.SugaredLogger, chainID *big.Int) *blockHistoryEstimatorMetrics {
57+
m := &blockHistoryEstimatorMetrics{chainID: chainID.String()}
58+
59+
// otel
60+
if g, err := beholder.GetMeter().Float64Gauge("gas_updater_set_gas_price"); err != nil {
61+
lggr.Errorw("Failed to register Beholder gas_updater_set_gas_price gauge", "err", err)
62+
} else {
63+
m.gasPriceGauge = g
64+
}
65+
if g, err := beholder.GetMeter().Float64Gauge("gas_updater_set_tip_cap"); err != nil {
66+
lggr.Errorw("Failed to register Beholder gas_updater_set_tip_cap gauge", "err", err)
67+
} else {
68+
m.tipCapGauge = g
69+
}
70+
if g, err := beholder.GetMeter().Float64Gauge("gas_updater_all_gas_price_percentiles"); err != nil {
71+
lggr.Errorw("Failed to register Beholder gas_updater_all_gas_price_percentiles gauge", "err", err)
72+
} else {
73+
m.allGasPricePercentilesGauge = g
74+
}
75+
if g, err := beholder.GetMeter().Float64Gauge("gas_updater_all_tip_cap_percentiles"); err != nil {
76+
lggr.Errorw("Failed to register Beholder gas_updater_all_tip_cap_percentiles gauge", "err", err)
77+
} else {
78+
m.allTipCapPercentilesGauge = g
79+
}
80+
if g, err := beholder.GetMeter().Float64Gauge("gas_updater_current_base_fee"); err != nil {
81+
lggr.Errorw("Failed to register Beholder gas_updater_current_base_fee gauge", "err", err)
82+
} else {
83+
m.currentBaseFeeGauge = g
84+
}
85+
if c, err := beholder.GetMeter().Int64Counter("block_history_estimator_connectivity_failure_count"); err != nil {
86+
lggr.Errorw("Failed to register Beholder block_history_estimator_connectivity_failure_count counter", "err", err)
87+
} else {
88+
m.connectivityFailureCountCounter = c
89+
}
90+
91+
return m
92+
}
93+
94+
func (m *blockHistoryEstimatorMetrics) RecordSetGasPrice(ctx context.Context, percentileLabel string, value float64) {
95+
promBlockHistoryEstimatorSetGasPrice.WithLabelValues(percentileLabel, m.chainID).Set(value)
96+
if m.gasPriceGauge != nil {
97+
m.gasPriceGauge.Record(ctx, value, metric.WithAttributes(
98+
attribute.String("chainID", m.chainID),
99+
attribute.String("percentile", percentileLabel),
100+
))
101+
}
102+
}
103+
104+
func (m *blockHistoryEstimatorMetrics) RecordSetTipCap(ctx context.Context, percentileLabel string, value float64) {
105+
promBlockHistoryEstimatorSetTipCap.WithLabelValues(percentileLabel, m.chainID).Set(value)
106+
if m.tipCapGauge != nil {
107+
m.tipCapGauge.Record(ctx, value, metric.WithAttributes(
108+
attribute.String("chainID", m.chainID),
109+
attribute.String("percentile", percentileLabel),
110+
))
111+
}
112+
}
113+
114+
func (m *blockHistoryEstimatorMetrics) RecordAllGasPricePercentile(ctx context.Context, percentileLabel string, value float64) {
115+
promBlockHistoryEstimatorAllGasPricePercentiles.WithLabelValues(percentileLabel, m.chainID).Set(value)
116+
if m.allGasPricePercentilesGauge != nil {
117+
m.allGasPricePercentilesGauge.Record(ctx, value, metric.WithAttributes(
118+
attribute.String("chainID", m.chainID),
119+
attribute.String("percentile", percentileLabel),
120+
))
121+
}
122+
}
123+
124+
func (m *blockHistoryEstimatorMetrics) RecordAllTipCapPercentile(ctx context.Context, percentileLabel string, value float64) {
125+
promBlockHistoryEstimatorAllTipCapPercentiles.WithLabelValues(percentileLabel, m.chainID).Set(value)
126+
if m.allTipCapPercentilesGauge != nil {
127+
m.allTipCapPercentilesGauge.Record(ctx, value, metric.WithAttributes(
128+
attribute.String("chainID", m.chainID),
129+
attribute.String("percentile", percentileLabel),
130+
))
131+
}
132+
}
133+
134+
func (m *blockHistoryEstimatorMetrics) RecordCurrentBaseFee(ctx context.Context, value float64) {
135+
promBlockHistoryEstimatorCurrentBaseFee.WithLabelValues(m.chainID).Set(value)
136+
if m.currentBaseFeeGauge != nil {
137+
m.currentBaseFeeGauge.Record(ctx, value, metric.WithAttributes(
138+
attribute.String("chainID", m.chainID),
139+
))
140+
}
141+
}
142+
143+
func (m *blockHistoryEstimatorMetrics) RecordConnectivityFailure(ctx context.Context, mode string) {
144+
promBlockHistoryEstimatorConnectivityFailureCount.WithLabelValues(m.chainID, mode).Inc()
145+
if m.connectivityFailureCountCounter != nil {
146+
m.connectivityFailureCountCounter.Add(ctx, 1, metric.WithAttributes(
147+
attribute.String("chainID", m.chainID),
148+
attribute.String("mode", mode),
149+
))
150+
}
151+
}

0 commit comments

Comments
 (0)