feat: initialize metrics on startup (#1812)

Author: Thegaram

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.7.15"
+var tag = "v4.7.16"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

rollup/internal/controller/relayer/l2_relayer.go

@@ -194,6 +194,7 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm.
 		return nil, fmt.Errorf("failed to initialize and commit genesis batch, err: %v", err)
 	}
 	layer2Relayer.metrics = initL2RelayerMetrics(reg)
+	layer2Relayer.initializeMetrics(ctx)
 
 	switch serviceType {
 	case ServiceTypeL2RollupRelayer:
@@ -205,6 +206,28 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm.
 	return layer2Relayer, nil
 }
 
+// initializeMetrics seeds the commit block height gauge from DB so that a restart
+// does not leave it at 0, which would otherwise make the commit-lag alert fire
+// until the next batch is committed.
+func (r *Layer2Relayer) initializeMetrics(ctx context.Context) {
+	latestBatch, err := r.batchOrm.GetLatestCommittedBatch(ctx)
+	if err != nil {
+		log.Warn("failed to initialize commit block height metric", "err", err)
+		return
+	}
+	if latestBatch == nil {
+		return
+	}
+	endChunk, err := r.chunkOrm.GetChunkByIndex(ctx, latestBatch.EndChunkIndex)
+	if err != nil {
+		log.Warn("failed to initialize commit block height metric", "err", err)
+		return
+	}
+	if endChunk != nil {
+		r.metrics.rollupL2RelayerCommitBlockHeight.Set(float64(endChunk.EndBlockNumber))
+	}
+}
+
 func (r *Layer2Relayer) initializeGenesis() error {
 	if count, err := r.batchOrm.GetBatchCount(r.ctx); err != nil {
 		return fmt.Errorf("failed to get batch count: %v", err)

rollup/internal/controller/watcher/batch_proposer.go

@@ -2,6 +2,7 @@ package watcher
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -119,9 +120,33 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC
 		}),
 	}
 
+	p.initializeMetrics(ctx)
+
 	return p
 }
 
+// initializeMetrics seeds the propose block height gauge from DB so that a restart
+// does not leave it at 0, which would otherwise make the propose-lag alert fire
+// until the next batch is proposed.
+func (p *BatchProposer) initializeMetrics(ctx context.Context) {
+	latestBatch, err := p.batchOrm.GetLatestBatch(ctx)
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return // no batches proposed yet, nothing to seed
+		}
+		log.Warn("failed to initialize batch propose block height metric", "err", err)
+		return
+	}
+	endChunk, err := p.chunkOrm.GetChunkByIndex(ctx, latestBatch.EndChunkIndex)
+	if err != nil {
+		log.Warn("failed to initialize batch propose block height metric", "err", err)
+		return
+	}
+	if endChunk != nil {
+		p.batchProposeBlockHeight.Set(float64(endChunk.EndBlockNumber))
+	}
+}
+
 // SetReplayDB sets the replay database for the BatchProposer.
 // This is used for the proposer tool only, to change the l2_block data source.
 // This function is not thread-safe and should be called after initializing the BatchProposer and before starting to propose chunks.

rollup/internal/controller/watcher/chunk_proposer.go

@@ -127,9 +127,25 @@ func NewChunkProposer(ctx context.Context, cfg *config.ChunkProposerConfig, minC
 		}),
 	}
 
+	p.initializeMetrics(ctx)
+
 	return p
 }
 
+// initializeMetrics seeds the propose block height gauge from DB so that a restart
+// does not leave it at 0, which would otherwise make the propose-lag alert fire
+// until the next chunk is proposed.
+func (p *ChunkProposer) initializeMetrics(ctx context.Context) {
+	latestChunk, err := p.chunkOrm.GetLatestChunk(ctx)
+	if err != nil {
+		log.Warn("failed to initialize chunk propose block height metric", "err", err)
+		return
+	}
+	if latestChunk != nil {
+		p.chunkProposeBlockHeight.Set(float64(latestChunk.EndBlockNumber))
+	}
+}
+
 // SetReplayDB sets the replay database for the ChunkProposer.
 // This is used for the proposer tool only, to change the l2_block data source.
 // This function is not thread-safe and should be called after initializing the ChunkProposer and before starting to propose chunks.

rollup/internal/controller/watcher/l1_watcher.go

@@ -44,7 +44,7 @@ func NewL1WatcherClient(ctx context.Context, rpcClient *rpc.Client, startHeight
 		savedL1BlockHeight = startHeight
 	}
 
-	return &L1WatcherClient{
+	w := &L1WatcherClient{
 		ctx:        ctx,
 		rpcClient:  rpcClient,
 		client:     ethclient.NewClient(rpcClient),
@@ -53,6 +53,11 @@ func NewL1WatcherClient(ctx context.Context, rpcClient *rpc.Client, startHeight
 		processedBlockHeight: savedL1BlockHeight,
 		metrics:              initL1WatcherMetrics(reg),
 	}
+
+	// Seed the gauge from the resumed height instead of starting from 0.
+	w.metrics.l1WatcherFetchBlockHeaderProcessedBlockHeight.Set(float64(w.processedBlockHeight))
+
+	return w
 }
 
 // ProcessedBlockHeight get processedBlockHeight

rollup/internal/controller/watcher/l2_watcher.go

@@ -43,7 +43,7 @@ type L2WatcherClient struct {
 
 // NewL2WatcherClient take a l2geth instance to generate a l2watcherclient instance
 func NewL2WatcherClient(ctx context.Context, client *rpc.Client, confirmations rpc.BlockNumber, messageQueueAddress common.Address, withdrawTrieRootSlot common.Hash, chainCfg *params.ChainConfig, db *gorm.DB, validiumMode bool, reg prometheus.Registerer) *L2WatcherClient {
-	return &L2WatcherClient{
+	w := &L2WatcherClient{
 		ctx:    ctx,
 		Client: ethclient.NewClient(client),
 		rpcCli: client,
@@ -61,6 +61,15 @@ func NewL2WatcherClient(ctx context.Context, client *rpc.Client, confirmations r
 
 		chainCfg: chainCfg,
 	}
+
+	// Seed the gauge from the latest stored height instead of starting from 0.
+	if latestHeight, err := w.l2BlockOrm.GetL2BlocksLatestHeight(ctx); err != nil {
+		log.Warn("failed to initialize l2 watcher fetched height metric", "err", err)
+	} else {
+		w.metrics.fetchRunningMissingBlocksHeight.Set(float64(latestHeight))
+	}
+
+	return w
 }
 
 const blocksFetchLimit = uint64(10)

rollup/internal/orm/batch.go

@@ -155,6 +155,24 @@ func (o *Batch) GetLatestBatch(ctx context.Context) (*Batch, error) {
 	return &latestBatch, nil
 }
 
+// GetLatestCommittedBatch retrieves the highest-index batch that has a commit tx hash set.
+// It returns nil if no such batch exists.
+func (o *Batch) GetLatestCommittedBatch(ctx context.Context) (*Batch, error) {
+	db := o.db.WithContext(ctx)
+	db = db.Model(&Batch{})
+	db = db.Where("commit_tx_hash IS NOT NULL AND commit_tx_hash != ''")
+	db = db.Order("index desc")
+
+	var latestBatch Batch
+	if err := db.First(&latestBatch).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("Batch.GetLatestCommittedBatch error: %w", err)
+	}
+	return &latestBatch, nil
+}
+
 // GetFirstUnbatchedChunkIndex retrieves the first unbatched chunk index.
 func (o *Batch) GetFirstUnbatchedChunkIndex(ctx context.Context) (uint64, error) {
 	// Get the latest batch