[c-930] use metrics v2 to trace relevant code paths (#10)

* fix: leftover go mod tidy

* chore: wire up metrics v2

* chore: trace CListMempool

* impl: extend node ctor to accept a metrics meter

* chore: trace CListMempool

* chore: trace state store

* chore: trace consensus Receive handler

* chore: trace state sync reactor

* chore: trace consensus handleMsg method

* chore: fix test compilation

* fix: set node meter from arg

* fix: use proper ctx for recheckTxs

* chore: refactor changes to make diff smaller

Additionally:
* stop() to be called with &err reference
* handle checktx callback separately from the original CheckTx func

* chore: stabilize tests

* fix: fix test 04 in CI

---------

Co-authored-by: Max <maxim@injectivelabs.org>

Author: dbrajovic

abci/tutorials/abci-v2-forum-app/forum.go

@@ -77,6 +77,7 @@ func main() {
 		cfg.DefaultDBProvider,
 		nm.DefaultMetricsProvider(config.Instrumentation),
 		logger,
+		nil,
 	)
 
 	defer func() {

go.mod

@@ -1,7 +1,6 @@
 module github.com/cometbft/cometbft
 
-go 1.23
-toolchain go1.23.7
+go 1.23.9
 
 require (
 	github.com/BurntSushi/toml v1.4.0
@@ -33,15 +32,16 @@ require (
 	github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.10.0
-	golang.org/x/crypto v0.36.0
-	golang.org/x/net v0.37.0
-	google.golang.org/grpc v1.71.0
+	github.com/stretchr/testify v1.11.1
+	golang.org/x/crypto v0.41.0
+	golang.org/x/net v0.43.0
+	google.golang.org/grpc v1.75.0
 )
 
 require github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 
 require (
+	github.com/InjectiveLabs/metrics/v2 v2.0.0-beta.8
 	github.com/Masterminds/semver/v3 v3.3.1
 	github.com/go-git/go-git/v5 v5.14.0
 	github.com/goccmack/goutil v1.2.3
@@ -51,9 +51,9 @@ require (
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae
 	github.com/supranational/blst v0.3.14
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
-	golang.org/x/sync v0.12.0
-	gonum.org/v1/gonum v0.15.1
-	google.golang.org/protobuf v1.36.5
+	golang.org/x/sync v0.16.0
+	gonum.org/v1/gonum v0.16.0
+	google.golang.org/protobuf v1.36.8
 )
 
 require (
@@ -64,6 +64,7 @@ require (
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/ProtonMail/go-crypto v1.1.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.6.0 // indirect
 	github.com/cockroachdb/errors v1.11.3 // indirect
@@ -85,7 +86,7 @@ require (
 	github.com/getsentry/sentry-go v0.31.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -95,6 +96,7 @@ require (
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -131,13 +133,20 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.etcd.io/bbolt v1.4.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.opentelemetry.io/otel v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
+	go.opentelemetry.io/otel/metric v1.38.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.38.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
+	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -152,3 +161,5 @@ retract (
 	// superseeded by v0.38.3 because of ASA-2024-001
 	[v0.38.0, v0.38.2]
 )
+
+replace github.com/cometbft/cometbft/api => ./api

go.sum

@@ -1,4 +1,4 @@
-go 1.23
+go 1.23.9
 
 use (
 	.

go.work

@@ -3,7 +3,7 @@ package clist
 import (
 	"fmt"
 	"runtime"
-	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -73,55 +73,56 @@ func TestGCFifo(t *testing.T) {
 		t.Skipf("Skipping on non-amd64 machine")
 	}
 
-	const numElements = 1000000
-	l := New()
+	const numElements = 100000
+	finalized := func() *atomic.Int64 {
+		l := New()
+		finalized := new(atomic.Int64)
 
-	// Use a WaitGroup to wait for all finalizers to run.
-	var wg sync.WaitGroup
-	wg.Add(numElements)
+		type value struct {
+			Int int
+		}
 
-	type value struct {
-		Int int
-	}
+		for i := 0; i < numElements; i++ {
+			v := new(value)
+			v.Int = i
+			l.PushBack(v)
+			runtime.SetFinalizer(v, func(_ *value) {
+				finalized.Add(1)
+			})
+		}
 
-	for i := 0; i < numElements; i++ {
-		v := new(value)
-		v.Int = i
-		l.PushBack(v)
-		runtime.SetFinalizer(v, func(_ *value) {
-			wg.Done()
-		})
-	}
+		for el := l.Front(); el != nil; {
+			next := el.Next()
+			l.Remove(el)
+			el.DetachPrev()
+			el.DetachNext()
+			el = next
+		}
 
-	for el := l.Front(); el != nil; {
-		next := el.Next()
-		l.Remove(el)
-		el = next
-	}
+		if l.Len() != 0 {
+			t.Errorf("expected list to be empty, got %v elements", l.Len())
+		}
 
-	// Wait for all finalizers to run.
-	wg.Wait()
+		return finalized
+	}()
 
-	if l.Len() != 0 {
-		t.Errorf("expected list to be empty, got %v elements", l.Len())
-	}
+	assert.Eventually(t, func() bool {
+		runtime.GC()
+		return finalized.Load() == numElements
+	}, 30*time.Second, 50*time.Millisecond)
 }
 
-// This test is quite hacky because it relies on SetFinalizer
-// which isn't guaranteed to run at all.
+// This test exercises random removals. Finalizer-based GC assertions are
+// covered by TestGCFifo and proved too nondeterministic here.
 func TestGCRandom(t *testing.T) {
 	t.Helper()
 	if runtime.GOARCH != "amd64" {
 		t.Skipf("Skipping on non-amd64 machine")
 	}
 
-	const numElements = 1000000
+	const numElements = 100000
 	l := New()
 
-	// Use a WaitGroup to wait for all finalizers to run.
-	var wg sync.WaitGroup
-	wg.Add(numElements)
-
 	type value struct {
 		Int int
 	}
@@ -130,9 +131,6 @@ func TestGCRandom(t *testing.T) {
 		v := new(value)
 		v.Int = i
 		l.PushBack(v)
-		runtime.SetFinalizer(v, func(_ *value) {
-			wg.Done()
-		})
 	}
 
 	els := make([]*CElement, 0, numElements)
@@ -143,12 +141,11 @@ func TestGCRandom(t *testing.T) {
 	for _, i := range cmtrand.Perm(numElements) {
 		el := els[i]
 		l.Remove(el)
-		_ = el.Next()
+		el.DetachPrev()
+		el.DetachNext()
+		els[i] = nil
 	}
 
-	// Wait for all finalizers to run.
-	wg.Wait()
-
 	if l.Len() != 0 {
 		t.Errorf("expected list to be empty, got %v elements", l.Len())
 	}

internal/clist/clist_test.go

@@ -288,7 +288,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 				assert.Equal(t, prevoteHeight, ev.Height())
 			}
 		}
-	case <-time.After(20 * time.Second):
+	case <-time.After(60 * time.Second):
 		t.Fatalf("Timed out waiting for validators to commit evidence")
 	}
 }

internal/consensus/byzantine_test.go

@@ -1,13 +1,15 @@
 package consensus
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"reflect"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	injmetrics "github.com/InjectiveLabs/metrics/v2"
 	cmtcons "github.com/cometbft/cometbft/api/cometbft/consensus/v1"
 	"github.com/cometbft/cometbft/internal/bits"
 	cstypes "github.com/cometbft/cometbft/internal/consensus/types"
@@ -51,6 +53,7 @@ type Reactor struct {
 	initialHeight int64 // under rsMtx
 
 	Metrics *Metrics
+	meter   injmetrics.Meter
 }
 
 type ReactorOption func(*Reactor)
@@ -63,6 +66,7 @@ func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption)
 		rs:            consensusState.GetRoundState(),
 		initialHeight: consensusState.state.InitialHeight,
 		Metrics:       NopMetrics(),
+		meter:         injmetrics.NewNilMeter(),
 	}
 	conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR)
 	if waitSync {
@@ -265,6 +269,9 @@ func (conR *Reactor) Receive(e p2p.Envelope) {
 		panic(fmt.Sprintf("Peer %v has no state", e.Src))
 	}
 
+	_, stopFn := conR.meter.FuncTimingCtx(context.Background(), "Receive", injmetrics.Tag("msg", fmt.Sprintf("%T", msg)))
+	defer stopFn()
+
 	switch e.ChannelID {
 	case StateChannel:
 		switch msg := msg.(type) {
@@ -1079,6 +1086,11 @@ func ReactorMetrics(metrics *Metrics) ReactorOption {
 	return func(conR *Reactor) { conR.Metrics = metrics }
 }
 
+// ReactorMeter sets the tracing meter.
+func ReactorMeter(meter injmetrics.Meter) ReactorOption {
+	return func(conR *Reactor) { conR.meter = meter }
+}
+
 // -----------------------------------------------------------------------------
 
 // PeerState contains the known state of a peer, including its connection and

internal/consensus/reactor.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"time"
 
+	injmetrics "github.com/InjectiveLabs/metrics/v2"
 	"github.com/cosmos/gogoproto/proto"
 
 	cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1"
@@ -134,6 +135,7 @@ type State struct {
 
 	// for reporting metrics
 	metrics *Metrics
+	meter   injmetrics.Meter
 
 	// offline state sync height indicating to which height the node synced offline
 	offlineStateSyncHeight int64
@@ -171,6 +173,7 @@ func NewState(
 		evpool:           evpool,
 		evsw:             cmtevents.NewEventSwitch(),
 		metrics:          NopMetrics(),
+		meter:            injmetrics.NewNilMeter(),
 	}
 	for _, option := range options {
 		option(cs)
@@ -220,6 +223,11 @@ func StateMetrics(metrics *Metrics) StateOption {
 	return func(cs *State) { cs.metrics = metrics }
 }
 
+// StateMeter sets the tracing meter.
+func StateMeter(meter injmetrics.Meter) StateOption {
+	return func(cs *State) { cs.meter = meter }
+}
+
 // OfflineStateSyncHeight indicates the height at which the node
 // statesync offline - before booting sets the metrics.
 func OfflineStateSyncHeight(height int64) StateOption {
@@ -877,6 +885,9 @@ func (cs *State) receiveRoutine(maxSteps int) {
 
 // state transitions on complete-proposal, 2/3-any, 2/3-one.
 func (cs *State) handleMsg(mi msgInfo) {
+	_, stopFn := cs.meter.FuncTimingCtx(context.Background(), "handleMsg", injmetrics.Tag("msg", fmt.Sprintf("%T", mi.Msg)))
+	defer stopFn()
+
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	var (