update

Author: Larry Li

docs/rpc_observability.md

@@ -1,16 +1,23 @@
 package metrics
 
 import (
+	"context"
+	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 )
 
 var (
 	RPCCallLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Name: "rpc_call_latency",
-		Help: "The duration of an RPC call in milliseconds",
+		Help: "The duration of an RPC call in seconds",
 		Buckets: []float64{
 			float64(50 * time.Millisecond),
 			float64(100 * time.Millisecond),
@@ -22,4 +29,100 @@ var (
 			float64(8 * time.Second),
 		},
 	}, []string{"chainFamily", "chainID", "rpcUrl", "isSendOnly", "success", "rpcCallName"})
+
+	RPCCallErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "rpc_call_errors_total",
+		Help: "The total number of failed RPC calls",
+	}, []string{"chainFamily", "chainID", "rpcUrl", "isSendOnly", "rpcCallName"})
+)
+
+const (
+	rpcCallLatencyBeholder     = "rpc_call_latency"
+	rpcCallErrorsTotalBeholder = "rpc_call_errors_total"
 )
+
+// RPCClientMetrics records RPC latency and errors to Prometheus and Beholder (same pattern as multinode metrics).
+type RPCClientMetrics interface {
+	// RecordRequest records latency for an RPC call (observed in seconds for Prometheus).
+	// If err is non-nil, increments rpc_call_errors_total.
+	RecordRequest(ctx context.Context, callName string, latency time.Duration, err error)
+}
+
+var _ RPCClientMetrics = (*rpcClientMetrics)(nil)
+
+type rpcClientMetrics struct {
+	chainFamily string
+	chainID     string
+	rpcURL      string
+	isSendOnly  bool
+	latency     metric.Float64Histogram
+	errorsTotal metric.Int64Counter
+}
+
+// RPCClientMetricsConfig holds fixed labels for an RPC client instance.
+type RPCClientMetricsConfig struct {
+	ChainFamily string
+	ChainID     string
+	RPCURL      string
+	IsSendOnly  bool
+}
+
+// NewRPCClientMetrics creates RPC client metrics that publish to Prometheus and Beholder.
+func NewRPCClientMetrics(cfg RPCClientMetricsConfig) (RPCClientMetrics, error) {
+	latency, err := beholder.GetMeter().Float64Histogram(rpcCallLatencyBeholder)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register RPC call latency metric: %w", err)
+	}
+	errorsTotal, err := beholder.GetMeter().Int64Counter(rpcCallErrorsTotalBeholder)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register RPC call errors metric: %w", err)
+	}
+	return &rpcClientMetrics{
+		chainFamily: cfg.ChainFamily,
+		chainID:     cfg.ChainID,
+		rpcURL:      cfg.RPCURL,
+		isSendOnly:  cfg.IsSendOnly,
+		latency:     latency,
+		errorsTotal: errorsTotal,
+	}, nil
+}
+
+func (m *rpcClientMetrics) RecordRequest(ctx context.Context, callName string, latency time.Duration, err error) {
+	successStr := "true"
+	if err != nil {
+		successStr = "false"
+	}
+	sendStr := strconv.FormatBool(m.isSendOnly)
+	sec := latency.Seconds()
+
+	RPCCallLatency.WithLabelValues(m.chainFamily, m.chainID, m.rpcURL, sendStr, successStr, callName).Observe(sec)
+
+	latAttrs := metric.WithAttributes(
+		attribute.String("chainFamily", m.chainFamily),
+		attribute.String("chainID", m.chainID),
+		attribute.String("rpcUrl", m.rpcURL),
+		attribute.String("isSendOnly", sendStr),
+		attribute.String("success", successStr),
+		attribute.String("rpcCallName", callName),
+	)
+	m.latency.Record(ctx, sec, latAttrs)
+
+	if err != nil {
+		RPCCallErrorsTotal.WithLabelValues(m.chainFamily, m.chainID, m.rpcURL, sendStr, callName).Inc()
+		errAttrs := metric.WithAttributes(
+			attribute.String("chainFamily", m.chainFamily),
+			attribute.String("chainID", m.chainID),
+			attribute.String("rpcUrl", m.rpcURL),
+			attribute.String("isSendOnly", sendStr),
+			attribute.String("rpcCallName", callName),
+		)
+		m.errorsTotal.Add(ctx, 1, errAttrs)
+	}
+}
+
+// NoopRPCClientMetrics is a no-op implementation for when metrics are disabled.
+type NoopRPCClientMetrics struct{}
+
+func (NoopRPCClientMetrics) RecordRequest(context.Context, string, time.Duration, error) {}
+
+var _ RPCClientMetrics = NoopRPCClientMetrics{}

metrics/client.go

@@ -4,29 +4,29 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestNewRPCClientMetrics(t *testing.T) {
 	m, err := NewRPCClientMetrics(RPCClientMetricsConfig{
-		Env:         "staging",
-		Network:     "ethereum",
+		ChainFamily: "evm",
 		ChainID:     "1",
-		RPCProvider: "primary",
+		RPCURL:      "http://localhost:8545",
+		IsSendOnly:  false,
 	})
 	require.NoError(t, err)
 	require.NotNil(t, m)
 
 	ctx := context.Background()
-	m.RecordRequest(ctx, "latest_block", 100.0, nil)
-	m.RecordRequest(ctx, "latest_block", 50.0, errors.New("rpc error"))
+	m.RecordRequest(ctx, "latest_block", 100*time.Millisecond, nil)
+	m.RecordRequest(ctx, "latest_block", 50*time.Millisecond, errors.New("rpc error"))
 }
 
 func TestNoopRPCClientMetrics_RecordRequest(t *testing.T) {
 	var m NoopRPCClientMetrics
 	ctx := context.Background()
-	m.RecordRequest(ctx, "latest_block", 100.0, nil)
-	m.RecordRequest(ctx, "latest_block", 50.0, errors.New("rpc error"))
-	// Noop should not panic
+	m.RecordRequest(ctx, "latest_block", 100*time.Millisecond, nil)
+	m.RecordRequest(ctx, "latest_block", 50*time.Millisecond, errors.New("rpc error"))
 }