Instrument traces and metrics with opentelemetry (#232)

Author: hazim-j

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ethereum/go-ethereum v1.11.5
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-gonic/gin v1.9.0
-	github.com/go-logr/logr v1.2.3
+	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zerologr v1.2.3
 	github.com/go-playground/validator/v10 v10.12.0
 	github.com/google/go-cmp v0.5.9
@@ -18,15 +18,26 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.15.0
 	github.com/wangjia184/sortedset v0.0.0-20220209072355-af6d6d227aa7
+	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
+	go.opentelemetry.io/otel/metric v1.16.0
+	go.opentelemetry.io/otel/sdk v1.16.0
+	go.opentelemetry.io/otel/sdk/metric v0.39.0
+	go.opentelemetry.io/otel/trace v1.16.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/text v0.8.0
+	golang.org/x/text v0.9.0
+	google.golang.org/grpc v1.55.0
 )
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/bytedance/sonic v1.8.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
@@ -35,19 +46,21 @@ require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/goccy/go-json v0.10.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -73,11 +86,15 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.9 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -32,11 +32,29 @@ type Values struct {
 	EthBuilderUrl     string
 	BlocksInTheFuture int
 
+	// Observability variables.
+	OTELServiceName      string
+	OTELCollectorHeaders map[string]string
+	OTELCollectorUrl     string
+	OTELInsecureMode     bool
+
 	// Undocumented variables.
 	DebugMode bool
 	GinMode   string
 }
 
+func envKeyValStringToMap(s string) map[string]string {
+	out := map[string]string{}
+	for _, pair := range strings.Split(s, "&") {
+		kv := strings.Split(pair, "=")
+		if len(kv) != 2 {
+			break
+		}
+		out[kv[0]] = kv[1]
+	}
+	return out
+}
+
 func envArrayToAddressSlice(s string) []common.Address {
 	env := strings.Split(s, ",")
 	slc := []common.Address{}
@@ -62,6 +80,7 @@ func GetValues() *Values {
 	viper.SetDefault("erc4337_bundler_max_batch_gas_limit", 25000000)
 	viper.SetDefault("erc4337_bundler_max_ops_for_unstaked_sender", 4)
 	viper.SetDefault("erc4337_bundler_blocks_in_the_future", 25)
+	viper.SetDefault("erc4337_bundler_otel_insecure_mode", false)
 	viper.SetDefault("erc4337_bundler_debug_mode", false)
 	viper.SetDefault("erc4337_bundler_gin_mode", gin.ReleaseMode)
 
@@ -92,6 +111,10 @@ func GetValues() *Values {
 	_ = viper.BindEnv("erc4337_bundler_relayer_banned_time_window")
 	_ = viper.BindEnv("erc4337_bundler_eth_builder_url")
 	_ = viper.BindEnv("erc4337_bundler_blocks_in_the_future")
+	_ = viper.BindEnv("erc4337_bundler_otel_service_name")
+	_ = viper.BindEnv("erc4337_bundler_otel_collector_headers")
+	_ = viper.BindEnv("erc4337_bundler_otel_collector_url")
+	_ = viper.BindEnv("erc4337_bundler_otel_insecure_mode")
 	_ = viper.BindEnv("erc4337_bundler_debug_mode")
 	_ = viper.BindEnv("erc4337_bundler_gin_mode")
 
@@ -119,6 +142,12 @@ func GetValues() *Values {
 		}
 	}
 
+	// Validate O11Y variables
+	if viper.IsSet("erc4337_bundler_otel_service_name") &&
+		variableNotSetOrIsNil("erc4337_bundler_otel_collector_url") {
+		panic("Fatal config error: erc4337_bundler_otel_service_name is set without a collector URL")
+	}
+
 	// Return Values
 	privateKey := viper.GetString("erc4337_bundler_private_key")
 	ethClientUrl := viper.GetString("erc4337_bundler_eth_client_url")
@@ -133,6 +162,10 @@ func GetValues() *Values {
 	relayerBannedTimeWindow := viper.GetInt("erc4337_bundler_relayer_banned_time_window") * int(time.Second)
 	ethBuilderUrl := viper.GetString("erc4337_bundler_eth_builder_url")
 	blocksInTheFuture := viper.GetInt("erc4337_bundler_blocks_in_the_future")
+	otelServiceName := viper.GetString("erc4337_bundler_otel_service_name")
+	otelCollectorHeader := envKeyValStringToMap(viper.GetString("erc4337_bundler_otel_collector_headers"))
+	otelCollectorUrl := viper.GetString("erc4337_bundler_otel_collector_url")
+	otelInsecureMode := viper.GetBool("erc4337_bundler_otel_insecure_mode")
 	debugMode := viper.GetBool("erc4337_bundler_debug_mode")
 	ginMode := viper.GetString("erc4337_bundler_gin_mode")
 	return &Values{
@@ -149,6 +182,10 @@ func GetValues() *Values {
 		RelayerBannedTimeWindow: time.Duration(relayerBannedTimeWindow),
 		EthBuilderUrl:           ethBuilderUrl,
 		BlocksInTheFuture:       blocksInTheFuture,
+		OTELServiceName:         otelServiceName,
+		OTELCollectorHeaders:    otelCollectorHeader,
+		OTELCollectorUrl:        otelCollectorUrl,
+		OTELInsecureMode:        otelInsecureMode,
 		DebugMode:               debugMode,
 		GinMode:                 ginMode,
 	}

internal/config/values.go

@@ -0,0 +1,111 @@
+package o11y
+
+import (
+	"context"
+	"log"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"google.golang.org/grpc/credentials"
+
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Opts struct {
+	ServiceName     string
+	CollectorHeader map[string]string
+	CollectorUrl    string
+	InsecureMode    bool
+
+	// Bundler specific attributes
+	ChainID *big.Int
+	Address common.Address
+}
+
+func initResources(opts *Opts) *resource.Resource {
+	resources, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			attribute.String("service.name", opts.ServiceName),
+			attribute.String("library.language", "go"),
+			attribute.String("bundler.address", opts.Address.Hex()),
+			attribute.Int64("bundler.chain_id", opts.ChainID.Int64()),
+		),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return resources
+}
+
+func IsEnabled(serviceName string) bool {
+	return len(serviceName) > 0
+}
+
+func InitTracer(opts *Opts) func() {
+	secureOption := otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))
+	if opts.InsecureMode {
+		secureOption = otlptracegrpc.WithInsecure()
+	}
+
+	exporter, err := otlptrace.New(
+		context.Background(),
+		otlptracegrpc.NewClient(
+			secureOption,
+			otlptracegrpc.WithHeaders(opts.CollectorHeader),
+			otlptracegrpc.WithEndpoint(opts.CollectorUrl),
+		),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	otel.SetTracerProvider(
+		sdktrace.NewTracerProvider(
+			sdktrace.WithSampler(sdktrace.AlwaysSample()),
+			sdktrace.WithBatcher(exporter),
+			sdktrace.WithResource(initResources(opts)),
+		),
+	)
+	return func() {
+		_ = exporter.Shutdown(context.Background())
+	}
+}
+
+func InitMetrics(opts *Opts) func() {
+	secureOption := otlpmetricgrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))
+	if opts.InsecureMode {
+		secureOption = otlpmetricgrpc.WithInsecure()
+	}
+
+	exporter, err := otlpmetricgrpc.New(
+		context.Background(),
+		secureOption,
+		otlpmetricgrpc.WithHeaders(opts.CollectorHeader),
+		otlpmetricgrpc.WithEndpoint(opts.CollectorUrl),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	otel.SetMeterProvider(
+		sdkmetric.NewMeterProvider(
+			sdkmetric.WithResource(initResources(opts)),
+			sdkmetric.WithReader(
+				sdkmetric.NewPeriodicReader(exporter, sdkmetric.WithInterval(30*time.Second)),
+			),
+		),
+	)
+	return func() {
+		_ = exporter.Shutdown(context.Background())
+	}
+}

internal/o11y/init.go

@@ -14,6 +14,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/stackup-wallet/stackup-bundler/internal/config"
 	"github.com/stackup-wallet/stackup-bundler/internal/logger"
+	"github.com/stackup-wallet/stackup-bundler/internal/o11y"
 	"github.com/stackup-wallet/stackup-bundler/pkg/bundler"
 	"github.com/stackup-wallet/stackup-bundler/pkg/client"
 	"github.com/stackup-wallet/stackup-bundler/pkg/gas"
@@ -25,6 +26,8 @@ import (
 	"github.com/stackup-wallet/stackup-bundler/pkg/modules/paymaster"
 	"github.com/stackup-wallet/stackup-bundler/pkg/modules/relay"
 	"github.com/stackup-wallet/stackup-bundler/pkg/signer"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
+	"go.opentelemetry.io/otel"
 )
 
 func PrivateMode() {
@@ -59,6 +62,24 @@ func PrivateMode() {
 		log.Fatal(err)
 	}
 
+	if o11y.IsEnabled(conf.OTELServiceName) {
+		o11yOpts := &o11y.Opts{
+			ServiceName:     conf.OTELServiceName,
+			CollectorHeader: conf.OTELCollectorHeaders,
+			CollectorUrl:    conf.OTELCollectorUrl,
+			InsecureMode:    conf.OTELInsecureMode,
+
+			ChainID: chain,
+			Address: eoa.Address,
+		}
+
+		tracerCleanup := o11y.InitTracer(o11yOpts)
+		defer tracerCleanup()
+
+		metricsCleanup := o11y.InitMetrics(o11yOpts)
+		defer metricsCleanup()
+	}
+
 	ov := gas.NewDefaultOverhead()
 	if chain.Cmp(config.ArbitrumOneChainID) == 0 || chain.Cmp(config.ArbitrumGoerliChainID) == 0 {
 		ov.SetCalcPreVerificationGasFunc(gas.CalcArbitrumPVGWithEthClient(rpc, conf.SupportedEntryPoints[0]))
@@ -115,6 +136,9 @@ func PrivateMode() {
 	b.SetGetGasTipFunc(gasprice.GetGasTipWithEthClient(eth))
 	b.SetGetLegacyGasPriceFunc(gasprice.GetLegacyGasPriceWithEthClient(eth))
 	b.UseLogger(logr)
+	if err := b.UserMeter(otel.GetMeterProvider().Meter("bundler")); err != nil {
+		log.Fatal(err)
+	}
 	b.UseModules(
 		gasprice.SortByGasPrice(),
 		gasprice.FilterUnderpriced(),
@@ -145,6 +169,9 @@ func PrivateMode() {
 	if err := r.SetTrustedProxies(nil); err != nil {
 		log.Fatal(err)
 	}
+	if o11y.IsEnabled(conf.OTELServiceName) {
+		r.Use(otelgin.Middleware(conf.OTELServiceName))
+	}
 	r.Use(
 		cors.Default(),
 		logger.WithLogr(logr),
@@ -156,6 +183,7 @@ func PrivateMode() {
 	handlers := []gin.HandlerFunc{
 		relayer.FilterByClientID(),
 		jsonrpc.Controller(client.NewRpcAdapter(c, d)),
+		jsonrpc.WithOTELTracerAttributes(),
 		relayer.MapUserOpHashToClientID(),
 	}
 	r.POST("/", handlers...)