feat: add OpenTelemetry instrumentation with OTLP trace and metric exporters

- Create pkg/telemetry package with OTLP gRPC trace and metric exporters
- Add configurable trace sampling and periodic metric read interval
- Add otelgrpc.UnaryServerInterceptor to gRPC middleware chain
- Add Telemetry config section to CLI config
- Initialize OTel on server start with graceful shutdown cleanup

Author: ravisuhag

cli/config.go

@@ -11,6 +11,7 @@ import (
 	esStore "github.com/raystack/compass/internal/store/elasticsearch"
 	"github.com/raystack/compass/internal/store/postgres"
 	"github.com/raystack/compass/pkg/metrics"
+	"github.com/raystack/compass/pkg/telemetry"
 	"github.com/raystack/salt/cmdx"
 	"github.com/raystack/salt/config"
 	"github.com/spf13/cobra"
@@ -81,6 +82,9 @@ type Config struct {
 	// NewRelic
 	NewRelic metrics.NewRelicConfig `mapstructure:"newrelic"`
 
+	// Telemetry
+	Telemetry telemetry.Config `mapstructure:"telemetry"`
+
 	// Elasticsearch
 	Elasticsearch esStore.Config `mapstructure:"elasticsearch"`
 

cli/server.go

@@ -18,6 +18,7 @@ import (
 	compassserver "github.com/raystack/compass/internal/server"
 	esStore "github.com/raystack/compass/internal/store/elasticsearch"
 	"github.com/raystack/compass/internal/store/postgres"
+	"github.com/raystack/compass/pkg/telemetry"
 	"github.com/raystack/salt/log"
 	"github.com/spf13/cobra"
 )
@@ -100,6 +101,12 @@ func runServer(config *Config) error {
 		return err
 	}
 
+	otelCleanup, err := telemetry.Init(ctx, config.Telemetry, logger)
+	if err != nil {
+		return fmt.Errorf("failed to initialize telemetry: %w", err)
+	}
+	defer otelCleanup()
+
 	esClient, err := initElasticsearch(logger, config.Elasticsearch)
 	if err != nil {
 		return err

go.mod

@@ -9,13 +9,13 @@ require (
 	github.com/Masterminds/squirrel v1.5.2
 	github.com/elastic/go-elasticsearch v0.0.0
 	github.com/elastic/go-elasticsearch/v7 v7.16.0
-	github.com/envoyproxy/protoc-gen-validate v0.6.7
+	github.com/envoyproxy/protoc-gen-validate v0.10.0
 	github.com/go-playground/locales v0.14.0
 	github.com/go-playground/universal-translator v0.18.0
 	github.com/go-playground/validator/v10 v10.10.0
 	github.com/golang-migrate/migrate/v4 v4.15.2
 	github.com/golang-module/carbon/v2 v2.1.8
-	github.com/google/go-cmp v0.5.8
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/handlers v1.4.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
@@ -36,9 +36,15 @@ require (
 	github.com/raystack/salt v0.3.1
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.8.4
-	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc
-	google.golang.org/grpc v1.49.0
-	google.golang.org/protobuf v1.28.1
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
+	go.opentelemetry.io/otel/sdk v1.16.0
+	go.opentelemetry.io/otel/sdk/metric v0.39.0
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4
+	google.golang.org/grpc v1.55.0
+	google.golang.org/protobuf v1.30.0
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 )
@@ -50,7 +56,7 @@ require (
 	github.com/alecthomas/chroma v0.8.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/briandowns/spinner v1.18.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/charmbracelet/glamour v0.3.0 // indirect
 	github.com/cli/safeexec v1.0.0 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
@@ -66,7 +72,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -136,6 +142,8 @@ require (
 require (
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -144,6 +152,12 @@ require (
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/viper v1.11.0 // indirect
 	github.com/subosito/gotenv v1.4.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect