feat: setup Sentry traces (#1107)

Setup Sentry traces for Amalthea:
* Add more context when initializing Sentry
* Send `Reconcile()` errors to Sentry -> We may want to filter out "expected errors" in the future.
* Setup Sentry transactions in the `Reconcile()` function:
  * Record `reconcileID` (this ID is also printed in logs)
  * Record request (session namespace and name)
  * Record result (requeue)
  * HTTP sub-requests are also traced; e.g. `PUT` or `PATCH` requests

Author: leafty

cmd/amalthea/main.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
+	sentryhttpclient "github.com/getsentry/sentry-go/httpclient"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -33,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -68,6 +70,7 @@ func main() {
 	var sentryDsn string
 	var sentryEnvironment string
 	var sentryTracesSampleRate float64
+	var sentryRelease string
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
@@ -89,6 +92,7 @@ func main() {
 		0,
 		"The sample rate for Sentry performance monitoring.",
 	)
+	flag.StringVar(&sentryRelease, "sentry-release", "", "The release value used for Sentry.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -103,18 +107,29 @@ func main() {
 		setupLog.Info("Sentry config",
 			"DSN", sentryDsn,
 			"environment", sentryEnvironment,
+			"release", sentryRelease,
 			"tracesSampleRate", sentryTracesSampleRate,
 		)
 		err := sentry.Init(sentry.ClientOptions{
 			Dsn:              sentryDsn,
 			SendDefaultPII:   false,
+			Environment:      sentryEnvironment,
+			Release:          sentryRelease,
 			EnableTracing:    sentryTracesSampleRate > 0,
 			TracesSampleRate: sentryTracesSampleRate,
 		})
 		if err != nil {
 			setupLog.Error(err, "failed to initialize Sentry")
 			os.Exit(1)
 		}
+		defer func() {
+			err := recover()
+			if err != nil {
+				sentry.CurrentHub().Recover(err)
+				sentry.Flush(2 * time.Second)
+				panic(err)
+			}
+		}()
 		defer sentry.Flush(2 * time.Second)
 	}
 
@@ -166,17 +181,24 @@ func main() {
 	if releaseNamespace == "" {
 		releaseNamespace = "default"
 	}
+
+	config := ctrl.GetConfigOrDie()
+
+	httpClient, err := rest.HTTPClientFor(config)
+	if err != nil {
+		setupLog.Error(err, "unable to get http client for config")
+		os.Exit(1)
+	}
+	httpClient.Transport = sentryhttpclient.NewSentryRoundTripper(httpClient.Transport)
+
 	cacheOptions := cache.Options{
+		HTTPClient:        httpClient,
 		DefaultNamespaces: map[string]cache.Config{releaseNamespace: {}},
 	}
 	if clusterScoped {
-		cacheOptions = cache.Options{
-			DefaultNamespaces: nil,
-		}
+		cacheOptions.DefaultNamespaces = nil
 	}
 
-	config := ctrl.GetConfigOrDie()
-
 	mgr, err := ctrl.NewManager(config, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
@@ -196,6 +218,9 @@ func main() {
 		// after the manager stops then its usage might be unsafe.
 		// LeaderElectionReleaseOnCancel: true,
 		Cache: cacheOptions,
+		Client: client.Options{
+			HTTPClient: httpClient,
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/elazarl/goproxy v1.7.2
 	github.com/getkin/kin-openapi v0.132.0
-	github.com/getsentry/sentry-go v0.36.2
+	github.com/getsentry/sentry-go v0.45.1
 	github.com/go-git/go-git/v5 v5.16.0
 	github.com/go-logr/logr v1.4.2
 	github.com/golang-jwt/jwt/v5 v5.3.0

go.sum

@@ -74,8 +74,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/getkin/kin-openapi v0.132.0 h1:3ISeLMsQzcb5v26yeJrBcdTCEQTag36ZjaGk7MIRUwk=
 github.com/getkin/kin-openapi v0.132.0/go.mod h1:3OlG51PCYNsPByuiMB0t4fjnNlIDnaEDsjiKUV8nL58=
-github.com/getsentry/sentry-go v0.36.2 h1:uhuxRPTrUy0dnSzTd0LrYXlBYygLkKY0hhlG5LXarzM=
-github.com/getsentry/sentry-go v0.36.2/go.mod h1:p5Im24mJBeruET8Q4bbcMfCQ+F+Iadc4L48tB1apo2c=
+github.com/getsentry/sentry-go v0.45.1 h1:9rfzJtGiJG+MGIaWZXidDGHcH5GU1Z5y0WVJGf9nysw=
+github.com/getsentry/sentry-go v0.45.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=

helm-chart/amalthea-sessions/templates/deployment.yaml

@@ -28,6 +28,7 @@ spec:
         - --sentry-dsn={{ .Values.controllerManager.manager.sentry.dsn | default "unset" }}
         - --sentry-environment={{ .Values.controllerManager.manager.sentry.environment | default "" }}
         - --sentry-traces-sample-rate={{ .Values.controllerManager.manager.sentry.tracesSampleRate | default 0.1 }}
+        - --sentry-release={{ .Values.controllerManager.manager.image.tag | default "" }}
         {{- end }}
         command:
         - /manager

internal/controller/amaltheasession_controller.go

@@ -19,9 +19,12 @@ package controller
 import (
 	"context"
 	"errors"
+	"fmt"
 	"reflect"
 	"time"
 
+	"github.com/getsentry/sentry-go"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -32,6 +35,7 @@ import (
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -71,6 +75,53 @@ const secretCleanupFinalizerName = "amalthea.dev/secrets-finalizer"
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile
 func (r *AmaltheaSessionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Sentry Trace
+	hub := sentry.GetHubFromContext(ctx)
+	if hub == nil {
+		hub = sentry.CurrentHub().Clone()
+		ctx = sentry.SetHubOnContext(ctx, hub)
+	}
+	// Capture panics
+	defer func() {
+		err := recover()
+		if err != nil {
+			hub.RecoverWithContext(ctx, err)
+			panic(err)
+		}
+	}()
+	// Start Sentry transaction
+	options := []sentry.SpanOption{
+		sentry.WithOpName("function.reconcile"),
+		sentry.WithDescription(fmt.Sprintf("RECONCILE AmaltheaSession %s %s", req.Namespace, req.Name)),
+		sentry.WithSpanOrigin(sentry.SpanOriginManual),
+		sentry.WithTransactionSource(sentry.SourceComponent),
+	}
+	transaction := sentry.StartTransaction(ctx, "AmaltheaSessionReconciler.Reconcile", options...)
+	transaction.SetData("controller.request.namespace", req.Namespace)
+	transaction.SetData("controller.request.name", req.Name)
+	reconcileID := controller.ReconcileIDFromContext(ctx)
+	if reconcileID != "" {
+		transaction.SetData("controller.reconcile_id", string(reconcileID))
+	}
+
+	defer transaction.Finish()
+
+	res, err := r.reconcileInner(transaction.Context(), req)
+	// Set transaction results
+	if err == nil {
+		transaction.Status = sentry.SpanStatusOK
+	} else {
+		transaction.Status = sentry.SpanStatusInternalError
+		hub.CaptureException(err)
+	}
+	transaction.SetData("controller.result.requeue", res.Requeue || res.RequeueAfter > 0)
+	if res.RequeueAfter != 0 {
+		transaction.SetData("controller.result.requeue_after", res.RequeueAfter)
+	}
+	return res, err
+}
+
+func (r *AmaltheaSessionReconciler) reconcileInner(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 
 	amaltheasession := &amaltheadevv1alpha1.AmaltheaSession{}