fix(tracing): properly shutdown tracer provider

Fix tracer provider reloads in different scenarios:
- disabled -> enabled: install new tracer provider
- enabled -> enabled: keep provider up and swap with new one
- enabled -> disabled: shutdown old tracer provider, use noop provider
- build failure: keep current provider

Signed-off-by: Siavash Safi <siavash@cloudflare.com>

Author: siavashs

tracing/tracing.go

@@ -100,31 +100,40 @@ func (m *Manager) ApplyConfig(cfg TracingConfig) error {
 		return nil
 	}
 
-	if m.shutdownFunc != nil {
-		if err := m.shutdownFunc(); err != nil {
-			return fmt.Errorf("failed to shut down the tracer provider: %w", err)
-		}
-	}
-
-	// If no endpoint is set, assume tracing should be disabled.
+	// If no endpoint is set, disable tracing and shut down the old provider.
 	if cfg.Endpoint == "" {
 		tracingEnabled.Store(false)
-		m.config = cfg
-		m.shutdownFunc = nil
 		otel.SetTracerProvider(noop.NewTracerProvider())
+		if m.shutdownFunc != nil {
+			if err := m.shutdownFunc(); err != nil {
+				m.logger.Warn("Failed to shut down the previous tracer provider", "err", err)
+			}
+			m.shutdownFunc = nil
+		}
+		m.config = cfg
 		m.logger.Info("Tracing provider uninstalled.")
 		return nil
 	}
 
+	// Build the new provider before tearing down the old one so that
+	// tracing remains available throughout the reload.
 	tp, shutdownFunc, err := buildTracerProvider(context.Background(), cfg)
 	if err != nil {
-		return fmt.Errorf("failed to install a new tracer provider: %w", err)
+		return fmt.Errorf("failed to build a new tracer provider: %w", err)
 	}
 
-	m.shutdownFunc = shutdownFunc
-	m.config = cfg
+	// Swap to the new provider, then shut down the old one.
+	oldShutdown := m.shutdownFunc
 	otel.SetTracerProvider(tp)
 	tracingEnabled.Store(true)
+	m.shutdownFunc = shutdownFunc
+	m.config = cfg
+
+	if oldShutdown != nil {
+		if err := oldShutdown(); err != nil {
+			m.logger.Warn("Failed to shut down the previous tracer provider", "err", err)
+		}
+	}
 
 	m.logger.Info("Successfully installed a new tracer provider.")
 	return nil
@@ -190,12 +199,7 @@ func buildTracerProvider(ctx context.Context, tracingCfg TracingConfig) (trace.T
 	return tp, func() error {
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
-		err := tp.Shutdown(ctx)
-		if err != nil {
-			return err
-		}
-
-		return nil
+		return tp.Shutdown(ctx)
 	}, nil
 }