fix(telemetry): TLS + NR api-key header for OTLP exporter (P0-2)

Distributed tracing was 100% dead in prod across api, worker, and
provisioner — every pod logged a trace-export failure every 15-30s and
`trace_id` was empty on every log line, making cross-service request
correlation impossible. Root cause from BUGHUNT-2026-05-20-T21 P0-2:
tracer.go called `otlptracegrpc.WithInsecure()` (plaintext gRPC) but
the OTel endpoint pointed at the TLS host
`https://otlp.nr-data.net:4317`. The exporter sent cleartext HTTP/2
into a TLS listener; the TLS ServerHello bytes were misread as an
oversized HTTP/2 frame and every export silently dropped. Compounding
the issue, no `api-key` header was sent — NR OTLP ingest requires it,
so even a fixed handshake would have returned UNAUTHENTICATED.

Fix (mirrors the api repo change):
  - Auto-detect TLS from endpoint scheme: `https://` prefix, NR's
    `otlp.*nr-data.net` host, or a `:443` suffix → TLS;
    everything else → plaintext (for local dev / in-cluster
    collectors).
  - When NEW_RELIC_LICENSE_KEY is set, attach it as the `api-key`
    gRPC header on every export (NR ingest contract).
  - Fail-open: empty endpoint = noop shutdown; empty/CHANGE_ME license
    key = WARN + exporter is still constructed (it dials lazily so
    boot never blocks).
  - Stamp INFO `telemetry.tracer_initialized` with endpoint + tls
    + nr_auth flags at boot so an operator can verify the config
    from a single log line.

Regression tests (tracer_test.go) — same set as api / provisioner:
  - TestInitTracer_EmptyEndpointNoop
  - TestInitTracer_Boots
  - TestShouldUseTLS
  - TestStripScheme

`go test ./internal/telemetry/...` green (the rest of the worker
tree currently has an unrelated in-flight edit in internal/jobs/
expire.go from a parallel agent — out of scope for this commit).

Refs: BUGHUNT-2026-05-20-T21.md (P0-2), CLAUDE-AGENT-RELIABILITY-RULES.md
rule 22 (contract changes touch all surfaces — mirror commits in api,
provisioner, infra).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: mastermanas805

internal/telemetry/tracer.go

@@ -2,6 +2,7 @@ package telemetry
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"log/slog"
 	"os"
@@ -14,31 +15,84 @@ import (
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"google.golang.org/grpc/credentials"
 )
 
 // InitTracer configures the global OpenTelemetry tracer provider.
-// When otlpEndpoint is empty, the noop provider remains (fail open).
+//
+// Endpoint selection (in order of precedence):
+//  1. otlpEndpoint argument (typically os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"))
+//  2. if empty → tracing disabled (noop), return a no-op shutdown
+//
+// TLS vs plaintext is auto-detected from the scheme: an `https://` prefix
+// (or an endpoint targeting the well-known NR OTLP host `otlp.*nr-data.net`)
+// uses TLS; everything else (no scheme, `http://`, an in-cluster host like
+// `otel-collector:4317`) falls back to plaintext for local dev.
+//
+// New Relic auth: when NEW_RELIC_LICENSE_KEY is set (and non-sentinel), it
+// is sent as the `api-key` gRPC header on every export — this is the NR
+// OTLP ingest contract. When unset/empty/`CHANGE_ME`, we still construct
+// a working exporter (it just won't be accepted by NR) and log a WARN so
+// the operator knows tracing is configured-but-unauthenticated.
+//
+// Returns shutdown which should be deferred; shutdown is a no-op when
+// tracing is disabled. NEVER crashes — every failure mode falls back to
+// a no-op shutdown so a misconfigured tracer can never block service boot.
+//
+// Historical note (2026-05-20 P0-2): the prior implementation called
+// `otlptracegrpc.WithInsecure()` against the TLS endpoint
+// `https://otlp.nr-data.net:4317` and never sent the NR `api-key` header.
+// Result: every export silently failed (`http2 frame too large`), every
+// log line had `trace_id=""`. The TLS-by-scheme + NR-key-header pair
+// below is the fix; do not revert.
 func InitTracer(serviceName, otlpEndpoint string) func(context.Context) error {
-	ep := strings.TrimSpace(otlpEndpoint)
-	if ep == "" {
+	raw := strings.TrimSpace(otlpEndpoint)
+	if raw == "" {
 		return func(context.Context) error { return nil }
 	}
+
 	if s := strings.TrimSpace(os.Getenv("OTEL_SERVICE_NAME")); s != "" {
 		serviceName = s
 	}
 
-	ep = strings.TrimPrefix(ep, "https://")
-	ep = strings.TrimPrefix(ep, "http://")
+	useTLS := shouldUseTLS(raw)
+	ep := stripScheme(raw)
+
+	licenseKey := strings.TrimSpace(os.Getenv("NEW_RELIC_LICENSE_KEY"))
+	if licenseKey == "" || licenseKey == "CHANGE_ME" {
+		slog.Warn("telemetry.nr_license_missing",
+			"endpoint", ep,
+			"detail", "OTLP exporter constructed but NEW_RELIC_LICENSE_KEY is empty/sentinel; exports will be rejected by NR")
+		licenseKey = ""
+	}
+
+	opts := []otlptracegrpc.Option{
+		otlptracegrpc.WithEndpoint(ep),
+	}
+	if useTLS {
+		opts = append(opts,
+			otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{
+				MinVersion: tls.VersionTLS12,
+			})),
+		)
+	} else {
+		opts = append(opts, otlptracegrpc.WithInsecure())
+	}
+	if licenseKey != "" {
+		// NR OTLP requires this header on every request; without it the
+		// ingest path returns UNAUTHENTICATED and the exporter silently
+		// drops every span.
+		opts = append(opts, otlptracegrpc.WithHeaders(map[string]string{
+			"api-key": licenseKey,
+		}))
+	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	exporter, err := otlptracegrpc.New(ctx,
-		otlptracegrpc.WithEndpoint(ep),
-		otlptracegrpc.WithInsecure(),
-	)
+	exporter, err := otlptracegrpc.New(ctx, opts...)
 	if err != nil {
-		slog.Error("telemetry.otlp_exporter_failed", "error", err, "endpoint", ep)
+		slog.Error("telemetry.otlp_exporter_failed", "error", err, "endpoint", ep, "tls", useTLS)
 		return func(context.Context) error { return nil }
 	}
 
@@ -61,6 +115,12 @@ func InitTracer(serviceName, otlpEndpoint string) func(context.Context) error {
 		propagation.Baggage{},
 	))
 
+	slog.Info("telemetry.tracer_initialized",
+		"service", serviceName,
+		"endpoint", ep,
+		"tls", useTLS,
+		"nr_auth", licenseKey != "")
+
 	return func(shutdownCtx context.Context) error {
 		ctx, cancel := context.WithTimeout(shutdownCtx, 10*time.Second)
 		defer cancel()
@@ -70,3 +130,42 @@ func InitTracer(serviceName, otlpEndpoint string) func(context.Context) error {
 		return nil
 	}
 }
+
+// shouldUseTLS returns true when the OTLP endpoint should be dialed over
+// TLS. Heuristics, in order:
+//  1. explicit `https://` scheme → TLS
+//  2. explicit `http://` scheme → plaintext
+//  3. host matches `otlp.*nr-data.net` (NR's OTLP ingest hosts, all TLS) → TLS
+//  4. host ends in `:443` → TLS
+//  5. otherwise (no scheme, in-cluster collector, etc) → plaintext
+//
+// Exported only for tests.
+func shouldUseTLS(endpoint string) bool {
+	e := strings.ToLower(strings.TrimSpace(endpoint))
+	if strings.HasPrefix(e, "https://") {
+		return true
+	}
+	if strings.HasPrefix(e, "http://") {
+		return false
+	}
+	host := stripScheme(e)
+	// Bare host[:port] — sniff for NR's well-known OTLP hosts and the
+	// 443 port suffix.
+	if strings.Contains(host, "nr-data.net") {
+		return true
+	}
+	if strings.HasSuffix(host, ":443") {
+		return true
+	}
+	return false
+}
+
+// stripScheme removes a leading `http://` or `https://` from the endpoint,
+// returning the bare `host:port` form that otlptracegrpc.WithEndpoint
+// expects.
+func stripScheme(endpoint string) string {
+	e := strings.TrimSpace(endpoint)
+	e = strings.TrimPrefix(e, "https://")
+	e = strings.TrimPrefix(e, "http://")
+	return e
+}

internal/telemetry/tracer_test.go

@@ -0,0 +1,74 @@
+package telemetry
+
+import (
+	"context"
+	"testing"
+)
+
+// TestInitTracer_EmptyEndpointNoop — when the endpoint is unset, the
+// returned shutdown must be a working no-op. This is the fail-open
+// contract for local dev / CI runs where OTel is intentionally off.
+func TestInitTracer_EmptyEndpointNoop(t *testing.T) {
+	shutdown := InitTracer("instant-worker", "")
+	if shutdown == nil {
+		t.Fatal("InitTracer returned nil shutdown for empty endpoint")
+	}
+	if err := shutdown(context.Background()); err != nil {
+		t.Fatalf("noop shutdown returned error: %v", err)
+	}
+}
+
+// TestInitTracer_Boots — with a non-empty endpoint, InitTracer constructs
+// a real exporter without crashing even if NEW_RELIC_LICENSE_KEY is unset.
+// The exporter dials lazily on the first export, so construction must
+// succeed regardless of whether the endpoint is reachable.
+func TestInitTracer_Boots(t *testing.T) {
+	t.Setenv("NEW_RELIC_LICENSE_KEY", "")
+	shutdown := InitTracer("instant-worker", "https://otlp.nr-data.net:4317")
+	if shutdown == nil {
+		t.Fatal("InitTracer returned nil shutdown")
+	}
+	_ = shutdown(context.Background())
+}
+
+// TestShouldUseTLS — the regression case for P0-2: every `https://`
+// endpoint AND every `*nr-data.net` host MUST resolve to TLS=true.
+// Reverting to WithInsecure() for these would silently kill tracing
+// again (the symptom that produced this test).
+func TestShouldUseTLS(t *testing.T) {
+	cases := []struct {
+		endpoint string
+		want     bool
+	}{
+		{"https://otlp.nr-data.net:4317", true},
+		{"https://otlp.eu01.nr-data.net:4317", true},
+		{"otlp.nr-data.net:4317", true},
+		{"otlp.eu01.nr-data.net:4317", true},
+		{"foo.example.com:443", true},
+		{"http://otel-collector.observability:4317", false},
+		{"otel-collector.observability:4317", false},
+		{"localhost:4317", false},
+		{"", false},
+	}
+	for _, c := range cases {
+		got := shouldUseTLS(c.endpoint)
+		if got != c.want {
+			t.Errorf("shouldUseTLS(%q) = %v, want %v", c.endpoint, got, c.want)
+		}
+	}
+}
+
+// TestStripScheme — strips http:// and https:// uniformly.
+func TestStripScheme(t *testing.T) {
+	cases := map[string]string{
+		"https://otlp.nr-data.net:4317": "otlp.nr-data.net:4317",
+		"http://localhost:4317":         "localhost:4317",
+		"otlp.nr-data.net:4317":         "otlp.nr-data.net:4317",
+		"":                              "",
+	}
+	for in, want := range cases {
+		if got := stripScheme(in); got != want {
+			t.Errorf("stripScheme(%q) = %q, want %q", in, got, want)
+		}
+	}
+}