Thread per-backend headerForward through aggregator, workloads, and CLI

The static-mode discoverer now keys headerForward by normalized
backend name and stamps each Backend with its config at discovery
time. The Kubernetes workload discoverer surfaces the same field on
managed entries, and the health monitor forwards it through to client
calls. vMCP startup ingests the operator-emitted
TOOLHIVE_HEADER_FORWARD_* env vars and routes the resulting
per-backend map through serve into the discoverer.

Author: ChrisJBurns

pkg/vmcp/aggregator/discoverer.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stacklok/toolhive/pkg/groups"
 	"github.com/stacklok/toolhive/pkg/vmcp"
 	"github.com/stacklok/toolhive/pkg/vmcp/config"
+	"github.com/stacklok/toolhive/pkg/vmcp/headerforward/wirefmt"
 	"github.com/stacklok/toolhive/pkg/vmcp/workloads"
 	workloadsmgr "github.com/stacklok/toolhive/pkg/workloads"
 )
@@ -33,6 +34,15 @@ type backendDiscoverer struct {
 	authConfig       *config.OutgoingAuthConfig
 	staticBackends   []config.StaticBackendConfig // Pre-configured backends for static mode
 	groupRef         string                       // Group reference for static mode metadata
+
+	// headerForwardByBackend is keyed by the NORMALIZED backend name (the
+	// suffix the operator emits in TOOLHIVE_HEADER_FORWARD_<entry>). The
+	// canonical backend name from the static config is normalized on
+	// lookup via wirefmt.NormalizeForEnvVar so the keying matches
+	// the operator-side env-var encoding. Nil/empty when no entry declared
+	// headerForward. Only populated in static mode — dynamic mode reads
+	// headerForward directly from the MCPServerEntry CRD.
+	headerForwardByBackend map[string]*vmcp.HeaderForwardConfig
 }
 
 // NewUnifiedBackendDiscoverer creates a unified backend discoverer that works with both
@@ -55,17 +65,24 @@ func NewUnifiedBackendDiscoverer(
 
 // NewUnifiedBackendDiscovererWithStaticBackends creates a backend discoverer for static mode
 // with pre-configured backends, eliminating the need for K8s API access.
+//
+// headerForwardByBackend carries per-backend HeaderForwardConfig (keyed by
+// backend name) constructed at startup from the operator-emitted env vars.
+// Pass nil when no entry in the group declares headerForward — the discoverer
+// will simply leave Backend.HeaderForward nil for every backend.
 func NewUnifiedBackendDiscovererWithStaticBackends(
 	staticBackends []config.StaticBackendConfig,
 	authConfig *config.OutgoingAuthConfig,
 	groupRef string,
+	headerForwardByBackend map[string]*vmcp.HeaderForwardConfig,
 ) BackendDiscoverer {
 	return &backendDiscoverer{
-		workloadsManager: nil, // Not needed in static mode
-		groupsManager:    nil, // Not needed in static mode
-		authConfig:       authConfig,
-		staticBackends:   staticBackends,
-		groupRef:         groupRef,
+		workloadsManager:       nil, // Not needed in static mode
+		groupsManager:          nil, // Not needed in static mode
+		authConfig:             authConfig,
+		staticBackends:         staticBackends,
+		groupRef:               groupRef,
+		headerForwardByBackend: headerForwardByBackend,
 	}
 }
 
@@ -272,6 +289,7 @@ func (d *backendDiscoverer) discoverFromStaticConfig() []vmcp.Backend {
 			Type:          vmcp.BackendType(staticBackend.Type),
 			CABundlePath:  staticBackend.CABundlePath,
 			HealthStatus:  vmcp.BackendHealthy, // Assume healthy, actual health check happens later
+			HeaderForward: d.headerForwardByBackend[wirefmt.NormalizeForEnvVar(staticBackend.Name)],
 			Metadata:      staticBackend.Metadata,
 		}
 

pkg/vmcp/aggregator/discoverer_test.go

@@ -1186,6 +1186,7 @@ func TestStaticBackendDiscoverer_EmptyBackendList(t *testing.T) {
 		[]config.StaticBackendConfig{}, // Empty slice, not nil
 		nil,                            // No auth config
 		"test-group",
+		nil, // No headerForward map
 	)
 
 	// This should return empty list without panicking
@@ -1281,6 +1282,7 @@ func TestStaticBackendDiscoverer_MetadataGroupOverride(t *testing.T) {
 				tt.staticBackends,
 				nil, // No auth config needed for this test
 				tt.groupRef,
+				nil, // No headerForward map for this test
 			)
 
 			backends, err := discoverer.Discover(ctx, tt.groupRef)
@@ -1400,6 +1402,7 @@ func TestStaticBackendDiscoverer_EntryBackendFields(t *testing.T) {
 				tt.staticBackends,
 				nil, // No auth config needed for this test
 				tt.groupRef,
+				nil, // No headerForward map for this test
 			)
 
 			backends, err := discoverer.Discover(ctx, tt.groupRef)
@@ -1466,6 +1469,7 @@ func TestBackendDiscoverer_Discover_DeterministicOrdering(t *testing.T) {
 				tc.staticBackends,
 				nil, // No auth config needed for this test
 				"test-group",
+				nil, // No headerForward map for this test
 			)
 
 			backends, err := discoverer.Discover(ctx, "test-group")

pkg/vmcp/cli/header_forward_env.go

@@ -0,0 +1,74 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+	"encoding/json"
+	"log/slog"
+	"strings"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
+	"github.com/stacklok/toolhive/pkg/vmcp/headerforward/wirefmt"
+)
+
+// readHeaderForwardFromEnv reconstructs the per-backend
+// vmcp.HeaderForwardConfig map by walking environment variables emitted by
+// the operator on the vMCP pod.
+//
+// The operator emits one JSON-encoded manifest env var per backend named
+// "TOOLHIVE_HEADER_FORWARD_<NORMALIZED_ENTRY>". The JSON value carries
+// every configured header for that backend with original (un-normalized)
+// names preserved:
+//
+//	{
+//	  "addPlaintextHeaders": {"X-MCP-Toolsets":"projects,issues"},
+//	  "addHeadersFromSecret": {"X-Api-Key":"HEADER_FORWARD_X_API_KEY_<entry>"}
+//	}
+//
+// AddHeadersFromSecret values are secret IDENTIFIERS, not values. Secret
+// values resolve later inside resolveHeaderForward via
+// secrets.EnvironmentProvider, which reads TOOLHIVE_SECRET_<identifier>
+// env vars (delivered separately via valueFrom.secretKeyRef so the value
+// never enters the operator's view of the world).
+//
+// The map key is the normalized entry segment from the env-var suffix —
+// the SAME value the operator's GenerateHeaderForwardManifestEnvVarName
+// produces for that backend's name. Callers look up by passing the
+// original backend name through ctrlutil.NormalizeHeaderForEnvVar before
+// indexing.
+func readHeaderForwardFromEnv(envEntries []string) map[string]*vmcp.HeaderForwardConfig {
+	out := make(map[string]*vmcp.HeaderForwardConfig)
+	for _, entry := range envEntries {
+		name, value, ok := strings.Cut(entry, "=")
+		if !ok || !strings.HasPrefix(name, wirefmt.ManifestEnvVarPrefix) {
+			continue
+		}
+		ownerSegment := strings.TrimPrefix(name, wirefmt.ManifestEnvVarPrefix)
+
+		var cfg vmcp.HeaderForwardConfig
+		if err := json.Unmarshal([]byte(value), &cfg); err != nil {
+			// A malformed manifest is a programmer error in the operator;
+			// log and skip rather than fail the whole startup. The backend
+			// will simply have no headerForward attached.
+			slog.Warn("invalid headerForward manifest from env, skipping",
+				"envvar", name, "error", err)
+			continue
+		}
+		// wirefmt.NormalizeForEnvVar maps two distinct entry names with the
+		// same uppercased+sanitized form to the same env-var suffix (e.g.
+		// "github-copilot" and "github_copilot"). DNS-1123 forbids
+		// underscores in entry names, so this collision is unreachable in
+		// production today — but a future relaxation, a migration that
+		// allows mixed casing, or a third-party producing manifests can all
+		// land us here. Surface the collision loudly: the second config
+		// silently overwriting the first would mask a misconfiguration that
+		// is otherwise extremely hard to debug.
+		if _, dup := out[ownerSegment]; dup {
+			slog.Warn("duplicate headerForward manifest env var; later value overrides earlier",
+				"envvar", name, "ownerSegment", ownerSegment)
+		}
+		out[ownerSegment] = &cfg
+	}
+	return out
+}

pkg/vmcp/cli/header_forward_env_test.go

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
+)
+
+// TestReadHeaderForwardFromEnv covers reconstructing the per-backend
+// HeaderForwardConfig from the JSON-encoded TOOLHIVE_HEADER_FORWARD_<entry>
+// env vars the operator emits on the vMCP pod. Original header casing /
+// punctuation is preserved by the JSON value, so the runtime reconstructs
+// "X-MCP-Toolsets" rather than "X_MCP_TOOLSETS".
+//
+// The map is keyed by the normalized entry segment from the env-var
+// suffix; the discoverer normalizes Backend.Name through
+// ctrlutil.NormalizeHeaderForEnvVar before indexing.
+func TestReadHeaderForwardFromEnv(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		env      []string
+		validate func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig)
+	}{
+		{
+			name: "no env vars yields empty map",
+			env:  []string{"HOME=/root", "PATH=/bin"},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				assert.Empty(t, m)
+			},
+		},
+		{
+			name: "plaintext header decodes preserving original casing and hyphens",
+			env: []string{
+				`TOOLHIVE_HEADER_FORWARD_GITHUB_COPILOT={"addPlaintextHeaders":{"X-MCP-Toolsets":"projects,issues"}}`,
+				"PATH=/bin",
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				cfg := m["GITHUB_COPILOT"]
+				require.NotNil(t, cfg)
+				assert.Equal(t, map[string]string{"X-MCP-Toolsets": "projects,issues"}, cfg.AddPlaintextHeaders)
+				assert.Empty(t, cfg.AddHeadersFromSecret)
+			},
+		},
+		{
+			name: "secret header decodes to identifier (value not in manifest)",
+			env: []string{
+				`TOOLHIVE_HEADER_FORWARD_STRIPE={"addHeadersFromSecret":{"X-Api-Key":"HEADER_FORWARD_X_API_KEY_STRIPE"}}`,
+				// The secret-value env var carries the resolved value via
+				// secretKeyRef; the walker must NOT treat it as a manifest.
+				"TOOLHIVE_SECRET_HEADER_FORWARD_X_API_KEY_STRIPE=resolved-secret-value",
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				assert.NotContains(t, m, "X_API_KEY_STRIPE", "secret env var must not be parsed as a manifest")
+				cfg := m["STRIPE"]
+				require.NotNil(t, cfg)
+				assert.Empty(t, cfg.AddPlaintextHeaders)
+				assert.Equal(t,
+					map[string]string{"X-Api-Key": "HEADER_FORWARD_X_API_KEY_STRIPE"},
+					cfg.AddHeadersFromSecret,
+				)
+			},
+		},
+		{
+			name: "malformed manifest is skipped without failing other backends",
+			env: []string{
+				`TOOLHIVE_HEADER_FORWARD_BROKEN=not-json`,
+				`TOOLHIVE_HEADER_FORWARD_DEMO={"addPlaintextHeaders":{"X-Trace":"ok"}}`,
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				assert.NotContains(t, m, "BROKEN")
+				cfg := m["DEMO"]
+				require.NotNil(t, cfg)
+				assert.Equal(t, map[string]string{"X-Trace": "ok"}, cfg.AddPlaintextHeaders)
+			},
+		},
+		{
+			name: "mixed plaintext + secret in one manifest scoped per backend",
+			env: []string{
+				`TOOLHIVE_HEADER_FORWARD_ALPHA={"addPlaintextHeaders":{"X-Trace":"alpha-trace"},"addHeadersFromSecret":{"X-Token":"HEADER_FORWARD_X_TOKEN_ALPHA"}}`,
+				`TOOLHIVE_HEADER_FORWARD_BETA={"addPlaintextHeaders":{"X-Trace":"beta-trace"}}`,
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+
+				alpha := m["ALPHA"]
+				require.NotNil(t, alpha)
+				assert.Equal(t, map[string]string{"X-Trace": "alpha-trace"}, alpha.AddPlaintextHeaders)
+				assert.Equal(t,
+					map[string]string{"X-Token": "HEADER_FORWARD_X_TOKEN_ALPHA"},
+					alpha.AddHeadersFromSecret,
+				)
+
+				beta := m["BETA"]
+				require.NotNil(t, beta)
+				assert.Equal(t, map[string]string{"X-Trace": "beta-trace"}, beta.AddPlaintextHeaders)
+				assert.Empty(t, beta.AddHeadersFromSecret)
+			},
+		},
+		{
+			name: "malformed env entry without '=' is skipped",
+			env: []string{
+				"NO_EQUALS_HERE",
+				`TOOLHIVE_HEADER_FORWARD_DEMO={"addPlaintextHeaders":{"X-Trace":"ok"}}`,
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				cfg := m["DEMO"]
+				require.NotNil(t, cfg)
+				assert.Equal(t, map[string]string{"X-Trace": "ok"}, cfg.AddPlaintextHeaders)
+			},
+		},
+		{
+			// DNS-1123 forbids underscores in MCPServerEntry names today, so
+			// real entries with the same NormalizeForEnvVar output cannot
+			// coexist. We pass two manifests that share the same suffix
+			// directly to verify the runtime keeps "last wins" behavior and
+			// surfaces a warning rather than silently dropping the first.
+			name: "duplicate manifest suffix retains last value",
+			env: []string{
+				`TOOLHIVE_HEADER_FORWARD_DUP={"addPlaintextHeaders":{"X-Trace":"first"}}`,
+				`TOOLHIVE_HEADER_FORWARD_DUP={"addPlaintextHeaders":{"X-Trace":"second"}}`,
+			},
+			validate: func(t *testing.T, m map[string]*vmcp.HeaderForwardConfig) {
+				t.Helper()
+				cfg := m["DUP"]
+				require.NotNil(t, cfg)
+				assert.Equal(t, map[string]string{"X-Trace": "second"}, cfg.AddPlaintextHeaders,
+					"duplicate manifest must keep the later value (last-write-wins)")
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			tt.validate(t, readHeaderForwardFromEnv(tt.env))
+		})
+	}
+}

pkg/vmcp/cli/serve.go

@@ -566,10 +566,21 @@ func discoverBackends(
 	if len(cfg.Backends) > 0 {
 		// Static mode: use pre-configured backends from config.
 		slog.Info(fmt.Sprintf("Static mode: using %d pre-configured backends", len(cfg.Backends)))
+
+		// Reconstruct per-backend HeaderForwardConfig from env vars the
+		// operator emitted on this pod. Plaintext header values are inline
+		// in the JSON manifest; secret-backed headers carry only identifiers
+		// here and resolve later via secrets.EnvironmentProvider at request
+		// time. Map keys are the normalized entry segment from the env-var
+		// suffix; the discoverer normalizes Backend.Name through
+		// ctrlutil.NormalizeHeaderForEnvVar to look up the matching entry.
+		// Returns an empty map when no entry in the group declared
+		// headerForward — the common case.
 		discoverer = aggregator.NewUnifiedBackendDiscovererWithStaticBackends(
 			cfg.Backends,
 			cfg.OutgoingAuth,
 			cfg.Group,
+			readHeaderForwardFromEnv(os.Environ()),
 		)
 	} else {
 		// Dynamic mode: discover backends at runtime from the active workload manager (K8s or local).

pkg/vmcp/health/monitor.go

@@ -427,13 +427,19 @@ func (m *Monitor) performHealthCheck(ctx context.Context, backend *vmcp.Backend)
 		return
 	}
 
-	// Create BackendTarget from Backend
+	// Create BackendTarget from Backend. Carry CA bundle and header-forward config
+	// so health checks hit the backend with the same TLS trust and header injection
+	// as list/call requests — otherwise a healthy-to-the-monitor backend could fail
+	// for real traffic (or vice versa).
 	target := &vmcp.BackendTarget{
 		WorkloadID:    backend.ID,
 		WorkloadName:  backend.Name,
 		BaseURL:       backend.BaseURL,
 		TransportType: backend.TransportType,
+		CABundlePath:  backend.CABundlePath,
+		CABundleData:  backend.CABundleData,
 		AuthConfig:    backend.AuthConfig,
+		HeaderForward: backend.HeaderForward,
 		HealthStatus:  vmcp.BackendUnknown, // Status is determined by the health check
 		Metadata:      backend.Metadata,
 	}