Fix transparent proxy for remote MCP servers behind redirects

Three issues prevented MCPRemoteProxy from connecting to third-party
upstream MCP servers that use HTTP redirects:

1. X-Forwarded-Host leaked the proxy's hostname to the upstream. The
   upstream used it to construct 307 redirect URLs pointing back to
   the proxy, creating a redirect loop. Fix: skip SetXForwarded() for
   remote upstreams (isRemote == true).

2. Go's http.Transport.RoundTrip does not follow redirects, but
   httputil.ReverseProxy uses Transport directly. Upstream 307/308
   redirects (e.g. HTTPS→HTTP scheme changes, path canonicalization)
   were returned to the MCP client which cannot follow them through
   the proxy. Fix: add forwardFollowingRedirects that transparently
   follows up to 10 redirects, preserving method and body for
   307/308 (RFC 7538).

3. When disableUpstreamTokenInjection is true, the client's ToolHive
   JWT was still forwarded to the upstream in the Authorization
   header. Fix: add strip-auth middleware that removes the
   Authorization header before forwarding.

Also adds debug logging for outbound request headers and upstream
response status codes to aid diagnosis of remote proxy issues.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: aron-muon

cmd/thv-operator/controllers/mcpremoteproxy_controller.go

@@ -444,6 +444,9 @@ func (*MCPRemoteProxyReconciler) validateOIDCIssuerURL(proxy *mcpv1alpha1.MCPRem
 // validateJWKSURL validates the JWKS URL scheme in the OIDC config.
 func (*MCPRemoteProxyReconciler) validateJWKSURL(proxy *mcpv1alpha1.MCPRemoteProxy) error {
 	oidcConfig := proxy.Spec.OIDCConfig
+	if oidcConfig == nil {
+		return nil
+	}
 
 	switch oidcConfig.Type {
 	case mcpv1alpha1.OIDCConfigTypeInline:

cmd/thv-operator/controllers/mcpremoteproxy_reconciler_test.go

@@ -916,7 +916,7 @@ func TestValidateSpecConfigurationConditions(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "invalid-cedar-proxy", Namespace: "default"},
 				Spec: mcpv1alpha1.MCPRemoteProxySpec{
 					RemoteURL: "https://mcp.example.com",
-					OIDCConfig: mcpv1alpha1.OIDCConfigRef{
+					OIDCConfig: &mcpv1alpha1.OIDCConfigRef{
 						Type: mcpv1alpha1.OIDCConfigTypeInline,
 						Inline: &mcpv1alpha1.InlineOIDCConfig{
 							Issuer:   "https://auth.example.com",
@@ -942,7 +942,7 @@ func TestValidateSpecConfigurationConditions(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "missing-configmap-proxy", Namespace: "default"},
 				Spec: mcpv1alpha1.MCPRemoteProxySpec{
 					RemoteURL: "https://mcp.example.com",
-					OIDCConfig: mcpv1alpha1.OIDCConfigRef{
+					OIDCConfig: &mcpv1alpha1.OIDCConfigRef{
 						Type: mcpv1alpha1.OIDCConfigTypeInline,
 						Inline: &mcpv1alpha1.InlineOIDCConfig{
 							Issuer:   "https://auth.example.com",
@@ -968,7 +968,7 @@ func TestValidateSpecConfigurationConditions(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "missing-header-secret-proxy", Namespace: "default"},
 				Spec: mcpv1alpha1.MCPRemoteProxySpec{
 					RemoteURL: "https://mcp.example.com",
-					OIDCConfig: mcpv1alpha1.OIDCConfigRef{
+					OIDCConfig: &mcpv1alpha1.OIDCConfigRef{
 						Type: mcpv1alpha1.OIDCConfigTypeInline,
 						Inline: &mcpv1alpha1.InlineOIDCConfig{
 							Issuer:   "https://auth.example.com",
@@ -999,7 +999,7 @@ func TestValidateSpecConfigurationConditions(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "bad-scheme-proxy", Namespace: "default"},
 				Spec: mcpv1alpha1.MCPRemoteProxySpec{
 					RemoteURL: "ftp://bad-scheme.example.com",
-					OIDCConfig: mcpv1alpha1.OIDCConfigRef{
+					OIDCConfig: &mcpv1alpha1.OIDCConfigRef{
 						Type: mcpv1alpha1.OIDCConfigTypeInline,
 						Inline: &mcpv1alpha1.InlineOIDCConfig{
 							Issuer:   "https://auth.example.com",
@@ -1019,7 +1019,7 @@ func TestValidateSpecConfigurationConditions(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "http-jwks-proxy", Namespace: "default"},
 				Spec: mcpv1alpha1.MCPRemoteProxySpec{
 					RemoteURL: "https://mcp.example.com",
-					OIDCConfig: mcpv1alpha1.OIDCConfigRef{
+					OIDCConfig: &mcpv1alpha1.OIDCConfigRef{
 						Type: mcpv1alpha1.OIDCConfigTypeInline,
 						Inline: &mcpv1alpha1.InlineOIDCConfig{
 							Issuer:   "https://auth.example.com",

cmd/thv-operator/test-integration/mcp-remote-proxy/remoteproxy_helpers.go

@@ -152,7 +152,7 @@ func (rb *RemoteProxyBuilder) WithRemoteURL(url string) *RemoteProxyBuilder {
 func (rb *RemoteProxyBuilder) WithInlineOIDCConfigAndJWKS(
 	issuer, audience, jwksURL string,
 ) *RemoteProxyBuilder {
-	rb.proxy.Spec.OIDCConfig = mcpv1alpha1.OIDCConfigRef{
+	rb.proxy.Spec.OIDCConfig = &mcpv1alpha1.OIDCConfigRef{
 		Type: mcpv1alpha1.OIDCConfigTypeInline,
 		Inline: &mcpv1alpha1.InlineOIDCConfig{
 			Issuer:   issuer,

pkg/runner/middleware.go

@@ -5,6 +5,7 @@ package runner
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/stacklok/toolhive/pkg/audit"
 	"github.com/stacklok/toolhive/pkg/auth"
@@ -37,6 +38,7 @@ func GetSupportedMiddlewareFactories() map[string]types.MiddlewareFactory {
 		audit.MiddlewareType:                  audit.CreateMiddleware,
 		recovery.MiddlewareType:               recovery.CreateMiddleware,
 		headerfwd.HeaderForwardMiddlewareName: headerfwd.CreateMiddleware,
+		stripAuthMiddlewareType:               createStripAuthMiddleware,
 	}
 }
 
@@ -263,9 +265,10 @@ func addUpstreamSwapMiddleware(
 		return middlewares, nil
 	}
 
-	// Skip upstream token injection if explicitly disabled
+	// When upstream token injection is disabled, strip the Authorization header
+	// so the client's ToolHive JWT doesn't leak to the upstream server.
 	if config.EmbeddedAuthServerConfig.DisableUpstreamTokenInjection {
-		return middlewares, nil
+		return addAuthHeaderStripMiddleware(middlewares)
 	}
 
 	// Use provided config or defaults
@@ -287,6 +290,45 @@ func addUpstreamSwapMiddleware(
 	return append(middlewares, *upstreamSwapMwConfig), nil
 }
 
+// stripAuthMiddlewareType is the type identifier for the auth header stripping middleware.
+const stripAuthMiddlewareType = "strip-auth"
+
+// addAuthHeaderStripMiddleware adds a middleware that removes the Authorization header
+// before forwarding to the upstream. This prevents the client's ToolHive JWT from
+// leaking to upstream servers that don't expect it.
+func addAuthHeaderStripMiddleware(
+	middlewares []types.MiddlewareConfig,
+) ([]types.MiddlewareConfig, error) {
+	mwConfig, err := types.NewMiddlewareConfig(stripAuthMiddlewareType, struct{}{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create strip-auth middleware config: %w", err)
+	}
+	return append(middlewares, *mwConfig), nil
+}
+
+// createStripAuthMiddleware is the factory function for the auth header stripping middleware.
+func createStripAuthMiddleware(_ *types.MiddlewareConfig, runner types.MiddlewareRunner) error {
+	mw := &stripAuthMiddleware{}
+	runner.AddMiddleware(stripAuthMiddlewareType, mw)
+	return nil
+}
+
+// stripAuthMiddleware removes the Authorization header from requests.
+type stripAuthMiddleware struct{}
+
+// Handler returns the middleware function.
+func (*stripAuthMiddleware) Handler() types.MiddlewareFunction {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			r.Header.Del("Authorization")
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// Close cleans up resources.
+func (*stripAuthMiddleware) Close() error { return nil }
+
 // addAWSStsMiddleware adds AWS STS middleware if configured.
 // Returns an error if AWSStsConfig is set but RemoteURL is empty, because
 // SigV4 signing is only meaningful for remote MCP servers.

pkg/runner/middleware_test.go

@@ -256,6 +256,7 @@ func TestAddUpstreamSwapMiddleware(t *testing.T) {
 		name         string
 		config       *RunConfig
 		wantAppended bool
+		wantType     string // expected middleware type when appended
 	}{
 		{
 			name:         "nil EmbeddedAuthServerConfig returns input unchanged",
@@ -269,15 +270,17 @@ func TestAddUpstreamSwapMiddleware(t *testing.T) {
 				UpstreamSwapConfig:       nil,
 			},
 			wantAppended: true,
+			wantType:     upstreamswap.MiddlewareType,
 		},
 		{
-			name: "EmbeddedAuthServerConfig with DisableUpstreamTokenInjection skips middleware",
+			name: "DisableUpstreamTokenInjection adds strip-auth middleware instead",
 			config: func() *RunConfig {
 				cfg := createMinimalAuthServerConfig()
 				cfg.DisableUpstreamTokenInjection = true
 				return &RunConfig{EmbeddedAuthServerConfig: cfg}
 			}(),
-			wantAppended: false,
+			wantAppended: true,
+			wantType:     stripAuthMiddlewareType,
 		},
 		{
 			name: "EmbeddedAuthServerConfig set with explicit UpstreamSwapConfig uses provided config",
@@ -288,6 +291,7 @@ func TestAddUpstreamSwapMiddleware(t *testing.T) {
 				},
 			},
 			wantAppended: true,
+			wantType:     upstreamswap.MiddlewareType,
 		},
 		{
 			name: "EmbeddedAuthServerConfig with custom header strategy config",
@@ -299,6 +303,7 @@ func TestAddUpstreamSwapMiddleware(t *testing.T) {
 				},
 			},
 			wantAppended: true,
+			wantType:     upstreamswap.MiddlewareType,
 		},
 	}
 
@@ -318,20 +323,20 @@ func TestAddUpstreamSwapMiddleware(t *testing.T) {
 			// Should have one additional entry.
 			require.Len(t, got, len(initial)+1)
 			added := got[len(got)-1]
-			assert.Equal(t, upstreamswap.MiddlewareType, added.Type)
-
-			// Verify serialized params contain the expected config.
-			var params upstreamswap.MiddlewareParams
-			require.NoError(t, json.Unmarshal(added.Parameters, &params))
+			assert.Equal(t, tt.wantType, added.Type)
 
-			if tt.config.UpstreamSwapConfig != nil {
-				// Should use the provided config
-				require.NotNil(t, params.Config)
-				assert.Equal(t, tt.config.UpstreamSwapConfig.HeaderStrategy, params.Config.HeaderStrategy)
-				assert.Equal(t, tt.config.UpstreamSwapConfig.CustomHeaderName, params.Config.CustomHeaderName)
-			} else {
-				// Should use defaults (empty config is valid)
-				require.NotNil(t, params.Config)
+			// For upstreamswap type, verify serialized params
+			if tt.wantType == upstreamswap.MiddlewareType {
+				var params upstreamswap.MiddlewareParams
+				require.NoError(t, json.Unmarshal(added.Parameters, &params))
+
+				if tt.config.UpstreamSwapConfig != nil {
+					require.NotNil(t, params.Config)
+					assert.Equal(t, tt.config.UpstreamSwapConfig.HeaderStrategy, params.Config.HeaderStrategy)
+					assert.Equal(t, tt.config.UpstreamSwapConfig.CustomHeaderName, params.Config.CustomHeaderName)
+				} else {
+					require.NotNil(t, params.Config)
+				}
 			}
 		})
 	}
@@ -344,6 +349,7 @@ func TestPopulateMiddlewareConfigs_UpstreamSwap(t *testing.T) {
 		name               string
 		config             *RunConfig
 		wantUpstreamSwap   bool
+		wantStripAuth      bool
 		wantHeaderStrategy string
 	}{
 		{
@@ -357,13 +363,14 @@ func TestPopulateMiddlewareConfigs_UpstreamSwap(t *testing.T) {
 			wantUpstreamSwap: false,
 		},
 		{
-			name: "DisableUpstreamTokenInjection omits upstream-swap",
+			name: "DisableUpstreamTokenInjection adds strip-auth instead of upstream-swap",
 			config: func() *RunConfig {
 				cfg := createMinimalAuthServerConfig()
 				cfg.DisableUpstreamTokenInjection = true
 				return &RunConfig{EmbeddedAuthServerConfig: cfg}
 			}(),
 			wantUpstreamSwap: false,
+			wantStripAuth:    true,
 		},
 		{
 			name: "explicit UpstreamSwapConfig is used",
@@ -385,20 +392,25 @@ func TestPopulateMiddlewareConfigs_UpstreamSwap(t *testing.T) {
 			err := PopulateMiddlewareConfigs(tt.config)
 			require.NoError(t, err)
 
-			var found bool
+			var foundSwap bool
+			var foundStrip bool
 			var foundConfig *types.MiddlewareConfig
 			for i, mw := range tt.config.MiddlewareConfigs {
 				if mw.Type == upstreamswap.MiddlewareType {
-					found = true
+					foundSwap = true
 					foundConfig = &tt.config.MiddlewareConfigs[i]
-					break
+				}
+				if mw.Type == stripAuthMiddlewareType {
+					foundStrip = true
 				}
 			}
-			assert.Equal(t, tt.wantUpstreamSwap, found,
+			assert.Equal(t, tt.wantUpstreamSwap, foundSwap,
 				"upstream-swap middleware presence mismatch")
+			assert.Equal(t, tt.wantStripAuth, foundStrip,
+				"strip-auth middleware presence mismatch")
 
 			// Verify config values if we expect the middleware and have specific expectations
-			if found && tt.wantHeaderStrategy != "" {
+			if foundSwap && tt.wantHeaderStrategy != "" {
 				var params upstreamswap.MiddlewareParams
 				require.NoError(t, json.Unmarshal(foundConfig.Parameters, &params))
 				require.NotNil(t, params.Config)