test(e2e): mTLS datapath scenario for Kind

config/ingress/base/traefik.yaml

@@ -74,9 +74,23 @@ rules:
   - apiGroups: ["networking.k8s.io"]
     resources: ["ingressclasses"]
     verbs: ["get", "list", "watch"]
-  - apiGroups: ["traefik.io"]
+  # Traefik v2.10 starts informers for every CRD kind in BOTH the traefik.io and
+  # legacy traefik.containo.us groups when those CRDs are installed. A forbidden
+  # list/watch on any one of them stalls the CRD provider so no IngressRoute is
+  # loaded and mtls hosts 404. Grant the full kind set in both groups.
+  - apiGroups: ["traefik.io", "traefik.containo.us"]
     resources:
-      ["ingressroutes", "ingressroutetcps", "middlewares", "tlsoptions", "tlsstores", "serverstransports", "traefikservices"]
+      [
+        "ingressroutes",
+        "ingressroutetcps",
+        "ingressrouteudps",
+        "middlewares",
+        "middlewaretcps",
+        "tlsoptions",
+        "tlsstores",
+        "serverstransports",
+        "traefikservices",
+      ]
     verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1

config/ingress/overlays/http/deployment-args.patch.yaml

@@ -3,15 +3,27 @@
   value:
     - --providers.kubernetesingress=true
     - --providers.kubernetesingress.namespaces=registry,mcp-sentinel,mcp-servers,mcp-servers-org,mcp-servers-public
+    # The mtls terminate+re-encrypt path is delivered entirely through Traefik
+    # CRDs (IngressRoute + Middleware + TLSOption + ServersTransport). Without the
+    # CRD provider Traefik ignores them and the mtls host 404s, so enable it for
+    # the same server namespaces. web stays plaintext (no redirect) for the HTTP
+    # test flows.
+    - --providers.kubernetescrd=true
+    - --providers.kubernetescrd.namespaces=mcp-servers,mcp-servers-org,mcp-servers-public
     - --entrypoints.web.address=:8000
     - --entrypoints.web.forwardedHeaders.insecure=false
     - --entrypoints.websecure.address=:8443
     - --entrypoints.websecure.forwardedHeaders.insecure=false
+    - --entrypoints.websecure.http.tls=true
     - --entrypoints.traefik.address=:8080
     - --ping=true
     - --ping.entryPoint=traefik
     - --providers.file.filename=/etc/traefik/dynamic/dynamic.yml
     - --providers.file.watch=true
+    # The mtls Middleware injects the verified SPIFFE identity via this local
+    # plugin (source is mounted by the base Deployment); register it so the
+    # middleware resolves instead of failing the route.
+    - --experimental.localPlugins.spiffe-identity.moduleName=github.com/Agent-Hellboy/mcp-runtime/traefik-plugins/spiffe-identity
 - op: replace
   path: /spec/template/spec/containers/0/ports
   value:

internal/cli/setup/platform/analytics.go

@@ -677,9 +677,6 @@ func renderAnalyticsManifest(content string, images AnalyticsImageSet, imagePull
 	if mode, ok := setupplan.NormalizePlatformMode(platformMode); ok && mode != "" {
 		replacements[`PLATFORM_MODE: "tenant"`] = fmt.Sprintf(`PLATFORM_MODE: "%s"`, mode)
 	}
-	if issuer := strings.TrimSpace(os.Getenv("MCP_MTLS_CLUSTER_ISSUER")); issuer != "" {
-		replacements[`MCP_MTLS_CLUSTER_ISSUER: ""`] = fmt.Sprintf(`MCP_MTLS_CLUSTER_ISSUER: %q`, issuer)
-	}
 	if strings.TrimSpace(images.Ingest) != "" {
 		replacements["image: mcp-sentinel-ingest:latest"] = "image: " + images.Ingest
 	}
@@ -800,6 +797,7 @@ func renderAnalyticsConfigManifestWithReaders(content, platformMode string, imag
 		"PLATFORM_REGISTRY_URL",
 		"PLATFORM_TRAEFIK_NAMESPACE",
 		"PLATFORM_TEAM_TRAEFIK_WATCH",
+		"MCP_MTLS_CLUSTER_ISSUER",
 	} {
 		if envValue := setupAnalyticsConfigEnvValue(key); envValue != "" {
 			manifest.Data[key] = envValue

internal/cli/setup/platform/helpers_test.go

@@ -1485,6 +1485,39 @@ func TestRenderAnalyticsSecretManifestDisablesExistingDevLoginsOutsideTestMode(t
 	}
 }
 
+func TestRenderAnalyticsConfigManifestInjectsMTLSClusterIssuer(t *testing.T) {
+	t.Setenv("MCP_MTLS_CLUSTER_ISSUER", "mcp-runtime-ca")
+	mock := &core.MockExecutor{
+		CommandFunc: func(spec core.ExecSpec) *core.MockCommand {
+			return &core.MockCommand{Args: spec.Args}
+		},
+	}
+	kubectl := core.NewTestKubectlClient(mock)
+
+	rendered, err := renderAnalyticsConfigManifest(kubectl, `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mcp-sentinel-config
+  namespace: mcp-sentinel
+data:
+  MCP_MTLS_CLUSTER_ISSUER: ""
+  PLATFORM_MODE: "tenant"
+`, setupplan.PlatformModeTenant, AnalyticsImageSet{})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var payload struct {
+		Data map[string]string `yaml:"data"`
+	}
+	if err := yaml.Unmarshal([]byte(rendered), &payload); err != nil {
+		t.Fatalf("unmarshal rendered config: %v", err)
+	}
+	if got := payload.Data["MCP_MTLS_CLUSTER_ISSUER"]; got != "mcp-runtime-ca" {
+		t.Fatalf("expected MCP_MTLS_CLUSTER_ISSUER to be injected into the sentinel ConfigMap, got %q", got)
+	}
+}
+
 func TestRenderAnalyticsConfigManifestPreservesExistingPublicOAuthConfig(t *testing.T) {
 	mock := &core.MockExecutor{
 		CommandFunc: func(spec core.ExecSpec) *core.MockCommand {

internal/operator/controller_test.go

@@ -1089,6 +1089,34 @@ func TestCheckIngressReady(t *testing.T) {
 		assertEqual(t, "ready", ready, false)
 	})
 
+	t.Run("mtls server is ready when its IngressRoute exists", func(t *testing.T) {
+		mtlsScheme := traefikScheme(t)
+		server := mtlsServer()
+		route := crFixture(ingressRouteGVK, server.Name, server.Namespace)
+		client := fake.NewClientBuilder().WithScheme(mtlsScheme).WithObjects(server, route).Build()
+		r := MCPServerReconciler{Client: client, Scheme: mtlsScheme}
+		ready, err := r.checkIngressReady(context.Background(), server)
+		if err != nil {
+			t.Fatalf("failed to check ingress readiness: %v", err)
+		}
+		assertEqual(t, "ready", ready, true)
+	})
+
+	t.Run("mtls server is not ready without an IngressRoute", func(t *testing.T) {
+		mtlsScheme := traefikScheme(t)
+		server := mtlsServer()
+		// Only the legacy passthrough IngressRouteTCP exists; the terminate-and-
+		// re-encrypt model no longer uses it, so the server must not read ready.
+		legacy := crFixture(ingressRouteTCPGVK, server.Name, server.Namespace)
+		client := fake.NewClientBuilder().WithScheme(mtlsScheme).WithObjects(server, legacy).Build()
+		r := MCPServerReconciler{Client: client, Scheme: mtlsScheme}
+		ready, err := r.checkIngressReady(context.Background(), server)
+		if err != nil {
+			t.Fatalf("failed to check ingress readiness: %v", err)
+		}
+		assertEqual(t, "ready", ready, false)
+	})
+
 	t.Run("returns true when ingress has load balancer status", func(t *testing.T) {
 		mcpServer := &mcpv1alpha1.MCPServer{
 			ObjectMeta: metav1.ObjectMeta{Name: "test-server", Namespace: "default"},

internal/operator/mtls.go

@@ -414,9 +414,20 @@ func (r *MCPServerReconciler) reconcileMTLSIngress(ctx context.Context, mcpServe
 			"match":       match,
 			"kind":        "Rule",
 			"middlewares": []any{map[string]any{"name": mtlsMiddlewareName(mcpServer)}},
+			// The IngressRoute references the Kubernetes Service port (ServicePort,
+			// typically 80), not the gateway container port. Traefik resolves
+			// endpoints via the Service and connects to each pod at the targetPort
+			// (Gateway.Port, e.g. 8091) automatically. Using the container port
+			// directly causes "service port not found" because that port number does
+			// not appear in the Service's spec.ports.
 			"services": []any{map[string]any{
-				"name":             mcpServer.Name,
-				"port":             int64(mcpServer.Spec.Gateway.Port),
+				"name": mcpServer.Name,
+				"port": int64(mcpServer.Spec.ServicePort),
+				// scheme: https tells Traefik to connect to the backend over TLS.
+				// Without it, Traefik defaults to HTTP (port 80 is not 443), and
+				// the ServersTransport TLS config is ignored, causing the gateway to
+				// return 400 "Client sent an HTTP request to an HTTPS server".
+				"scheme":           "https",
 				"serversTransport": mtlsServersTransportName(mcpServer),
 			}},
 		}},

internal/operator/mtls_certs_test.go

@@ -19,9 +19,10 @@ func mtlsServer() *mcpv1alpha1.MCPServer {
 	return &mcpv1alpha1.MCPServer{
 		ObjectMeta: metav1.ObjectMeta{Name: "secure-server", Namespace: "mcp-servers"},
 		Spec: mcpv1alpha1.MCPServerSpec{
-			Image:   "example.com/secure-server",
-			Gateway: &mcpv1alpha1.GatewayConfig{Enabled: true, Port: 8091, Image: "example.com/gw:latest"},
-			Auth:    &mcpv1alpha1.AuthConfig{Mode: mcpv1alpha1.AuthModeMTLS, TrustDomain: "example.org"},
+			Image:       "example.com/secure-server",
+			ServicePort: 80,
+			Gateway:     &mcpv1alpha1.GatewayConfig{Enabled: true, Port: 8091, Image: "example.com/gw:latest"},
+			Auth:        &mcpv1alpha1.AuthConfig{Mode: mcpv1alpha1.AuthModeMTLS, TrustDomain: "example.org"},
 		},
 	}
 }

internal/operator/mtls_ingress_test.go

@@ -102,13 +102,27 @@ func TestReconcileMTLSIngressGeneratesTraefikResources(t *testing.T) {
 	if len(routes) != 1 {
 		t.Fatalf("routes = %d, want 1", len(routes))
 	}
-	match, _ := routes[0].(map[string]any)["match"].(string)
+	route0 := routes[0].(map[string]any)
+	match, _ := route0["match"].(string)
 	if !strings.Contains(match, "Host(`mcp.example.com`)") || !strings.Contains(match, "PathPrefix(`/secure-server/mcp`)") {
 		t.Fatalf("match = %q, want host + path prefix", match)
 	}
 	if tlsName, _, _ := unstructured.NestedString(ir.Object, "spec", "tls", "options", "name"); tlsName != mtlsTLSOptionName(server) {
 		t.Fatalf("tls.options.name = %q", tlsName)
 	}
+	// The IngressRoute must reference the Kubernetes Service port (80), not the
+	// gateway container port (8091). Traefik resolves endpoints from the Service
+	// and then connects to pod:targetPort — using the container port causes
+	// "service port not found" because it never appears in Service.spec.ports.
+	services, _, _ := unstructured.NestedSlice(route0, "services")
+	if len(services) != 1 {
+		t.Fatalf("route services = %d, want 1", len(services))
+	}
+	svcPort, _, _ := unstructured.NestedFieldNoCopy(services[0].(map[string]any), "port")
+	if svcPort != int64(server.Spec.ServicePort) {
+		t.Fatalf("IngressRoute service port = %v, want ServicePort (%d) not Gateway.Port (%d)",
+			svcPort, server.Spec.ServicePort, server.Spec.Gateway.Port)
+	}
 }
 
 func TestReconcileMTLSIngressNeverSetsPerRouteSecretName(t *testing.T) {

internal/operator/status.go

@@ -70,8 +70,11 @@ func (r *MCPServerReconciler) checkServiceReady(ctx context.Context, mcpServer *
 
 func (r *MCPServerReconciler) checkIngressReady(ctx context.Context, mcpServer *mcpv1alpha1.MCPServer) (bool, error) {
 	if serverUsesMTLS(mcpServer) {
+		// The terminate-and-re-encrypt mTLS model serves traffic through a
+		// path-based Traefik IngressRoute (the legacy passthrough IngressRouteTCP
+		// is deleted during reconcile), so readiness must track the IngressRoute.
 		route := &unstructured.Unstructured{}
-		route.SetGroupVersionKind(ingressRouteTCPGVK)
+		route.SetGroupVersionKind(ingressRouteGVK)
 		if err := r.Get(ctx, types.NamespacedName{Name: mcpServer.Name, Namespace: mcpServer.Namespace}, route); err != nil {
 			if errors.IsNotFound(err) {
 				return false, nil

services/mcp-gateway/cert_reload_test.go

@@ -106,15 +106,18 @@ func TestCertReloaderWatchReloadsOnChange(t *testing.T) {
 	defer cancel()
 	go r.watch(ctx, 20*time.Millisecond)
 
-	// Bump modtime into the future so the poll reliably detects the change.
 	second := writeKeyPair(t, certPath, keyPath, "cert-2")
-	future := time.Now().Add(2 * time.Second)
-	_ = os.Chtimes(certPath, future, future)
 
-	// Generous deadline so the 20ms poll is detected even when CI runs the
-	// service test suites in parallel under a constrained CPU quota.
+	// watch() captures its baseline modtime asynchronously, so under CI load it
+	// may start after this rewrite and baseline on cert-2's modtime — then it
+	// would never observe a "change" and never reload. Push the modtime strictly
+	// forward on every poll so the watcher sees a fresh change regardless of when
+	// its goroutine started; that makes the reload deterministic rather than
+	// racing the goroutine start.
 	deadline := time.Now().Add(10 * time.Second)
-	for time.Now().Before(deadline) {
+	for i := 1; time.Now().Before(deadline); i++ {
+		future := time.Now().Add(time.Duration(i) * time.Second)
+		_ = os.Chtimes(certPath, future, future)
 		if string(servedLeaf(t, r)) == string(second) {
 			return
 		}

services/runtime-api/internal/runtimeapi/adapter_certificate.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"net/http"
 	"os"
@@ -196,7 +197,10 @@ func (s *AccessService) issueSessionCertificateDER(
 			},
 		},
 		"spec": map[string]any{
-			"request":  base64.StdEncoding.EncodeToString(csrDER),
+			// cert-manager's CertificateRequest.spec.request must be a PEM-encoded
+			// CSR; sending raw DER makes the admission webhook reject it with
+			// "error decoding certificate request PEM block".
+			"request":  base64.StdEncoding.EncodeToString(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrDER})),
 			"duration": duration.Round(time.Second).String(),
 			"usages":   []any{"digital signature", "client auth"},
 			"issuerRef": map[string]any{

services/runtime-api/internal/runtimeapi/adapter_certificate_test.go

@@ -1,12 +1,22 @@
 package runtimeapi
 
 import (
+	"context"
+	"encoding/base64"
+	"encoding/pem"
 	"strings"
 	"testing"
+	"time"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
+	clienttesting "k8s.io/client-go/testing"
 
 	"mcp-runtime/pkg/certauth"
+	"mcp-runtime/pkg/identity"
+	"mcp-runtime/pkg/k8sclient"
 )
 
 func TestValidateAdapterCSRUsesCertauth(t *testing.T) {
@@ -24,6 +34,53 @@ func TestValidateAdapterCSRUsesCertauth(t *testing.T) {
 	}
 }
 
+func TestIssueSessionCertificateSubmitsPEMRequest(t *testing.T) {
+	// cert-manager's admission webhook rejects a CertificateRequest whose
+	// spec.request is not a PEM-encoded CSR, so the submitted request must be
+	// PEM (base64 of the PEM block), not raw DER.
+	t.Setenv("MCP_MTLS_CLUSTER_ISSUER", "mcp-runtime-ca")
+
+	scheme := runtime.NewScheme()
+	dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme,
+		map[schema.GroupVersionResource]string{certificateRequestGVR: "CertificateRequestList"})
+	var capturedRequest string
+	dyn.PrependReactor("create", "certificaterequests", func(action clienttesting.Action) (bool, runtime.Object, error) {
+		obj := action.(clienttesting.CreateAction).GetObject().(*unstructured.Unstructured)
+		capturedRequest, _, _ = unstructured.NestedString(obj.Object, "spec", "request")
+		return false, nil, nil
+	})
+
+	svc := &AccessService{k8sClients: &k8sclient.Clients{Dynamic: dyn}}
+
+	const trust, ns, sess = "cluster.local", "mcp-servers", "sess-1"
+	_, csrPEM, _, err := certauth.BuildSessionCSR(trust, ns, sess)
+	if err != nil {
+		t.Fatalf("BuildSessionCSR: %v", err)
+	}
+	csrDER, err := certauth.ValidateCSRPEM(string(csrPEM), identity.SessionSPIFFEID(trust, ns, sess))
+	if err != nil {
+		t.Fatalf("ValidateCSRPEM: %v", err)
+	}
+
+	// The wait step errors out (no accessMgr / no signer), but the create — and
+	// thus our capture — happens first, which is all this test asserts.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	_, _, _ = svc.issueSessionCertificateDER(ctx, ns, sess, csrDER, time.Hour)
+
+	if capturedRequest == "" {
+		t.Fatal("no CertificateRequest was created")
+	}
+	decoded, err := base64.StdEncoding.DecodeString(capturedRequest)
+	if err != nil {
+		t.Fatalf("spec.request is not valid base64: %v", err)
+	}
+	block, _ := pem.Decode(decoded)
+	if block == nil || block.Type != "CERTIFICATE REQUEST" {
+		t.Fatalf("spec.request must be a PEM CERTIFICATE REQUEST, got: %q", string(decoded))
+	}
+}
+
 func TestAdapterCertificateRequestFailure(t *testing.T) {
 	t.Parallel()
 	request := &unstructured.Unstructured{Object: map[string]any{