From 80b4e5ed4f7ff1c5a2775cc88d61ac8c9079752a Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 25 Jun 2026 00:59:36 +0530 Subject: [PATCH 01/21] test(e2e): add mtls datapath scenario for Kind Exercise test-mode workload PKI, Traefik websecure mTLS termination, adapter enroll, and session-bound client certificates without relying on governance headers. Co-authored-by: Cursor --- test/e2e/kind.sh | 16 ++- test/e2e/scenarios/mtls.sh | 246 ++++++++++++++++++++++++++++++++ test/e2e/scenarios_test.sh | 4 +- test/e2e/select_pr_scenarios.sh | 5 + 4 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 test/e2e/scenarios/mtls.sh diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 17d61ce2..e62b425a 100644 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -10,7 +10,7 @@ set -euo pipefail # # Set E2E_SCENARIOS to a comma-separated subset for local debugging. # Supported values: all, smoke-auth, governance, trust, oauth, observability, -# multitenancy, api-platform, ui-auth, adapter-proxy, cli-platform. +# multitenancy, api-platform, ui-auth, adapter-proxy, cli-platform, mtls. # observability requires the full traffic suite: smoke-auth, governance, trust, oauth. # # Set E2E_DEEP_REQUEST_FLOWS=1 for pre-release runs that should exercise @@ -366,11 +366,11 @@ validate_scenarios() { local scenario for scenario in "${E2E_SCENARIO_LIST[@]}"; do case "${scenario}" in - all|smoke-auth|governance|trust|oauth|observability|multitenancy|api-platform|ui-auth|adapter-proxy|cli-platform) + all|smoke-auth|governance|trust|oauth|observability|multitenancy|api-platform|ui-auth|adapter-proxy|cli-platform|mtls) ;; *) echo "unsupported E2E scenario: ${scenario}" >&2 - echo "supported values: all, smoke-auth, governance, trust, oauth, observability, multitenancy, api-platform, ui-auth, adapter-proxy, cli-platform" >&2 + echo "supported values: all, smoke-auth, governance, trust, oauth, observability, multitenancy, api-platform, ui-auth, adapter-proxy, cli-platform, mtls" >&2 exit 1 ;; esac @@ -388,7 +388,7 @@ validate_scenarios() { if deep_request_flows_enabled; then local required - for required in smoke-auth governance trust oauth observability multitenancy api-platform ui-auth adapter-proxy cli-platform; do + for required in smoke-auth governance trust oauth observability multitenancy api-platform ui-auth adapter-proxy cli-platform mtls; do if ! scenario_selected "${required}"; then echo "E2E_DEEP_REQUEST_FLOWS=1 requires all E2E scenarios" >&2 echo "set E2E_SCENARIOS=all or include every supported scenario" >&2 @@ -960,6 +960,9 @@ ensure_gateway_port_forward() { wait_port "${SENTINEL_PORT}" } +# shellcheck source=scenarios/mtls.sh +source "${PROJECT_ROOT}/test/e2e/scenarios/mtls.sh" + refresh_mcp_proxy_urls() { MCP_INGRESS_PATH="/${SERVER_NAME}/mcp" MCP_DIRECT_URL="http://127.0.0.1:${TRAEFIK_PORT}${MCP_INGRESS_PATH}" @@ -6057,6 +6060,11 @@ fi fi +if scenario_selected "mtls"; then + run_e2e_mtls_scenario + cleanup_mcp_server_and_wait "${MTLS_SERVER_NAME}" mcp-servers 120s +fi + echo "[cli] checking sentinel restart command" # The full E2E stack packs single-node Kind tightly, so avoid requiring surge CPU for this restart smoke. refresh_kind_kubeconfig diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh new file mode 100644 index 00000000..96b0a19b --- /dev/null +++ b/test/e2e/scenarios/mtls.sh @@ -0,0 +1,246 @@ +# mTLS datapath checks for Kind E2E. Sourced from kind.sh when the mtls scenario +# is selected. Requires test-mode setup (managed mcp-runtime-ca issuer). + +MTLS_SERVER_NAME="${MTLS_SERVER_NAME:-mtls-mcp-server}" +MTLS_TRUST_DOMAIN="${MTLS_TRUST_DOMAIN:-cluster.local}" +TRAEFIK_TLS_PORT="${TRAEFIK_TLS_PORT:-18443}" +MTLS_AGENT_ID="${MTLS_AGENT_ID:-e2e-mtls-agent}" + +run_e2e_mtls_scenario() { + log_line mtls "verifying test-mode workload PKI" + if ! kubectl get clusterissuer mcp-runtime-ca >/dev/null 2>&1; then + echo "expected ClusterIssuer mcp-runtime-ca from test-mode setup" >&2 + exit 1 + fi + kubectl wait --for=condition=Ready clusterissuer/mcp-runtime-ca --timeout=120s >/dev/null + + operator_issuer="$(kubectl get deploy/mcp-runtime-operator-controller-manager -n mcp-runtime \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MCP_MTLS_CLUSTER_ISSUER")].value}')" + if [[ "${operator_issuer}" != "mcp-runtime-ca" ]]; then + echo "operator MCP_MTLS_CLUSTER_ISSUER = ${operator_issuer:-}, want mcp-runtime-ca" >&2 + exit 1 + fi + + runtime_issuer="$(kubectl get deploy/mcp-runtime-api -n mcp-sentinel \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="runtime-api")].env[?(@.name=="MCP_MTLS_CLUSTER_ISSUER")].value}' 2>/dev/null || true)" + if [[ -n "${runtime_issuer}" && "${runtime_issuer}" != "mcp-runtime-ca" ]]; then + echo "runtime-api MCP_MTLS_CLUSTER_ISSUER = ${runtime_issuer}, want mcp-runtime-ca" >&2 + exit 1 + fi + + log_line mtls "deploying auth.mode mtls MCPServer ${MTLS_SERVER_NAME}" + MTLS_METADATA="${WORKDIR}/mtls-metadata.yaml" + MTLS_MANIFEST="${WORKDIR}/mtls-manifest.yaml" + MTLS_SECRET_NAME="${MTLS_SERVER_NAME}-analytics-creds" + MTLS_IMAGE="registry.registry.svc.cluster.local:5000/${MTLS_SERVER_NAME}:${E2E_WORKLOAD_TAG}" + kubectl create secret generic "${MTLS_SECRET_NAME}" \ + -n mcp-servers \ + --from-literal=api-key="${INGEST_API_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + cat > "${MTLS_METADATA}" </dev/null 2>&1; then + run_logged_stage "build mtls MCP server image" ./bin/mcp-runtime server build image "${MTLS_SERVER_NAME}" \ + --metadata-file "${MTLS_METADATA}" \ + --dockerfile "${GO_EXAMPLE_SOURCE_DIR}/Dockerfile" \ + --registry registry.registry.svc.cluster.local:5000 \ + --tag "${E2E_WORKLOAD_TAG}" \ + --context "${GO_EXAMPLE_SOURCE_DIR}" + fi + prune_kind_image "${MTLS_IMAGE}" + load_image_into_kind "${MTLS_IMAGE}" + + run_logged_stage "deploy mtls MCP server manifests" bash -lc \ + "MCP_RUNTIME_CONFIG_DIR=\"${E2E_PIPELINE_CONFIG_DIR}\" ./bin/mcp-runtime server generate --metadata-file \"${MTLS_METADATA}\" --output \"${WORKDIR}/mtls-manifests\" && ./bin/mcp-runtime server --use-kube apply --file \"${WORKDIR}/mtls-manifests/${MTLS_SERVER_NAME}.yaml\"" + wait_for_named_server_ready "${MTLS_SERVER_NAME}" "mcp-servers" 240 + + log_line mtls "waiting for gateway and trust-bundle certificates" + deadline=$((SECONDS + 240)) + while (( SECONDS < deadline )); do + if kubectl get secret "${MTLS_SERVER_NAME}-gateway-mtls" -n mcp-servers >/dev/null 2>&1 \ + && kubectl get secret "${MTLS_SERVER_NAME}-mtls-ca" -n mcp-servers >/dev/null 2>&1; then + break + fi + sleep 2 + done + if ! kubectl get secret "${MTLS_SERVER_NAME}-gateway-mtls" -n mcp-servers >/dev/null 2>&1; then + echo "timed out waiting for gateway TLS secret ${MTLS_SERVER_NAME}-gateway-mtls" >&2 + exit 1 + fi + + ensure_traefik_port_forward + if [[ -z "${TRAEFIK_TLS_PORT_FORWARD_PID:-}" ]]; then + echo "[port-forward] exposing traefik websecure on localhost:${TRAEFIK_TLS_PORT}" + port_forward_bg traefik traefik "${TRAEFIK_TLS_PORT}" 8443 "${WORKDIR}/traefik-tls-port-forward.log" + TRAEFIK_TLS_PORT_FORWARD_PID="${LAST_MANAGED_PID}" + fi + wait_port "${TRAEFIK_TLS_PORT}" + + MTLS_INGRESS_PATH="/${MTLS_SERVER_NAME}/mcp" + MTLS_URL="https://127.0.0.1:${TRAEFIK_TLS_PORT}${MTLS_INGRESS_PATH}" + MTLS_CA_FILE="${WORKDIR}/mtls-ca.crt" + kubectl get secret "${MTLS_SERVER_NAME}-mtls-ca" -n mcp-servers -o jsonpath='{.data.ca\.crt}' | decode_base64 >"${MTLS_CA_FILE}" + + log_line mtls "rejecting initialize without a client certificate" + if curl -fsS --cacert "${MTLS_CA_FILE}" \ + -H "Host: ${SERVER_HOST}" \ + -H "content-type: application/json" \ + -H "accept: application/json, text/event-stream" \ + -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ + --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"'"${MCP_PROTOCOL_VERSION}"'","capabilities":{},"clientInfo":{"name":"e2e","version":"1"}}}' \ + "${MTLS_URL}" >/dev/null 2>&1; then + echo "expected initialize without client certificate to fail" >&2 + exit 1 + fi + + ensure_gateway_port_forward + ensure_api_port_forward + if [[ -z "${ADAPTER_PLATFORM_TOKEN:-}" ]]; then + ADAPTER_PLATFORM_TOKEN="$(PLATFORM_ADMIN_EMAIL="${PLATFORM_ADMIN_EMAIL}" PLATFORM_ADMIN_PASSWORD="${PLATFORM_ADMIN_PASSWORD}" python3 -c ' +import json, os +print(json.dumps({"email": os.environ["PLATFORM_ADMIN_EMAIL"], "password": os.environ["PLATFORM_ADMIN_PASSWORD"]})) +' | curl -fsS -X POST \ + -H "content-type: application/json" \ + --data-binary @- \ + "http://127.0.0.1:${API_SERVICE_PORT}/api/v1/auth/login" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])')" + fi + + log_line mtls "applying grant and adapter session for mtls datapath" + cat >"${WORKDIR}/mtls-grant.yaml" <&2 + exit 1 + fi + done + + log_line mtls "accepting initialize with session-bound client certificate" + init_body="$(curl -fsS \ + --cert "${MTLS_CLIENT_CERT}" \ + --key "${MTLS_CLIENT_KEY}" \ + --cacert "${MTLS_CLIENT_CA}" \ + -H "Host: ${SERVER_HOST}" \ + -H "content-type: application/json" \ + -H "accept: application/json, text/event-stream" \ + -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ + --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"'"${MCP_PROTOCOL_VERSION}"'","capabilities":{},"clientInfo":{"name":"e2e-mtls","version":"1"}}}' \ + "${MTLS_URL}")" + echo "${init_body}" | python3 -c ' +import json, sys +doc = json.load(sys.stdin) +if doc.get("error"): + raise SystemExit(f"initialize failed: {doc}") +if "result" not in doc: + raise SystemExit(f"missing initialize result: {doc}") +print("mtls initialize ok") +' + + log_line mtls "ignoring spoofed governance headers on mtls path" + spoof_body="$(curl -fsS \ + --cert "${MTLS_CLIENT_CERT}" \ + --key "${MTLS_CLIENT_KEY}" \ + --cacert "${MTLS_CLIENT_CA}" \ + -H "Host: ${SERVER_HOST}" \ + -H "content-type: application/json" \ + -H "accept: application/json, text/event-stream" \ + -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ + -H "X-MCP-Human-ID: spoofed-human" \ + -H "X-MCP-Agent-ID: spoofed-agent" \ + -H "X-MCP-Agent-Session: definitely-not-${MTLS_SESSION_NAME}" \ + --data '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"aaa-ping","arguments":{}}}' \ + "${MTLS_URL}" 2>/dev/null || true)" + if [[ -n "${spoof_body}" ]]; then + echo "${spoof_body}" | python3 -c ' +import json, sys +doc = json.load(sys.stdin) +err = doc.get("error") or {} +msg = str(err.get("message", "")).lower() +if doc.get("result") and "pong" in json.dumps(doc.get("result")).lower(): + raise SystemExit("spoofed governance headers must not bypass mtls auth") +if err and any(token in msg for token in ("session", "denied", "unauthorized", "forbidden", "grant", "trust")): + raise SystemExit(0) +if err: + raise SystemExit(0) +raise SystemExit(f"unexpected tools/call response with spoofed headers: {doc}") +' + fi +} diff --git a/test/e2e/scenarios_test.sh b/test/e2e/scenarios_test.sh index f9f02a72..bee103bb 100644 --- a/test/e2e/scenarios_test.sh +++ b/test/e2e/scenarios_test.sh @@ -57,6 +57,7 @@ run_valid "api-platform" "api-platform" "api-platform" run_valid "ui-auth" "ui-auth" "ui-auth" run_valid "adapter-proxy" "adapter-proxy" "adapter-proxy" run_valid "cli-platform" "cli-platform" "cli-platform" +run_valid "mtls" "mtls" "mtls" run_valid "observability-with-deps" "smoke-auth,governance,trust,oauth,observability" "smoke-auth,governance,trust,oauth,observability" run_valid "whitespace-trimmed" " smoke-auth , governance " "smoke-auth,governance" run_valid "duplicates-deduped" "smoke-auth,smoke-auth" "smoke-auth" @@ -140,7 +141,8 @@ selector_expect "api" "smoke-auth,api-platform" "services/platform-api/auth/logi selector_expect "runtime-tools-api" "smoke-auth,api-platform,cli-platform" "services/runtime-api/internal/runtimeapi/tools.go" selector_expect "catalog-cli" "smoke-auth,cli-platform" "internal/cli/catalog/catalog.go" selector_expect "adapter" "smoke-auth,adapter-proxy,governance" "internal/cli/adapter/proxy.go" -selector_expect "gateway" "smoke-auth,governance,trust,oauth,adapter-proxy" "services/mcp-gateway/main.go" +selector_expect "mtls-operator" "smoke-auth,mtls" "internal/operator/mtls.go" +selector_expect "gateway" "smoke-auth,governance,trust,oauth,adapter-proxy,mtls" "services/mcp-gateway/main.go" selector_expect "observability" "smoke-auth,governance,trust,oauth,observability" "services/ingest/main.go" selector_expect "broad" "all" "api/v1alpha1/mcpserver_types.go" diff --git a/test/e2e/select_pr_scenarios.sh b/test/e2e/select_pr_scenarios.sh index 0a36e1ed..dfac3c5f 100755 --- a/test/e2e/select_pr_scenarios.sh +++ b/test/e2e/select_pr_scenarios.sh @@ -50,6 +50,10 @@ classify_path() { mark_all return ;; + internal/operator/mtls*|internal/cli/certmanager/*|traefik-plugins/spiffe-identity/*|config/cert-manager/*|pkg/identity/*|pkg/certauth/*|test/e2e/scenarios/mtls.sh) + add_scenario "mtls" + return + ;; api/*|cmd/operator/*|internal/operator/*|config/*|k8s/*|pkg/controlplane/*|pkg/k8sclient/*|pkg/kubeworkload/*|pkg/manifest/*|pkg/metadata/*) mark_all return @@ -108,6 +112,7 @@ classify_path() { add_scenario "trust" add_scenario "oauth" add_scenario "adapter-proxy" + add_scenario "mtls" return ;; services/ingest/*|services/processor/*|pkg/clickhouse/*|pkg/events/*|pkg/sentinel/*|pkg/serviceutil/*) From 12f35676b0b7cd3df33f78bad681cd0cbc7bf81e Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Tue, 30 Jun 2026 15:36:22 +0530 Subject: [PATCH 02/21] fix(operator): track IngressRoute for mTLS readiness; harden mtls e2e The terminate-and-re-encrypt mTLS model serves traffic through a Traefik IngressRoute and deletes the legacy passthrough IngressRouteTCP, but checkIngressReady still looked for the deleted IngressRouteTCP. That left every mTLS MCPServer stuck at PartiallyReady (ingressReady never true), which made the mTLS datapath e2e scenario time out waiting for phase Ready. Point the readiness check at the IngressRoute and cover both the ready and not-ready cases. Also address e2e review feedback in scenarios/mtls.sh: - drop curl -f on the spoofed-headers check so the gateway's 4xx/5xx body is captured and validated instead of silently skipped; - fail fast with a clear message if either the gateway-mtls or mtls-ca secret is missing after the wait loop. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/operator/controller_test.go | 28 ++++++++++++++++++++++++++++ internal/operator/status.go | 5 ++++- test/e2e/scenarios/mtls.sh | 10 +++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/internal/operator/controller_test.go b/internal/operator/controller_test.go index 7558df80..503e5085 100644 --- a/internal/operator/controller_test.go +++ b/internal/operator/controller_test.go @@ -1089,6 +1089,34 @@ func TestCheckIngressReady(t *testing.T) { assertEqual(t, "ready", ready, false) }) + t.Run("mtls server is ready when its IngressRoute exists", func(t *testing.T) { + mtlsScheme := traefikScheme(t) + server := mtlsServer() + route := crFixture(ingressRouteGVK, server.Name, server.Namespace) + client := fake.NewClientBuilder().WithScheme(mtlsScheme).WithObjects(server, route).Build() + r := MCPServerReconciler{Client: client, Scheme: mtlsScheme} + ready, err := r.checkIngressReady(context.Background(), server) + if err != nil { + t.Fatalf("failed to check ingress readiness: %v", err) + } + assertEqual(t, "ready", ready, true) + }) + + t.Run("mtls server is not ready without an IngressRoute", func(t *testing.T) { + mtlsScheme := traefikScheme(t) + server := mtlsServer() + // Only the legacy passthrough IngressRouteTCP exists; the terminate-and- + // re-encrypt model no longer uses it, so the server must not read ready. + legacy := crFixture(ingressRouteTCPGVK, server.Name, server.Namespace) + client := fake.NewClientBuilder().WithScheme(mtlsScheme).WithObjects(server, legacy).Build() + r := MCPServerReconciler{Client: client, Scheme: mtlsScheme} + ready, err := r.checkIngressReady(context.Background(), server) + if err != nil { + t.Fatalf("failed to check ingress readiness: %v", err) + } + assertEqual(t, "ready", ready, false) + }) + t.Run("returns true when ingress has load balancer status", func(t *testing.T) { mcpServer := &mcpv1alpha1.MCPServer{ ObjectMeta: metav1.ObjectMeta{Name: "test-server", Namespace: "default"}, diff --git a/internal/operator/status.go b/internal/operator/status.go index ea6f6061..0241bf6d 100644 --- a/internal/operator/status.go +++ b/internal/operator/status.go @@ -70,8 +70,11 @@ func (r *MCPServerReconciler) checkServiceReady(ctx context.Context, mcpServer * func (r *MCPServerReconciler) checkIngressReady(ctx context.Context, mcpServer *mcpv1alpha1.MCPServer) (bool, error) { if serverUsesMTLS(mcpServer) { + // The terminate-and-re-encrypt mTLS model serves traffic through a + // path-based Traefik IngressRoute (the legacy passthrough IngressRouteTCP + // is deleted during reconcile), so readiness must track the IngressRoute. route := &unstructured.Unstructured{} - route.SetGroupVersionKind(ingressRouteTCPGVK) + route.SetGroupVersionKind(ingressRouteGVK) if err := r.Get(ctx, types.NamespacedName{Name: mcpServer.Name, Namespace: mcpServer.Namespace}, route); err != nil { if errors.IsNotFound(err) { return false, nil diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 96b0a19b..70c6a32d 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -106,8 +106,9 @@ EOF fi sleep 2 done - if ! kubectl get secret "${MTLS_SERVER_NAME}-gateway-mtls" -n mcp-servers >/dev/null 2>&1; then - echo "timed out waiting for gateway TLS secret ${MTLS_SERVER_NAME}-gateway-mtls" >&2 + if ! kubectl get secret "${MTLS_SERVER_NAME}-gateway-mtls" -n mcp-servers >/dev/null 2>&1 \ + || ! kubectl get secret "${MTLS_SERVER_NAME}-mtls-ca" -n mcp-servers >/dev/null 2>&1; then + echo "timed out waiting for gateway TLS secrets (${MTLS_SERVER_NAME}-gateway-mtls and/or ${MTLS_SERVER_NAME}-mtls-ca)" >&2 exit 1 fi @@ -215,7 +216,10 @@ print("mtls initialize ok") ' log_line mtls "ignoring spoofed governance headers on mtls path" - spoof_body="$(curl -fsS \ + # Intentionally omit curl -f: the gateway is expected to reject the spoofed + # headers with a 4xx/5xx, and we still need to capture the response body so the + # Python check below can validate the error instead of being skipped. + spoof_body="$(curl -sS \ --cert "${MTLS_CLIENT_CERT}" \ --key "${MTLS_CLIENT_KEY}" \ --cacert "${MTLS_CLIENT_CA}" \ From 46169c0f9a8e4e08050c68fb9543fc4cda6530d6 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Tue, 30 Jun 2026 21:14:58 +0530 Subject: [PATCH 03/21] fix(test): drop unused web port-forward from mtls e2e scenario The mtls datapath drives traffic through Traefik's websecure entrypoint (TRAEFIK_TLS_PORT -> 8443); the plaintext web port-forward (18080) is never used by the scenario. Calling ensure_traefik_port_forward made the scenario time out waiting on localhost:18080 deep into the run for a port it doesn't need. Keep only the websecure port-forward. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/scenarios/mtls.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 70c6a32d..6bd3d07b 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -112,7 +112,8 @@ EOF exit 1 fi - ensure_traefik_port_forward + # The mtls datapath only needs Traefik's websecure (TLS) entrypoint; it never + # uses the plaintext web port-forward, so don't gate the scenario on it. if [[ -z "${TRAEFIK_TLS_PORT_FORWARD_PID:-}" ]]; then echo "[port-forward] exposing traefik websecure on localhost:${TRAEFIK_TLS_PORT}" port_forward_bg traefik traefik "${TRAEFIK_TLS_PORT}" 8443 "${WORKDIR}/traefik-tls-port-forward.log" From 52a38740773096728519bb50eae873969752864c Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Tue, 30 Jun 2026 22:15:02 +0530 Subject: [PATCH 04/21] fix(test): point mtls policy waits at the mtls server's policy ConfigMap wait_for_policy_text hardcoded the default ${SERVER_NAME}-gateway-policy ConfigMap, so the mtls scenario's waits for its grant and session never observed mtls-mcp-server-gateway-policy and timed out. Add an optional server argument (default SERVER_NAME) and pass MTLS_SERVER_NAME from the mtls scenario. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/kind.sh | 7 ++++--- test/e2e/scenarios/mtls.sh | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index e62b425a..1f7b4979 100644 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -1729,17 +1729,18 @@ PY wait_for_policy_text() { local text="$1" - local tries="${2:-40}" + local server="${2:-${SERVER_NAME}}" + local tries="${3:-40}" local i for i in $(seq 1 "${tries}"); do local current - current="$(kubectl get configmap "${SERVER_NAME}-gateway-policy" -n mcp-servers -o "jsonpath={.data.policy\.json}" 2>/dev/null || true)" + current="$(kubectl get configmap "${server}-gateway-policy" -n mcp-servers -o "jsonpath={.data.policy\.json}" 2>/dev/null || true)" if [[ "${current}" == *"${text}"* ]]; then return 0 fi sleep 2 done - echo "timed out waiting for policy text: ${text}" >&2 + echo "timed out waiting for policy text in ${server}-gateway-policy: ${text}" >&2 return 1 } diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 6bd3d07b..a06fff36 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -170,7 +170,7 @@ spec: decision: allow EOF (cd "${WORKDIR}" && "${PROJECT_ROOT}/bin/mcp-runtime" access --use-kube grant apply --file mtls-grant.yaml) - wait_for_policy_text "\"name\": \"${MTLS_SERVER_NAME}-grant\"" + wait_for_policy_text "\"name\": \"${MTLS_SERVER_NAME}-grant\"" "${MTLS_SERVER_NAME}" MTLS_CERT_DIR="${WORKDIR}/mtls-certs" mkdir -p "${MTLS_CERT_DIR}" @@ -183,7 +183,7 @@ EOF --trust-domain "${MTLS_TRUST_DOMAIN}" \ --output-dir "${MTLS_CERT_DIR}")" MTLS_SESSION_NAME="$(printf '%s\n' "${MTLS_ENROLL_OUT}" | python3 -c 'import re,sys; m=re.search(r"/session/([^\\s]+)", sys.stdin.read()); print(m.group(1) if m else ""); sys.exit(0 if m else 1)')" - wait_for_policy_text "\"name\": \"${MTLS_SESSION_NAME}\"" + wait_for_policy_text "\"name\": \"${MTLS_SESSION_NAME}\"" "${MTLS_SERVER_NAME}" MTLS_CLIENT_CERT="${MTLS_CERT_DIR}/client.crt" MTLS_CLIENT_KEY="${MTLS_CERT_DIR}/client.key" From 19436ec1c47b1e9f18ff5d578c3b405e50d8d118 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Tue, 30 Jun 2026 23:29:03 +0530 Subject: [PATCH 05/21] fix(cli): propagate MCP_MTLS_CLUSTER_ISSUER into sentinel ConfigMap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mcp-sentinel-config ConfigMap (rendered by renderAnalyticsConfigManifest) never received MCP_MTLS_CLUSTER_ISSUER, so runtime-api — which consumes it via envFrom and uses it to sign adapter/session CSRs — saw an empty issuer and returned "503 workload certificate issuer is not configured" on adapter enroll. The replacement only lived in renderAnalyticsManifest, which handles the other manifests that don't carry the key. Inject the issuer in the config renderer and drop the dead replacement. Tighten the mtls e2e check to assert the issuer on the ConfigMap (the real source via envFrom) instead of an inline env var that never existed, so a propagation regression fails fast with a clear message. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/cli/setup/platform/analytics.go | 4 +-- internal/cli/setup/platform/helpers_test.go | 33 +++++++++++++++++++++ test/e2e/scenarios/mtls.sh | 11 ++++--- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/internal/cli/setup/platform/analytics.go b/internal/cli/setup/platform/analytics.go index e0b6d90c..46b0afd1 100644 --- a/internal/cli/setup/platform/analytics.go +++ b/internal/cli/setup/platform/analytics.go @@ -677,9 +677,6 @@ func renderAnalyticsManifest(content string, images AnalyticsImageSet, imagePull if mode, ok := setupplan.NormalizePlatformMode(platformMode); ok && mode != "" { replacements[`PLATFORM_MODE: "tenant"`] = fmt.Sprintf(`PLATFORM_MODE: "%s"`, mode) } - if issuer := strings.TrimSpace(os.Getenv("MCP_MTLS_CLUSTER_ISSUER")); issuer != "" { - replacements[`MCP_MTLS_CLUSTER_ISSUER: ""`] = fmt.Sprintf(`MCP_MTLS_CLUSTER_ISSUER: %q`, issuer) - } if strings.TrimSpace(images.Ingest) != "" { replacements["image: mcp-sentinel-ingest:latest"] = "image: " + images.Ingest } @@ -800,6 +797,7 @@ func renderAnalyticsConfigManifestWithReaders(content, platformMode string, imag "PLATFORM_REGISTRY_URL", "PLATFORM_TRAEFIK_NAMESPACE", "PLATFORM_TEAM_TRAEFIK_WATCH", + "MCP_MTLS_CLUSTER_ISSUER", } { if envValue := setupAnalyticsConfigEnvValue(key); envValue != "" { manifest.Data[key] = envValue diff --git a/internal/cli/setup/platform/helpers_test.go b/internal/cli/setup/platform/helpers_test.go index e40da7b1..86cba240 100644 --- a/internal/cli/setup/platform/helpers_test.go +++ b/internal/cli/setup/platform/helpers_test.go @@ -1485,6 +1485,39 @@ func TestRenderAnalyticsSecretManifestDisablesExistingDevLoginsOutsideTestMode(t } } +func TestRenderAnalyticsConfigManifestInjectsMTLSClusterIssuer(t *testing.T) { + t.Setenv("MCP_MTLS_CLUSTER_ISSUER", "mcp-runtime-ca") + mock := &core.MockExecutor{ + CommandFunc: func(spec core.ExecSpec) *core.MockCommand { + return &core.MockCommand{Args: spec.Args} + }, + } + kubectl := core.NewTestKubectlClient(mock) + + rendered, err := renderAnalyticsConfigManifest(kubectl, `apiVersion: v1 +kind: ConfigMap +metadata: + name: mcp-sentinel-config + namespace: mcp-sentinel +data: + MCP_MTLS_CLUSTER_ISSUER: "" + PLATFORM_MODE: "tenant" +`, setupplan.PlatformModeTenant, AnalyticsImageSet{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var payload struct { + Data map[string]string `yaml:"data"` + } + if err := yaml.Unmarshal([]byte(rendered), &payload); err != nil { + t.Fatalf("unmarshal rendered config: %v", err) + } + if got := payload.Data["MCP_MTLS_CLUSTER_ISSUER"]; got != "mcp-runtime-ca" { + t.Fatalf("expected MCP_MTLS_CLUSTER_ISSUER to be injected into the sentinel ConfigMap, got %q", got) + } +} + func TestRenderAnalyticsConfigManifestPreservesExistingPublicOAuthConfig(t *testing.T) { mock := &core.MockExecutor{ CommandFunc: func(spec core.ExecSpec) *core.MockCommand { diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index a06fff36..132e4db9 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -21,10 +21,13 @@ run_e2e_mtls_scenario() { exit 1 fi - runtime_issuer="$(kubectl get deploy/mcp-runtime-api -n mcp-sentinel \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="runtime-api")].env[?(@.name=="MCP_MTLS_CLUSTER_ISSUER")].value}' 2>/dev/null || true)" - if [[ -n "${runtime_issuer}" && "${runtime_issuer}" != "mcp-runtime-ca" ]]; then - echo "runtime-api MCP_MTLS_CLUSTER_ISSUER = ${runtime_issuer}, want mcp-runtime-ca" >&2 + # runtime-api issues adapter/session certificates and reads the issuer from the + # mcp-sentinel-config ConfigMap (via envFrom), so assert it there rather than on + # an inline env var. An empty value makes `adapter enroll` fail with a 503. + runtime_issuer="$(kubectl get configmap mcp-sentinel-config -n mcp-sentinel \ + -o jsonpath='{.data.MCP_MTLS_CLUSTER_ISSUER}' 2>/dev/null || true)" + if [[ "${runtime_issuer}" != "mcp-runtime-ca" ]]; then + echo "mcp-sentinel-config MCP_MTLS_CLUSTER_ISSUER = ${runtime_issuer:-}, want mcp-runtime-ca" >&2 exit 1 fi From 18a4cd975d95af4091d07fb33fff78140e1686bc Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 1 Jul 2026 11:54:32 +0530 Subject: [PATCH 06/21] fix(services-api): send PEM-encoded CSR to cert-manager for adapter certs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HandleAdapterCertificate base64-encoded the raw CSR DER into the CertificateRequest spec.request, but cert-manager's admission webhook requires a PEM-encoded CSR and rejected it with "error decoding certificate request PEM block" — surfacing as "503 issue adapter certificate" on `adapter enroll` and breaking the mTLS datapath. PEM-encode the CSR before base64. Verified against a live mcp-runtime-ca issuer (request now issues) and covered by a regression test asserting spec.request decodes to a PEM CERTIFICATE REQUEST. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../runtimeapi/adapter_certificate.go | 6 +- .../runtimeapi/adapter_certificate_test.go | 57 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/services/runtime-api/internal/runtimeapi/adapter_certificate.go b/services/runtime-api/internal/runtimeapi/adapter_certificate.go index e28b775d..013e7fae 100644 --- a/services/runtime-api/internal/runtimeapi/adapter_certificate.go +++ b/services/runtime-api/internal/runtimeapi/adapter_certificate.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "net/http" "os" @@ -196,7 +197,10 @@ func (s *AccessService) issueSessionCertificateDER( }, }, "spec": map[string]any{ - "request": base64.StdEncoding.EncodeToString(csrDER), + // cert-manager's CertificateRequest.spec.request must be a PEM-encoded + // CSR; sending raw DER makes the admission webhook reject it with + // "error decoding certificate request PEM block". + "request": base64.StdEncoding.EncodeToString(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrDER})), "duration": duration.Round(time.Second).String(), "usages": []any{"digital signature", "client auth"}, "issuerRef": map[string]any{ diff --git a/services/runtime-api/internal/runtimeapi/adapter_certificate_test.go b/services/runtime-api/internal/runtimeapi/adapter_certificate_test.go index 31d60e58..7e90737c 100644 --- a/services/runtime-api/internal/runtimeapi/adapter_certificate_test.go +++ b/services/runtime-api/internal/runtimeapi/adapter_certificate_test.go @@ -1,12 +1,22 @@ package runtimeapi import ( + "context" + "encoding/base64" + "encoding/pem" "strings" "testing" + "time" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + clienttesting "k8s.io/client-go/testing" "mcp-runtime/pkg/certauth" + "mcp-runtime/pkg/identity" + "mcp-runtime/pkg/k8sclient" ) func TestValidateAdapterCSRUsesCertauth(t *testing.T) { @@ -24,6 +34,53 @@ func TestValidateAdapterCSRUsesCertauth(t *testing.T) { } } +func TestIssueSessionCertificateSubmitsPEMRequest(t *testing.T) { + // cert-manager's admission webhook rejects a CertificateRequest whose + // spec.request is not a PEM-encoded CSR, so the submitted request must be + // PEM (base64 of the PEM block), not raw DER. + t.Setenv("MCP_MTLS_CLUSTER_ISSUER", "mcp-runtime-ca") + + scheme := runtime.NewScheme() + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, + map[schema.GroupVersionResource]string{certificateRequestGVR: "CertificateRequestList"}) + var capturedRequest string + dyn.PrependReactor("create", "certificaterequests", func(action clienttesting.Action) (bool, runtime.Object, error) { + obj := action.(clienttesting.CreateAction).GetObject().(*unstructured.Unstructured) + capturedRequest, _, _ = unstructured.NestedString(obj.Object, "spec", "request") + return false, nil, nil + }) + + svc := &AccessService{k8sClients: &k8sclient.Clients{Dynamic: dyn}} + + const trust, ns, sess = "cluster.local", "mcp-servers", "sess-1" + _, csrPEM, _, err := certauth.BuildSessionCSR(trust, ns, sess) + if err != nil { + t.Fatalf("BuildSessionCSR: %v", err) + } + csrDER, err := certauth.ValidateCSRPEM(string(csrPEM), identity.SessionSPIFFEID(trust, ns, sess)) + if err != nil { + t.Fatalf("ValidateCSRPEM: %v", err) + } + + // The wait step errors out (no accessMgr / no signer), but the create — and + // thus our capture — happens first, which is all this test asserts. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, _, _ = svc.issueSessionCertificateDER(ctx, ns, sess, csrDER, time.Hour) + + if capturedRequest == "" { + t.Fatal("no CertificateRequest was created") + } + decoded, err := base64.StdEncoding.DecodeString(capturedRequest) + if err != nil { + t.Fatalf("spec.request is not valid base64: %v", err) + } + block, _ := pem.Decode(decoded) + if block == nil || block.Type != "CERTIFICATE REQUEST" { + t.Fatalf("spec.request must be a PEM CERTIFICATE REQUEST, got: %q", string(decoded)) + } +} + func TestAdapterCertificateRequestFailure(t *testing.T) { t.Parallel() request := &unstructured.Unstructured{Object: map[string]any{ From 1a86012eef4cbf410ca68221a4b43b35d334df06 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 1 Jul 2026 11:55:38 +0530 Subject: [PATCH 07/21] test(e2e): archive mtls control-plane diagnostics on failure An mtls scenario failure (e.g. adapter cert issuance) previously left no server-side evidence in the CI artifact bundle, forcing local repro to find the root cause. Add an ERR-trap diagnostics collector (with set -E so it fires inside functions and the adapter-enroll command substitution) that dumps the MCPServer, CertificateRequests, certs/secrets, sentinel ConfigMap, and runtime-api/platform-api/mtls-server logs into WORKDIR, which the EXIT trap archives. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/scenarios/mtls.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 132e4db9..ecb36547 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -6,7 +6,36 @@ MTLS_TRUST_DOMAIN="${MTLS_TRUST_DOMAIN:-cluster.local}" TRAEFIK_TLS_PORT="${TRAEFIK_TLS_PORT:-18443}" MTLS_AGENT_ID="${MTLS_AGENT_ID:-e2e-mtls-agent}" +# mtls_dump_diagnostics captures the control-plane state needed to debug an mtls +# datapath failure into WORKDIR, which the EXIT trap archives to the CI artifact +# bundle. Without this, an mtls failure (e.g. adapter cert issuance) leaves no +# server-side evidence in the artifacts and can only be diagnosed by local repro. +mtls_dump_diagnostics() { + local rc=$? + local out="${WORKDIR}/mtls-diagnostics" + mkdir -p "${out}" + echo "[mtls] capturing failure diagnostics (exit=${rc}) to ${out}" >&2 + { + kubectl get mcpserver "${MTLS_SERVER_NAME}" -n mcp-servers -o yaml + } >"${out}/mcpserver.yaml" 2>&1 || true + kubectl get certificaterequests -n mcp-servers -o yaml >"${out}/certificaterequests.yaml" 2>&1 || true + kubectl get certificates,secrets -n mcp-servers >"${out}/certs-and-secrets.txt" 2>&1 || true + kubectl get configmap mcp-sentinel-config -n mcp-sentinel -o yaml >"${out}/sentinel-config.yaml" 2>&1 || true + kubectl logs -n mcp-sentinel deploy/mcp-runtime-api --tail=400 >"${out}/runtime-api.log" 2>&1 || true + kubectl logs -n mcp-sentinel deploy/mcp-platform-api --tail=200 >"${out}/platform-api.log" 2>&1 || true + kubectl logs -n mcp-servers -l "app=${MTLS_SERVER_NAME}" --all-containers=true --tail=200 >"${out}/mtls-server.log" 2>&1 || true + kubectl get events -n mcp-servers --sort-by=.lastTimestamp >"${out}/mcp-servers-events.txt" 2>&1 || true + return "${rc}" +} + run_e2e_mtls_scenario() { + # Surface server-side diagnostics on any unexpected failure in this scenario. + # errtrace (set -E) propagates the ERR trap into functions and command + # substitutions (e.g. the `$(... adapter enroll ...)` capture) so those + # failures are archived too. This scenario runs last, so scoping is moot. + set -E + trap 'mtls_dump_diagnostics' ERR + log_line mtls "verifying test-mode workload PKI" if ! kubectl get clusterissuer mcp-runtime-ca >/dev/null 2>&1; then echo "expected ClusterIssuer mcp-runtime-ca from test-mode setup" >&2 From 1cd73641cdca5bac46c9d7b927f28b5ec26df064 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 1 Jul 2026 13:19:43 +0530 Subject: [PATCH 08/21] fix(test): extract mtls session name with a strict token class The session-name regex used [^\\s]+, which (double-escaped) matched "not backslash or the letter s" rather than non-whitespace. Once adapter enroll started succeeding, its `.../session/ (expires ...)` output made the capture swallow " (expire" up to the s in "expires", so the policy wait searched for a garbled name and timed out. Match the session token explicitly ([A-Za-z0-9._-]+). Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/scenarios/mtls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index ecb36547..7d932eef 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -214,7 +214,7 @@ EOF --agent "${MTLS_AGENT_ID}" \ --trust-domain "${MTLS_TRUST_DOMAIN}" \ --output-dir "${MTLS_CERT_DIR}")" - MTLS_SESSION_NAME="$(printf '%s\n' "${MTLS_ENROLL_OUT}" | python3 -c 'import re,sys; m=re.search(r"/session/([^\\s]+)", sys.stdin.read()); print(m.group(1) if m else ""); sys.exit(0 if m else 1)')" + MTLS_SESSION_NAME="$(printf '%s\n' "${MTLS_ENROLL_OUT}" | python3 -c 'import re,sys; m=re.search(r"/session/([A-Za-z0-9._-]+)", sys.stdin.read()); print(m.group(1) if m else ""); sys.exit(0 if m else 1)')" wait_for_policy_text "\"name\": \"${MTLS_SESSION_NAME}\"" "${MTLS_SERVER_NAME}" MTLS_CLIENT_CERT="${MTLS_CERT_DIR}/client.crt" From fb3127013e06b67735c0342c0917c91f772ed165 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 1 Jul 2026 15:03:24 +0530 Subject: [PATCH 09/21] fix(test): reach mtls datapath over ingress host with client-cert SNI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The datapath curls connected to https://127.0.0.1: (SNI 127.0.0.1), so Traefik served its default cert and did not bind the router's client-cert TLS options to that handshake — server verification failed (curl exit 60) and mTLS enforcement wasn't exercised. Give the mtls server an ingressHost so the operator emits a Host()-scoped IngressRoute, reach it via curl --resolve so the SNI selects that router (and its client-cert options), and use -k because the caller-facing server cert is Traefik's self-signed default in HTTP test-mode (no default TLSStore). The scenario still verifies the client-cert mTLS property: no client cert is rejected, the session-bound cert is accepted, spoofed headers denied. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/scenarios/mtls.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 7d932eef..23da1a68 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -5,6 +5,10 @@ MTLS_SERVER_NAME="${MTLS_SERVER_NAME:-mtls-mcp-server}" MTLS_TRUST_DOMAIN="${MTLS_TRUST_DOMAIN:-cluster.local}" TRAEFIK_TLS_PORT="${TRAEFIK_TLS_PORT:-18443}" MTLS_AGENT_ID="${MTLS_AGENT_ID:-e2e-mtls-agent}" +# A dedicated ingress host so the operator generates a Host()-scoped IngressRoute. +# Traefik binds the client-cert TLS options to that SNI, so the mtls handshake is +# only enforced when the caller's SNI matches (curl reaches it via --resolve). +MTLS_INGRESS_HOST="${MTLS_INGRESS_HOST:-mtls-mcp-server.e2e.local}" # mtls_dump_diagnostics captures the control-plane state needed to debug an mtls # datapath failure into WORKDIR, which the EXIT trap archives to the CI artifact @@ -77,6 +81,7 @@ servers: imageTag: ${MTLS_IMAGE##*:} route: /${MTLS_SERVER_NAME}/mcp publicPathPrefix: ${MTLS_SERVER_NAME} + ingressHost: ${MTLS_INGRESS_HOST} port: 8090 namespace: mcp-servers resources: @@ -154,13 +159,17 @@ EOF wait_port "${TRAEFIK_TLS_PORT}" MTLS_INGRESS_PATH="/${MTLS_SERVER_NAME}/mcp" - MTLS_URL="https://127.0.0.1:${TRAEFIK_TLS_PORT}${MTLS_INGRESS_PATH}" - MTLS_CA_FILE="${WORKDIR}/mtls-ca.crt" - kubectl get secret "${MTLS_SERVER_NAME}-mtls-ca" -n mcp-servers -o jsonpath='{.data.ca\.crt}' | decode_base64 >"${MTLS_CA_FILE}" + # Reach the websecure entrypoint over the ingress host so the SNI selects the + # Host()-scoped mtls IngressRoute (and its client-cert TLS options). --resolve + # maps the host to the port-forwarded Traefik on localhost. The caller-facing + # server certificate is Traefik's default (self-signed in test-mode, no default + # TLSStore), so use -k: this scenario verifies the *client* cert mTLS + # enforcement, not the server certificate. + MTLS_URL="https://${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}${MTLS_INGRESS_PATH}" + MTLS_RESOLVE="${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}:127.0.0.1" log_line mtls "rejecting initialize without a client certificate" - if curl -fsS --cacert "${MTLS_CA_FILE}" \ - -H "Host: ${SERVER_HOST}" \ + if curl -fsS -k --resolve "${MTLS_RESOLVE}" \ -H "content-type: application/json" \ -H "accept: application/json, text/event-stream" \ -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ @@ -228,11 +237,9 @@ EOF done log_line mtls "accepting initialize with session-bound client certificate" - init_body="$(curl -fsS \ + init_body="$(curl -fsS -k --resolve "${MTLS_RESOLVE}" \ --cert "${MTLS_CLIENT_CERT}" \ --key "${MTLS_CLIENT_KEY}" \ - --cacert "${MTLS_CLIENT_CA}" \ - -H "Host: ${SERVER_HOST}" \ -H "content-type: application/json" \ -H "accept: application/json, text/event-stream" \ -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ @@ -252,11 +259,9 @@ print("mtls initialize ok") # Intentionally omit curl -f: the gateway is expected to reject the spoofed # headers with a 4xx/5xx, and we still need to capture the response body so the # Python check below can validate the error instead of being skipped. - spoof_body="$(curl -sS \ + spoof_body="$(curl -sS -k --resolve "${MTLS_RESOLVE}" \ --cert "${MTLS_CLIENT_CERT}" \ --key "${MTLS_CLIENT_KEY}" \ - --cacert "${MTLS_CLIENT_CA}" \ - -H "Host: ${SERVER_HOST}" \ -H "content-type: application/json" \ -H "accept: application/json, text/event-stream" \ -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ From 49808f095bf4dc6940116aa02c7e6084e10dc3f9 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 00:52:34 +0530 Subject: [PATCH 10/21] fix(ci): enable Traefik CRD provider + spiffe plugin in http ingress overlay The mtls terminate-and-re-encrypt datapath is delivered through Traefik CRDs (IngressRoute/Middleware/TLSOption/ServersTransport) and the spiffe-identity local plugin, but the http overlay's args patch replaced the base args and dropped --providers.kubernetescrd, the websecure TLS default, and the spiffe-identity plugin registration. Traefik therefore ignored every IngressRoute the operator created and the mtls host 404'd. Re-add the CRD provider (scoped to the server namespaces), websecure http.tls, and the spiffe-identity plugin module while keeping web plaintext for the HTTP test flows. Verified on a live cluster: with the CRD provider enabled Traefik loads the IngressRoute and routes the host. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ingress/overlays/http/deployment-args.patch.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/ingress/overlays/http/deployment-args.patch.yaml b/config/ingress/overlays/http/deployment-args.patch.yaml index 49e5076a..ad88966c 100644 --- a/config/ingress/overlays/http/deployment-args.patch.yaml +++ b/config/ingress/overlays/http/deployment-args.patch.yaml @@ -3,15 +3,27 @@ value: - --providers.kubernetesingress=true - --providers.kubernetesingress.namespaces=registry,mcp-sentinel,mcp-servers,mcp-servers-org,mcp-servers-public + # The mtls terminate+re-encrypt path is delivered entirely through Traefik + # CRDs (IngressRoute + Middleware + TLSOption + ServersTransport). Without the + # CRD provider Traefik ignores them and the mtls host 404s, so enable it for + # the same server namespaces. web stays plaintext (no redirect) for the HTTP + # test flows. + - --providers.kubernetescrd=true + - --providers.kubernetescrd.namespaces=mcp-servers,mcp-servers-org,mcp-servers-public - --entrypoints.web.address=:8000 - --entrypoints.web.forwardedHeaders.insecure=false - --entrypoints.websecure.address=:8443 - --entrypoints.websecure.forwardedHeaders.insecure=false + - --entrypoints.websecure.http.tls=true - --entrypoints.traefik.address=:8080 - --ping=true - --ping.entryPoint=traefik - --providers.file.filename=/etc/traefik/dynamic/dynamic.yml - --providers.file.watch=true + # The mtls Middleware injects the verified SPIFFE identity via this local + # plugin (source is mounted by the base Deployment); register it so the + # middleware resolves instead of failing the route. + - --experimental.localPlugins.spiffe-identity.moduleName=github.com/Agent-Hellboy/mcp-runtime/traefik-plugins/spiffe-identity - op: replace path: /spec/template/spec/containers/0/ports value: From f32b0adc78322b51e6ec7f7322fde63411ceb002 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 00:52:48 +0530 Subject: [PATCH 11/21] test(e2e): expand mtls failure diagnostics with Traefik routing + replay Capture the Traefik routing layer (IngressRoute/Middleware/TLSOption/ ServersTransport CRs, the Traefik deployment and controller logs), the mcp-servers workload/pod describe/events, and a verbose curl replay of the datapath request (status, headers, body) into the archived artifact bundle, so a routing failure (404 vs 502 vs deny) is diagnosable from CI instead of requiring local repro. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/scenarios/mtls.sh | 43 ++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 23da1a68..3237c0db 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -17,18 +17,53 @@ MTLS_INGRESS_HOST="${MTLS_INGRESS_HOST:-mtls-mcp-server.e2e.local}" mtls_dump_diagnostics() { local rc=$? local out="${WORKDIR}/mtls-diagnostics" + local traefik_ns="${TRAEFIK_NAMESPACE:-traefik}" mkdir -p "${out}" echo "[mtls] capturing failure diagnostics (exit=${rc}) to ${out}" >&2 - { - kubectl get mcpserver "${MTLS_SERVER_NAME}" -n mcp-servers -o yaml - } >"${out}/mcpserver.yaml" 2>&1 || true + + # --- MCPServer / workload state --- + kubectl get mcpserver "${MTLS_SERVER_NAME}" -n mcp-servers -o yaml >"${out}/mcpserver.yaml" 2>&1 || true + kubectl get all -n mcp-servers >"${out}/mcp-servers-all.txt" 2>&1 || true + kubectl describe pods -n mcp-servers -l "app=${MTLS_SERVER_NAME}" >"${out}/mtls-server-pods-describe.txt" 2>&1 || true + kubectl get events -n mcp-servers --sort-by=.lastTimestamp >"${out}/mcp-servers-events.txt" 2>&1 || true + + # --- Certificates / issuance --- kubectl get certificaterequests -n mcp-servers -o yaml >"${out}/certificaterequests.yaml" 2>&1 || true kubectl get certificates,secrets -n mcp-servers >"${out}/certs-and-secrets.txt" 2>&1 || true kubectl get configmap mcp-sentinel-config -n mcp-sentinel -o yaml >"${out}/sentinel-config.yaml" 2>&1 || true + + # --- Traefik routing layer (why a request 404s / 502s) --- + # The mtls datapath depends on Traefik CRDs (IngressRoute + Middleware + + # TLSOption + ServersTransport) being provisioned AND picked up by Traefik's + # CRD provider. Dump the CRs and the controller so we can tell an operator + # provisioning gap from a Traefik provider/namespace-scope gap. + kubectl get ingressroutes,ingressroutetcps,middlewares,tlsoptions,serverstransports,tlsstores \ + -A -o yaml >"${out}/traefik-crs.yaml" 2>&1 || true + kubectl get deploy -n "${traefik_ns}" traefik -o yaml >"${out}/traefik-deploy.yaml" 2>&1 || true + kubectl logs -n "${traefik_ns}" deploy/traefik --tail=400 >"${out}/traefik.log" 2>&1 || true + + # --- Sentinel APIs (adapter cert issuance path) --- kubectl logs -n mcp-sentinel deploy/mcp-runtime-api --tail=400 >"${out}/runtime-api.log" 2>&1 || true kubectl logs -n mcp-sentinel deploy/mcp-platform-api --tail=200 >"${out}/platform-api.log" 2>&1 || true kubectl logs -n mcp-servers -l "app=${MTLS_SERVER_NAME}" --all-containers=true --tail=200 >"${out}/mtls-server.log" 2>&1 || true - kubectl get events -n mcp-servers --sort-by=.lastTimestamp >"${out}/mcp-servers-events.txt" 2>&1 || true + + # --- Replay the datapath request verbosely so the exact HTTP status, + # response headers and body (Traefik 404 vs gateway error vs deny) are in the + # artifact even when the assertion curl ran quietly. --- + if [[ -n "${MTLS_URL:-}" && -n "${MTLS_RESOLVE:-}" ]]; then + local -a replay=(curl -sS -k -v -o "${out}/datapath-replay-body.txt" + -w 'HTTP %{http_code}\n' --resolve "${MTLS_RESOLVE}" + -H "content-type: application/json" + -H "accept: application/json, text/event-stream" + -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" + --data '{"jsonrpc":"2.0","id":99,"method":"initialize","params":{"protocolVersion":"'"${MCP_PROTOCOL_VERSION}"'","capabilities":{},"clientInfo":{"name":"diag","version":"1"}}}') + if [[ -s "${MTLS_CLIENT_CERT:-}" && -s "${MTLS_CLIENT_KEY:-}" ]]; then + replay+=(--cert "${MTLS_CLIENT_CERT}" --key "${MTLS_CLIENT_KEY}") + fi + replay+=("${MTLS_URL}") + "${replay[@]}" >"${out}/datapath-replay-status.txt" 2>"${out}/datapath-replay-trace.txt" || true + fi + return "${rc}" } From 2d8f82dd1872396cd96660dd426f0b0fa4664646 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 01:25:42 +0530 Subject: [PATCH 12/21] test(mcp-gateway): make cert-reload watch test race-proof Widening the deadline wasn't enough: watch() captures its baseline modtime in its own goroutine, so under CI load it could start after the test rewrote the cert, baseline on cert-2's modtime, and never observe a change (the reload never fired, so it hit the deadline at ~10s). Push the cert modtime strictly forward on every poll so the watcher sees a fresh change no matter when its goroutine started, making the reload deterministic instead of racing the goroutine start. Co-Authored-By: Claude Opus 4.8 (1M context) --- services/mcp-gateway/cert_reload_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/services/mcp-gateway/cert_reload_test.go b/services/mcp-gateway/cert_reload_test.go index d1ceb1c6..84c45ab9 100644 --- a/services/mcp-gateway/cert_reload_test.go +++ b/services/mcp-gateway/cert_reload_test.go @@ -106,15 +106,18 @@ func TestCertReloaderWatchReloadsOnChange(t *testing.T) { defer cancel() go r.watch(ctx, 20*time.Millisecond) - // Bump modtime into the future so the poll reliably detects the change. second := writeKeyPair(t, certPath, keyPath, "cert-2") - future := time.Now().Add(2 * time.Second) - _ = os.Chtimes(certPath, future, future) - // Generous deadline so the 20ms poll is detected even when CI runs the - // service test suites in parallel under a constrained CPU quota. + // watch() captures its baseline modtime asynchronously, so under CI load it + // may start after this rewrite and baseline on cert-2's modtime — then it + // would never observe a "change" and never reload. Push the modtime strictly + // forward on every poll so the watcher sees a fresh change regardless of when + // its goroutine started; that makes the reload deterministic rather than + // racing the goroutine start. deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { + for i := 1; time.Now().Before(deadline); i++ { + future := time.Now().Add(time.Duration(i) * time.Second) + _ = os.Chtimes(certPath, future, future) if string(servedLeaf(t, r)) == string(second) { return } From ea70f476bf9877306b9dfe76440bcb022c706117 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 01:56:57 +0530 Subject: [PATCH 13/21] fix(ci): grant Traefik RBAC for all CRD kinds in both API groups Traefik v2.10 starts informers for every IngressRoute/Middleware/TLSOption/ ServersTransport/... kind in BOTH traefik.io and the legacy traefik.containo.us group whenever those CRDs are installed. The traefik-ingressclass ClusterRole only granted a subset of traefik.io, so list/watch on traefik.containo.us (and traefik.io udp/tcp kinds) was forbidden. That stalls the kubernetescrd provider, so no IngressRoute is loaded and mtls hosts fall through to Traefik's default cert and 404. Grant get/list/watch on the full kind set across both groups. Verified on a live cluster: the forbidden errors clear and the provider syncs. Co-Authored-By: Claude Opus 4.8 (1M context) --- config/ingress/base/traefik.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/config/ingress/base/traefik.yaml b/config/ingress/base/traefik.yaml index 493dce28..09b4a839 100644 --- a/config/ingress/base/traefik.yaml +++ b/config/ingress/base/traefik.yaml @@ -74,9 +74,23 @@ rules: - apiGroups: ["networking.k8s.io"] resources: ["ingressclasses"] verbs: ["get", "list", "watch"] - - apiGroups: ["traefik.io"] + # Traefik v2.10 starts informers for every CRD kind in BOTH the traefik.io and + # legacy traefik.containo.us groups when those CRDs are installed. A forbidden + # list/watch on any one of them stalls the CRD provider so no IngressRoute is + # loaded and mtls hosts 404. Grant the full kind set in both groups. + - apiGroups: ["traefik.io", "traefik.containo.us"] resources: - ["ingressroutes", "ingressroutetcps", "middlewares", "tlsoptions", "tlsstores", "serverstransports", "traefikservices"] + [ + "ingressroutes", + "ingressroutetcps", + "ingressrouteudps", + "middlewares", + "middlewaretcps", + "tlsoptions", + "tlsstores", + "serverstransports", + "traefikservices", + ] verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 From f0b752ad7b10c05664bbad61a39ca7361182c872 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 02:52:03 +0530 Subject: [PATCH 14/21] fix(operator): IngressRoute must use Service port not gateway container port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Traefik's KubernetesCRD provider resolves IngressRoute service references by looking up the port in Service.spec.ports. The mTLS IngressRoute was passing gateway.port (8091 — the container/targetPort) instead of servicePort (80 — the Service's exposed port). Traefik logged "service port not found: 8091" and skipped the route entirely, falling back to a 404 with its default cert. The correct value is ServicePort (80). Traefik discovers pod endpoints from the Service and connects to each pod at the resolved targetPort (8091) automatically — the IngressRoute only needs the Service-level port. Adds a regression assertion to TestReconcileMTLSIngressGeneratesTraefikResources and seeds ServicePort=80 in the shared mtlsServer() fixture. Co-Authored-By: Claude Sonnet 4.6 --- internal/operator/mtls.go | 8 +++++++- internal/operator/mtls_certs_test.go | 7 ++++--- internal/operator/mtls_ingress_test.go | 16 +++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/internal/operator/mtls.go b/internal/operator/mtls.go index b8fac959..22104f41 100644 --- a/internal/operator/mtls.go +++ b/internal/operator/mtls.go @@ -414,9 +414,15 @@ func (r *MCPServerReconciler) reconcileMTLSIngress(ctx context.Context, mcpServe "match": match, "kind": "Rule", "middlewares": []any{map[string]any{"name": mtlsMiddlewareName(mcpServer)}}, + // The IngressRoute references the Kubernetes Service port (ServicePort, + // typically 80), not the gateway container port. Traefik resolves + // endpoints via the Service and connects to each pod at the targetPort + // (Gateway.Port, e.g. 8091) automatically. Using the container port + // directly causes "service port not found" because that port number does + // not appear in the Service's spec.ports. "services": []any{map[string]any{ "name": mcpServer.Name, - "port": int64(mcpServer.Spec.Gateway.Port), + "port": int64(mcpServer.Spec.ServicePort), "serversTransport": mtlsServersTransportName(mcpServer), }}, }}, diff --git a/internal/operator/mtls_certs_test.go b/internal/operator/mtls_certs_test.go index 5bfc0ae6..01560a7e 100644 --- a/internal/operator/mtls_certs_test.go +++ b/internal/operator/mtls_certs_test.go @@ -19,9 +19,10 @@ func mtlsServer() *mcpv1alpha1.MCPServer { return &mcpv1alpha1.MCPServer{ ObjectMeta: metav1.ObjectMeta{Name: "secure-server", Namespace: "mcp-servers"}, Spec: mcpv1alpha1.MCPServerSpec{ - Image: "example.com/secure-server", - Gateway: &mcpv1alpha1.GatewayConfig{Enabled: true, Port: 8091, Image: "example.com/gw:latest"}, - Auth: &mcpv1alpha1.AuthConfig{Mode: mcpv1alpha1.AuthModeMTLS, TrustDomain: "example.org"}, + Image: "example.com/secure-server", + ServicePort: 80, + Gateway: &mcpv1alpha1.GatewayConfig{Enabled: true, Port: 8091, Image: "example.com/gw:latest"}, + Auth: &mcpv1alpha1.AuthConfig{Mode: mcpv1alpha1.AuthModeMTLS, TrustDomain: "example.org"}, }, } } diff --git a/internal/operator/mtls_ingress_test.go b/internal/operator/mtls_ingress_test.go index 2497c9c8..44f52176 100644 --- a/internal/operator/mtls_ingress_test.go +++ b/internal/operator/mtls_ingress_test.go @@ -102,13 +102,27 @@ func TestReconcileMTLSIngressGeneratesTraefikResources(t *testing.T) { if len(routes) != 1 { t.Fatalf("routes = %d, want 1", len(routes)) } - match, _ := routes[0].(map[string]any)["match"].(string) + route0 := routes[0].(map[string]any) + match, _ := route0["match"].(string) if !strings.Contains(match, "Host(`mcp.example.com`)") || !strings.Contains(match, "PathPrefix(`/secure-server/mcp`)") { t.Fatalf("match = %q, want host + path prefix", match) } if tlsName, _, _ := unstructured.NestedString(ir.Object, "spec", "tls", "options", "name"); tlsName != mtlsTLSOptionName(server) { t.Fatalf("tls.options.name = %q", tlsName) } + // The IngressRoute must reference the Kubernetes Service port (80), not the + // gateway container port (8091). Traefik resolves endpoints from the Service + // and then connects to pod:targetPort — using the container port causes + // "service port not found" because it never appears in Service.spec.ports. + services, _, _ := unstructured.NestedSlice(route0, "services") + if len(services) != 1 { + t.Fatalf("route services = %d, want 1", len(services)) + } + svcPort, _, _ := unstructured.NestedFieldNoCopy(services[0].(map[string]any), "port") + if svcPort != int64(server.Spec.ServicePort) { + t.Fatalf("IngressRoute service port = %v, want ServicePort (%d) not Gateway.Port (%d)", + svcPort, server.Spec.ServicePort, server.Spec.Gateway.Port) + } } func TestReconcileMTLSIngressNeverSetsPerRouteSecretName(t *testing.T) { From 77bb8560ae0ff10d575d609e849d916d07b5086a Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 14:50:41 +0530 Subject: [PATCH 15/21] fix(test): recover Traefik TLS port-forward after mTLS config reload When cert-manager issues the CA bundle secrets after MCPServer creation, Traefik receives a watch event and reloads its TLS config (TLSOption and ServersTransport). The reload briefly closes the websecure (8443) listener, which breaks the port-forward with "broken pipe" and leaves port 18443 refusing connections. Add recover_traefik_tls_port_forward_if_needed() to kind.sh (mirrors recover_traefik_port_forward_if_needed for the HTTP port) and call it in mtls.sh before the authenticated mTLS curl, which runs ~30s after the initial port-forward start (grant + adapter-enroll round-trips in between). Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/kind.sh | 25 +++++++++++++++++++++++++ test/e2e/scenarios/mtls.sh | 6 ++++++ 2 files changed, 31 insertions(+) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 1f7b4979..9e53c279 100644 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -842,6 +842,31 @@ recover_traefik_port_forward_if_needed() { wait_port "${TRAEFIK_PORT}" 30 } +# recover_traefik_tls_port_forward_if_needed mirrors recover_traefik_port_forward_if_needed +# for the websecure (TLS) entrypoint. The TLS port-forward can die when Traefik +# reloads its mTLS config (TLSOption / ServersTransport) after cert-manager issues +# the CA bundle secrets — a watch-event-driven reload briefly closes the listener, +# which breaks the port-forward with "broken pipe". Call this before any mTLS curl +# that runs after the cert-issuance wait. +recover_traefik_tls_port_forward_if_needed() { + if port_is_listening "${TRAEFIK_TLS_PORT}"; then + return 0 + fi + + if [[ -n "${TRAEFIK_TLS_PORT_FORWARD_PID:-}" ]] && kill -0 "${TRAEFIK_TLS_PORT_FORWARD_PID}" >/dev/null 2>&1; then + kill "${TRAEFIK_TLS_PORT_FORWARD_PID}" >/dev/null 2>&1 || true + wait "${TRAEFIK_TLS_PORT_FORWARD_PID}" >/dev/null 2>&1 || true + fi + TRAEFIK_TLS_PORT_FORWARD_PID="" + + TRAEFIK_TLS_PORT_FORWARD_RESTARTS=$((TRAEFIK_TLS_PORT_FORWARD_RESTARTS + 1)) + local log_file="${WORKDIR}/traefik-tls-port-forward-restart-${TRAEFIK_TLS_PORT_FORWARD_RESTARTS}.log" + echo "[port-forward] restarting Traefik TLS port-forward on localhost:${TRAEFIK_TLS_PORT}" >&2 + port_forward_bg traefik traefik "${TRAEFIK_TLS_PORT}" 8443 "${log_file}" || return 1 + TRAEFIK_TLS_PORT_FORWARD_PID="${LAST_MANAGED_PID}" + wait_port "${TRAEFIK_TLS_PORT}" 30 +} + ensure_traefik_port_forward() { if [[ -n "${TRAEFIK_PORT_FORWARD_PID:-}" ]] && ! port_is_listening "${TRAEFIK_PORT}"; then if kill -0 "${TRAEFIK_PORT_FORWARD_PID}" >/dev/null 2>&1; then diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 3237c0db..7c744622 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -190,6 +190,7 @@ EOF echo "[port-forward] exposing traefik websecure on localhost:${TRAEFIK_TLS_PORT}" port_forward_bg traefik traefik "${TRAEFIK_TLS_PORT}" 8443 "${WORKDIR}/traefik-tls-port-forward.log" TRAEFIK_TLS_PORT_FORWARD_PID="${LAST_MANAGED_PID}" + TRAEFIK_TLS_PORT_FORWARD_RESTARTS=0 fi wait_port "${TRAEFIK_TLS_PORT}" @@ -271,6 +272,11 @@ EOF fi done + # Traefik reloads TLS config (TLSOption/ServersTransport) once the CA bundle + # secrets appear, briefly closing the websecure listener and breaking the + # port-forward. Recover before the authenticated curl so the connection is live. + recover_traefik_tls_port_forward_if_needed + log_line mtls "accepting initialize with session-bound client certificate" init_body="$(curl -fsS -k --resolve "${MTLS_RESOLVE}" \ --cert "${MTLS_CLIENT_CERT}" \ From 876858f4e9429e74bcb51a7eb7c5f1ee4a70d12e Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 15:46:14 +0530 Subject: [PATCH 16/21] test(e2e): wait for Traefik mTLS router to enforce client cert before curls Traefik applies TLSOption and ServersTransport atomically during a config reload triggered by cert-manager issuing the CA bundle secret. Until that reload completes, TLSOption lacks its CAFiles so RequireAndVerifyClientCert is not enforced: the TLS handshake succeeds without a client cert, Traefik routes the request via plain HTTP (ServersTransport not yet loaded), and the TLS-only gateway returns HTTP 400. Add wait_for_mtls_traefik_ready() in kind.sh that polls until a no-cert curl exits 35 (TLS handshake failure = certificate_required alert). This proves TLSOption is active; because both TLSOption and ServersTransport are reloaded in the same Traefik config snapshot, exit-35 also implies ServersTransport is using TLS to the backend. Call it in mtls.sh after the port-forward is up and before the reject/accept curl sequence. Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/kind.sh | 33 +++++++++++++++++++++++++++++++++ test/e2e/scenarios/mtls.sh | 2 ++ 2 files changed, 35 insertions(+) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 9e53c279..0a83b9be 100644 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -867,6 +867,39 @@ recover_traefik_tls_port_forward_if_needed() { wait_port "${TRAEFIK_TLS_PORT}" 30 } +# wait_for_mtls_traefik_ready polls until Traefik's mTLS router is fully active. +# The signal: a no-cert curl exits 35 (TLS handshake failure — server sent a +# certificate_required alert). When TLSOption has not yet loaded its CAFiles, +# Traefik omits RequireAndVerifyClientCert; the TLS handshake succeeds, the +# request reaches the TLS-only gateway, and the gateway returns HTTP 400 (curl +# exits 22). Exit 35 therefore proves TLSOption is enforcing client-cert +# requirements. Because Traefik applies TLSOption and ServersTransport atomically +# in the same config reload, exit 35 also implies ServersTransport is using TLS +# to the backend — so a subsequent accept-cert curl will not get the 400. +wait_for_mtls_traefik_ready() { + local url="$1" + local resolve="$2" + local deadline=$((SECONDS + 60)) + local code=0 + echo "[mtls] waiting for Traefik mTLS router to enforce client cert" >&2 + while [[ $SECONDS -lt $deadline ]]; do + recover_traefik_tls_port_forward_if_needed || true + code=0 + curl -sS -k --connect-timeout 3 --max-time 5 \ + --resolve "${resolve}" \ + -H "content-type: application/json" \ + --data '{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"probe","version":"1"}}}' \ + "${url}" >/dev/null 2>&1 || code=$? + if [[ "$code" -eq 35 ]]; then + echo "[mtls] Traefik mTLS router active (exit 35 on no-cert probe)" >&2 + return 0 + fi + sleep 2 + done + echo "[mtls] timed out waiting for Traefik mTLS router (last exit=${code})" >&2 + return 1 +} + ensure_traefik_port_forward() { if [[ -n "${TRAEFIK_PORT_FORWARD_PID:-}" ]] && ! port_is_listening "${TRAEFIK_PORT}"; then if kill -0 "${TRAEFIK_PORT_FORWARD_PID}" >/dev/null 2>&1; then diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 7c744622..e1921288 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -204,6 +204,8 @@ EOF MTLS_URL="https://${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}${MTLS_INGRESS_PATH}" MTLS_RESOLVE="${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}:127.0.0.1" + wait_for_mtls_traefik_ready "${MTLS_URL}" "${MTLS_RESOLVE}" + log_line mtls "rejecting initialize without a client certificate" if curl -fsS -k --resolve "${MTLS_RESOLVE}" \ -H "content-type: application/json" \ From f53e3aae0dd97d37ea7256539b8fa3488bcce2e2 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 20:54:07 +0530 Subject: [PATCH 17/21] test(e2e): replace TLS-probe readiness check with Traefik log stability check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Kind clusters, kubectl port-forward exits on any TCP error from the upstream pod — including the RST Traefik sends after a TLS certificate_required alert. The previous wait_for_mtls_traefik_ready approach probed the websecure port with curl, which triggered this RST on every iteration, creating an infinite port-forward restart loop that always timed out (exit 56 each time). Replace with wait_for_mtls_traefik_stable, which polls kubectl logs until no level=error lines mentioning the server name appear in a 6s window. This proves TLSOption (RequireAndVerifyClientCert + CA loaded) and ServersTransport (TLS to gateway) are both applied without touching the websecure port at all. After the stability wait, recover the port-forward once (Traefik's initial secret-loading retries may have broken it) before proceeding to the reject/accept curl sequence. Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/kind.sh | 50 ++++++++++++++++++-------------------- test/e2e/scenarios/mtls.sh | 5 +++- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 0a83b9be..3721da89 100644 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -867,37 +867,35 @@ recover_traefik_tls_port_forward_if_needed() { wait_port "${TRAEFIK_TLS_PORT}" 30 } -# wait_for_mtls_traefik_ready polls until Traefik's mTLS router is fully active. -# The signal: a no-cert curl exits 35 (TLS handshake failure — server sent a -# certificate_required alert). When TLSOption has not yet loaded its CAFiles, -# Traefik omits RequireAndVerifyClientCert; the TLS handshake succeeds, the -# request reaches the TLS-only gateway, and the gateway returns HTTP 400 (curl -# exits 22). Exit 35 therefore proves TLSOption is enforcing client-cert -# requirements. Because Traefik applies TLSOption and ServersTransport atomically -# in the same config reload, exit 35 also implies ServersTransport is using TLS -# to the backend — so a subsequent accept-cert curl will not get the 400. -wait_for_mtls_traefik_ready() { - local url="$1" - local resolve="$2" +# wait_for_mtls_traefik_stable polls Traefik pod logs until the mTLS server's +# router and transport config have been applied error-free for a stable window. +# Traefik retries loading missing secrets with exponential backoff, emitting +# level=error lines for each failed attempt. This function waits until no such +# errors appear in a 6s window, proving that TLSOption (RequireAndVerifyClientCert +# + CA) and ServersTransport (TLS to gateway) are both fully loaded. +# +# This log-based approach is used instead of probing the websecure port because +# in Kind clusters kubectl port-forward exits on any TCP error from the upstream +# pod (broken pipe, connection reset), including the RST Traefik sends after a +# TLS certificate_required alert. Probing via curl therefore creates an infinite +# port-forward restart loop and never produces a usable readiness signal. +wait_for_mtls_traefik_stable() { + local server_name="$1" + local traefik_ns="${TRAEFIK_NAMESPACE:-traefik}" local deadline=$((SECONDS + 60)) - local code=0 - echo "[mtls] waiting for Traefik mTLS router to enforce client cert" >&2 + echo "[mtls] waiting for Traefik to apply ${server_name} mTLS config without errors" >&2 while [[ $SECONDS -lt $deadline ]]; do - recover_traefik_tls_port_forward_if_needed || true - code=0 - curl -sS -k --connect-timeout 3 --max-time 5 \ - --resolve "${resolve}" \ - -H "content-type: application/json" \ - --data '{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"probe","version":"1"}}}' \ - "${url}" >/dev/null 2>&1 || code=$? - if [[ "$code" -eq 35 ]]; then - echo "[mtls] Traefik mTLS router active (exit 35 on no-cert probe)" >&2 + sleep 3 + local recent + recent="$(kubectl logs -n "${traefik_ns}" deploy/traefik --since=6s 2>/dev/null \ + | grep "level=error.*${server_name}" || true)" + if [[ -z "${recent}" ]]; then + echo "[mtls] Traefik mTLS config stable (no ${server_name} errors in last 6s)" >&2 return 0 fi - sleep 2 done - echo "[mtls] timed out waiting for Traefik mTLS router (last exit=${code})" >&2 - return 1 + echo "[mtls] WARNING: Traefik mTLS config may not be fully stable, proceeding" >&2 + return 0 } ensure_traefik_port_forward() { diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index e1921288..ec6cf637 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -204,7 +204,10 @@ EOF MTLS_URL="https://${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}${MTLS_INGRESS_PATH}" MTLS_RESOLVE="${MTLS_INGRESS_HOST}:${TRAEFIK_TLS_PORT}:127.0.0.1" - wait_for_mtls_traefik_ready "${MTLS_URL}" "${MTLS_RESOLVE}" + wait_for_mtls_traefik_stable "${MTLS_SERVER_NAME}" + # Initial Traefik config reloads (secret-loading retries) may have killed the + # port-forward via broken-pipe. Recover before the first mTLS curl. + recover_traefik_tls_port_forward_if_needed log_line mtls "rejecting initialize without a client certificate" if curl -fsS -k --resolve "${MTLS_RESOLVE}" \ From 756a6e69b64d61d466b8d8ce25956ced03794845 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 2 Jul 2026 23:23:38 +0530 Subject: [PATCH 18/21] fix(operator): add scheme https to IngressRoute service for mTLS backend Without scheme: https, Traefik defaults to plain HTTP for backends on port 80. The ServersTransport TLS config (CA, client cert, serverName) is only applied when Traefik connects via HTTPS, so omitting the scheme caused Traefik to forward requests over HTTP and the gateway responded with 400 "Client sent an HTTP request to an HTTPS server". Co-Authored-By: Claude Sonnet 4.6 --- internal/operator/mtls.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/operator/mtls.go b/internal/operator/mtls.go index 22104f41..9ff44820 100644 --- a/internal/operator/mtls.go +++ b/internal/operator/mtls.go @@ -421,8 +421,13 @@ func (r *MCPServerReconciler) reconcileMTLSIngress(ctx context.Context, mcpServe // directly causes "service port not found" because that port number does // not appear in the Service's spec.ports. "services": []any{map[string]any{ - "name": mcpServer.Name, - "port": int64(mcpServer.Spec.ServicePort), + "name": mcpServer.Name, + "port": int64(mcpServer.Spec.ServicePort), + // scheme: https tells Traefik to connect to the backend over TLS. + // Without it, Traefik defaults to HTTP (port 80 is not 443), and + // the ServersTransport TLS config is ignored, causing the gateway to + // return 400 "Client sent an HTTP request to an HTTPS server". + "scheme": "https", "serversTransport": mtlsServersTransportName(mcpServer), }}, }}, From 01188d5f8e53777019717884d1971547b87676dc Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Fri, 3 Jul 2026 00:14:13 +0530 Subject: [PATCH 19/21] test(e2e): retry accepting-initialize curl until gateway reloads policy The gateway reloads its policy from the volume-mounted ConfigMap every 5s. wait_for_policy_text confirms the ConfigMap API is updated, but the kubelet may not have propagated the change to the pod's volume mount yet, so the next gateway reload can still read the old policy. Previously the accepting initialize curl ran immediately after wait_for_policy_text, racing the policy reload cycle and returning 401 session_not_found. Retry up to 15 times (30s) until the response contains "result", with port-forward recovery and a 2s sleep between attempts. Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/scenarios/mtls.sh | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index ec6cf637..55c6e62d 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -283,14 +283,35 @@ EOF recover_traefik_tls_port_forward_if_needed log_line mtls "accepting initialize with session-bound client certificate" - init_body="$(curl -fsS -k --resolve "${MTLS_RESOLVE}" \ - --cert "${MTLS_CLIENT_CERT}" \ - --key "${MTLS_CLIENT_KEY}" \ - -H "content-type: application/json" \ - -H "accept: application/json, text/event-stream" \ - -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ - --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"'"${MCP_PROTOCOL_VERSION}"'","capabilities":{},"clientInfo":{"name":"e2e-mtls","version":"1"}}}' \ - "${MTLS_URL}")" + # The gateway reloads its policy from the volume-mounted ConfigMap every 5s. + # wait_for_policy_text confirmed the ConfigMap was updated, but the kubelet may + # not have propagated the change to the pod volume mount yet, so the gateway's + # next reload may still see the old policy. Retry until the session appears in + # the gateway's active policy (up to 30s). Each 401 means session not yet + # loaded; each iteration recovers the port-forward in case Traefik sent a + # TLS alert during a prior retry. + local _mtls_init_try + local init_body="" + for _mtls_init_try in $(seq 1 15); do + init_body="$(curl -sS -k --resolve "${MTLS_RESOLVE}" \ + --cert "${MTLS_CLIENT_CERT}" \ + --key "${MTLS_CLIENT_KEY}" \ + -H "content-type: application/json" \ + -H "accept: application/json, text/event-stream" \ + -H "Mcp-Protocol-Version: ${MCP_PROTOCOL_VERSION}" \ + --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"'"${MCP_PROTOCOL_VERSION}"'","capabilities":{},"clientInfo":{"name":"e2e-mtls","version":"1"}}}' \ + "${MTLS_URL}" 2>/dev/null || true)" + if [[ "${init_body}" == *'"result"'* ]]; then + break + fi + echo "[mtls] initialize attempt ${_mtls_init_try}/15 failed: ${init_body:0:120}" >&2 + if [[ ${_mtls_init_try} -eq 15 ]]; then + echo "[mtls] timed out: gateway did not materialize session in policy after 15 attempts" >&2 + false # triggers ERR trap → mtls_dump_diagnostics + fi + recover_traefik_tls_port_forward_if_needed + sleep 2 + done echo "${init_body}" | python3 -c ' import json, sys doc = json.load(sys.stdin) From 6de42718fe8eac5db9445e03cf91808eee145f30 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Fri, 3 Jul 2026 02:22:05 +0530 Subject: [PATCH 20/21] fix(test): gate initialize on in-pod policy file, not just ConfigMap API wait_for_policy_text checks the ConfigMap via the API server, but the kubelet propagates ConfigMap updates to volume mounts on its own sync period. On loaded Kind CI nodes this lag exceeded 30s, so all 15 curl retries saw an stale policy file in the gateway pod and returned 401 session_not_found. Add wait_for_gateway_policy_file: kubectl-execs into the gateway pod and polls /var/run/mcp-runtime/policy/policy.json (the actual file the gateway reloads every 5s) until the session name appears, with a 180s deadline. The curl retry loop is kept as a safety net for the gateway's 5s reload tick after the file is updated. Diagnostics improvements: - Dump the in-pod policy file and ConfigMap side by side so future failures can distinguish kubelet propagation lag from operator bugs. - Use fully-qualified group names for Traefik CRD dumps (traefik.io / traefik.containo.us) to avoid ambiguous bare names that return empty lists when both API groups are registered. Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/scenarios/mtls.sh | 65 +++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 55c6e62d..47cf3afe 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -37,8 +37,13 @@ mtls_dump_diagnostics() { # TLSOption + ServersTransport) being provisioned AND picked up by Traefik's # CRD provider. Dump the CRs and the controller so we can tell an operator # provisioning gap from a Traefik provider/namespace-scope gap. - kubectl get ingressroutes,ingressroutetcps,middlewares,tlsoptions,serverstransports,tlsstores \ + # Fully-qualified resource names: the cluster registers both traefik.io and + # traefik.containo.us API groups, so bare names are ambiguous and kubectl + # returns an empty list instead of the CRs. + kubectl get ingressroutes.traefik.io,ingressroutetcps.traefik.io,middlewares.traefik.io,tlsoptions.traefik.io,serverstransports.traefik.io,tlsstores.traefik.io \ -A -o yaml >"${out}/traefik-crs.yaml" 2>&1 || true + kubectl get ingressroutes.traefik.containo.us,middlewares.traefik.containo.us,tlsoptions.traefik.containo.us,serverstransports.traefik.containo.us \ + -A -o yaml >"${out}/traefik-crs-containous.yaml" 2>&1 || true kubectl get deploy -n "${traefik_ns}" traefik -o yaml >"${out}/traefik-deploy.yaml" 2>&1 || true kubectl logs -n "${traefik_ns}" deploy/traefik --tail=400 >"${out}/traefik.log" 2>&1 || true @@ -47,6 +52,20 @@ mtls_dump_diagnostics() { kubectl logs -n mcp-sentinel deploy/mcp-platform-api --tail=200 >"${out}/platform-api.log" 2>&1 || true kubectl logs -n mcp-servers -l "app=${MTLS_SERVER_NAME}" --all-containers=true --tail=200 >"${out}/mtls-server.log" 2>&1 || true + # --- Gateway policy: the ConfigMap (API view) vs the file the gateway + # actually reads (kubelet-propagated volume). A session present in the + # ConfigMap but missing from the file means kubelet volume propagation lag, + # not an operator or gateway bug. --- + kubectl get configmap "${MTLS_SERVER_NAME}-gateway-policy" -n mcp-servers -o yaml \ + >"${out}/gateway-policy-configmap.yaml" 2>&1 || true + local diag_pod + diag_pod="$(kubectl get pod -n mcp-servers -l "app=${MTLS_SERVER_NAME}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [[ -n "${diag_pod}" ]]; then + kubectl exec -n mcp-servers "${diag_pod}" -c mcp-gateway -- \ + cat /var/run/mcp-runtime/policy/policy.json >"${out}/gateway-policy-file-in-pod.json" 2>&1 || true + fi + # --- Replay the datapath request verbosely so the exact HTTP status, # response headers and body (Traefik 404 vs gateway error vs deny) are in the # artifact even when the assertion curl ran quietly. --- @@ -67,6 +86,37 @@ mtls_dump_diagnostics() { return "${rc}" } +# wait_for_gateway_policy_file blocks until the policy file mounted INSIDE the +# gateway pod contains the given text. wait_for_policy_text only checks the +# ConfigMap via the API server; the kubelet propagates ConfigMap updates to +# volume mounts on its own sync period, which can exceed 60s on a loaded Kind +# CI node. The gateway reloads from the mounted FILE (every 5s), so the file — +# not the ConfigMap — is what gates the datapath. Deadline is generous because +# kubelet sync jitter dominates; failure trips the ERR trap for diagnostics. +wait_for_gateway_policy_file() { + local text="$1" + local deadline=$((SECONDS + 180)) + local pod file_content + pod="$(kubectl get pod -n mcp-servers -l "app=${MTLS_SERVER_NAME}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [[ -z "${pod}" ]]; then + echo "[mtls] no gateway pod found for app=${MTLS_SERVER_NAME}" >&2 + return 1 + fi + echo "[mtls] waiting for gateway policy file in ${pod} to contain ${text}" >&2 + while ((SECONDS < deadline)); do + file_content="$(kubectl exec -n mcp-servers "${pod}" -c mcp-gateway -- \ + cat /var/run/mcp-runtime/policy/policy.json 2>/dev/null || true)" + if [[ "${file_content}" == *"${text}"* ]]; then + echo "[mtls] gateway policy file contains ${text}" >&2 + return 0 + fi + sleep 3 + done + echo "[mtls] timed out waiting for gateway policy file to contain ${text}" >&2 + return 1 +} + run_e2e_mtls_scenario() { # Surface server-side diagnostics on any unexpected failure in this scenario. # errtrace (set -E) propagates the ERR trap into functions and command @@ -266,6 +316,7 @@ EOF --output-dir "${MTLS_CERT_DIR}")" MTLS_SESSION_NAME="$(printf '%s\n' "${MTLS_ENROLL_OUT}" | python3 -c 'import re,sys; m=re.search(r"/session/([A-Za-z0-9._-]+)", sys.stdin.read()); print(m.group(1) if m else ""); sys.exit(0 if m else 1)')" wait_for_policy_text "\"name\": \"${MTLS_SESSION_NAME}\"" "${MTLS_SERVER_NAME}" + wait_for_gateway_policy_file "${MTLS_SESSION_NAME}" MTLS_CLIENT_CERT="${MTLS_CERT_DIR}/client.crt" MTLS_CLIENT_KEY="${MTLS_CERT_DIR}/client.key" @@ -283,13 +334,11 @@ EOF recover_traefik_tls_port_forward_if_needed log_line mtls "accepting initialize with session-bound client certificate" - # The gateway reloads its policy from the volume-mounted ConfigMap every 5s. - # wait_for_policy_text confirmed the ConfigMap was updated, but the kubelet may - # not have propagated the change to the pod volume mount yet, so the gateway's - # next reload may still see the old policy. Retry until the session appears in - # the gateway's active policy (up to 30s). Each 401 means session not yet - # loaded; each iteration recovers the port-forward in case Traefik sent a - # TLS alert during a prior retry. + # wait_for_gateway_policy_file confirmed the session reached the file the + # gateway reads, but the gateway's 5s reload tick may not have fired yet, and + # Traefik may still be settling its TLS config after the CA secrets appeared. + # Retry for up to 30s; each iteration recovers the port-forward in case + # Traefik sent a TLS alert during a prior attempt. local _mtls_init_try local init_body="" for _mtls_init_try in $(seq 1 15); do From 52580c5d55bd504d928cafa3250394e1dbfaa4dd Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Fri, 3 Jul 2026 11:51:51 +0530 Subject: [PATCH 21/21] fix(test): use gateway metrics to confirm policy reload, not kubectl exec cat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway container is built FROM scratch (distroless), so kubectl exec -- cat fails with "executable file not found". The previous approach polled the mounted policy file for 180s but silently got empty strings the whole time, causing a guaranteed timeout. Replace wait_for_gateway_policy_file with wait_for_gateway_policy_reload: - Reads the expected revision sha256 from the ConfigMap (fast, API call) - Port-forwards to the gateway pod's metrics port (:9103) - Polls mcp_gateway_policy_active_revision_info until the active revision label matches — confirming both kubelet propagation AND gateway reload This is precise: it waits for exactly the right event (the gateway's 5s reload ticker has fired with the new policy) and does not depend on any shell utilities inside the container. Co-Authored-By: Claude Sonnet 4.6 --- test/e2e/scenarios/mtls.sh | 74 ++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/test/e2e/scenarios/mtls.sh b/test/e2e/scenarios/mtls.sh index 47cf3afe..c824abb0 100644 --- a/test/e2e/scenarios/mtls.sh +++ b/test/e2e/scenarios/mtls.sh @@ -86,34 +86,64 @@ mtls_dump_diagnostics() { return "${rc}" } -# wait_for_gateway_policy_file blocks until the policy file mounted INSIDE the -# gateway pod contains the given text. wait_for_policy_text only checks the -# ConfigMap via the API server; the kubelet propagates ConfigMap updates to -# volume mounts on its own sync period, which can exceed 60s on a loaded Kind -# CI node. The gateway reloads from the mounted FILE (every 5s), so the file — -# not the ConfigMap — is what gates the datapath. Deadline is generous because -# kubelet sync jitter dominates; failure trips the ERR trap for diagnostics. -wait_for_gateway_policy_file() { - local text="$1" - local deadline=$((SECONDS + 180)) - local pod file_content +# wait_for_gateway_policy_reload waits until the gateway has loaded the policy +# revision recorded in the ConfigMap. The gateway runs in a scratch container +# (no shell / cat), so we cannot read the mounted file with kubectl exec. +# Instead we port-forward to the gateway's metrics port and poll +# mcp_gateway_policy_active_revision_info until its "revision" label matches +# the sha256 in the ConfigMap. This confirms: +# 1. kubelet propagated the ConfigMap update to the volume mount, AND +# 2. the gateway's 5-second reload ticker fired and loaded the new policy. +# Deadline is generous because kubelet volume sync jitter dominates on a loaded +# Kind CI node; failure trips the ERR trap → mtls_dump_diagnostics. +wait_for_gateway_policy_reload() { + local deadline=$((SECONDS + 240)) + local pod metrics_port metrics_pf_pid + metrics_port="${MTLS_METRICS_PF_PORT:-19103}" + pod="$(kubectl get pod -n mcp-servers -l "app=${MTLS_SERVER_NAME}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -z "${pod}" ]]; then echo "[mtls] no gateway pod found for app=${MTLS_SERVER_NAME}" >&2 return 1 fi - echo "[mtls] waiting for gateway policy file in ${pod} to contain ${text}" >&2 + + # Grab the expected revision from the ConfigMap (the value the gateway must + # load). The revision field is the sha256 of the policy document. + local want_rev + want_rev="$(kubectl get configmap "${MTLS_SERVER_NAME}-gateway-policy" -n mcp-servers \ + -o jsonpath='{.data.policy\.json}' 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("revision",""))' 2>/dev/null || true)" + if [[ -z "${want_rev}" ]]; then + echo "[mtls] could not read policy revision from ConfigMap; falling back to curl-only sync" >&2 + return 0 + fi + echo "[mtls] waiting for gateway to load policy revision ${want_rev}" >&2 + + kubectl port-forward -n mcp-servers "pod/${pod}" "${metrics_port}:9103" \ + >"${WORKDIR}/mtls-gateway-metrics-pf.log" 2>&1 & + metrics_pf_pid=$! + # Give the port-forward a moment to bind before polling. + local _pf_wait=0 + until curl -sf "http://127.0.0.1:${metrics_port}/metrics" >/dev/null 2>&1 || (( _pf_wait++ >= 10 )); do + sleep 1 + done + while ((SECONDS < deadline)); do - file_content="$(kubectl exec -n mcp-servers "${pod}" -c mcp-gateway -- \ - cat /var/run/mcp-runtime/policy/policy.json 2>/dev/null || true)" - if [[ "${file_content}" == *"${text}"* ]]; then - echo "[mtls] gateway policy file contains ${text}" >&2 + local active_rev + active_rev="$(curl -sf "http://127.0.0.1:${metrics_port}/metrics" 2>/dev/null \ + | grep '^mcp_gateway_policy_active_revision_info{' \ + | grep -o 'revision="[^"]*"' | cut -d'"' -f2 || true)" + if [[ "${active_rev}" == "${want_rev}" ]]; then + echo "[mtls] gateway loaded policy revision ${want_rev}" >&2 + kill "${metrics_pf_pid}" 2>/dev/null || true return 0 fi sleep 3 done - echo "[mtls] timed out waiting for gateway policy file to contain ${text}" >&2 + + kill "${metrics_pf_pid}" 2>/dev/null || true + echo "[mtls] timed out waiting for gateway to load revision ${want_rev}" >&2 return 1 } @@ -316,7 +346,7 @@ EOF --output-dir "${MTLS_CERT_DIR}")" MTLS_SESSION_NAME="$(printf '%s\n' "${MTLS_ENROLL_OUT}" | python3 -c 'import re,sys; m=re.search(r"/session/([A-Za-z0-9._-]+)", sys.stdin.read()); print(m.group(1) if m else ""); sys.exit(0 if m else 1)')" wait_for_policy_text "\"name\": \"${MTLS_SESSION_NAME}\"" "${MTLS_SERVER_NAME}" - wait_for_gateway_policy_file "${MTLS_SESSION_NAME}" + wait_for_gateway_policy_reload MTLS_CLIENT_CERT="${MTLS_CERT_DIR}/client.crt" MTLS_CLIENT_KEY="${MTLS_CERT_DIR}/client.key" @@ -334,11 +364,9 @@ EOF recover_traefik_tls_port_forward_if_needed log_line mtls "accepting initialize with session-bound client certificate" - # wait_for_gateway_policy_file confirmed the session reached the file the - # gateway reads, but the gateway's 5s reload tick may not have fired yet, and - # Traefik may still be settling its TLS config after the CA secrets appeared. - # Retry for up to 30s; each iteration recovers the port-forward in case - # Traefik sent a TLS alert during a prior attempt. + # wait_for_gateway_policy_reload confirmed the gateway loaded the new policy. + # Retry a few times as a safety net in case Traefik needs a moment to settle + # its TLS config, or the port-forward needs to be recovered after a TLS alert. local _mtls_init_try local init_body="" for _mtls_init_try in $(seq 1 15); do