fix(operator): make per-model API-key auth work end-to-end on AI Gateway v0.5 (#116)

charts/nebari-llm-serving/crds/llmmodel-crd.yaml

@@ -1336,6 +1336,17 @@ spec:
               serving:
                 description: serving configures the vLLM serving layer
                 properties:
+                  command:
+                    description: |-
+                      command overrides the container command for the vLLM container.
+                      Defaults to the standard vLLM OpenAI-server entrypoint
+                      (python3 -m vllm.entrypoints.openai.api_server), which the default
+                      serving image requires because its entrypoint is the NVIDIA CUDA
+                      wrapper with no default CMD. Set this when a custom serving image
+                      needs a different launcher; vllmArgs are appended as arguments.
+                    items:
+                      type: string
+                    type: array
                   dataParallelism:
                     default: 1
                     description: dataParallelism controls data parallelism

charts/nebari-llm-serving/templates/operator-clusterrole.yaml

@@ -41,6 +41,13 @@ rules:
   - apiGroups: ["inference.networking.k8s.io"]
     resources: ["inferencepools"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # The operator creates a per-model EPP Role granting these
+  # inference.networking.x-k8s.io permissions (see reconcilers/inferencepool.go).
+  # RBAC privilege-escalation prevention requires the operator to already hold
+  # any permission it grants, so it must carry them here too.
+  - apiGroups: ["inference.networking.x-k8s.io"]
+    resources: ["inferenceobjectives", "inferencemodelrewrites", "inferencepools", "inferencepoolimports"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["aigateway.envoyproxy.io"]
     resources: ["aigatewayroutes", "aiservicebackends", "backendsecuritypolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

dev/Makefile

@@ -24,7 +24,7 @@ AI_GATEWAY_VERSION ?= v0.5.0
 HELM_MAJOR := $(shell helm version --template '{{.Version}}' 2>/dev/null | sed -E 's/^v?([0-9]+).*/\1/')
 HELM_FORCE_CONFLICTS := $(if $(filter-out 1 2 3,$(HELM_MAJOR)),--force-conflicts,)
 
-.PHONY: setup teardown build-images load-images deploy deploy-operator deploy-key-manager deploy-keycloak apply-test-model apply-passthrough-model create-openrouter-secret run-dev ui test-auth logs-operator logs-key-manager logs-keycloak pf-key-manager pf-keycloak clean help
+.PHONY: setup teardown build-images load-images deploy deploy-operator deploy-key-manager deploy-keycloak apply-test-model apply-passthrough-model create-openrouter-secret run-dev ui test-auth test-model-scope logs-operator logs-key-manager logs-keycloak pf-key-manager pf-keycloak clean help
 
 help: ## Show this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
@@ -113,6 +113,9 @@ ui: ## Run the React UI dev server against the local cluster (alias for run-dev)
 test-auth: ## Verify the key-manager's real bearer-token JWKS validation (deploys Keycloak, flips off dev mode, mints a token, asserts 200/401). KEEP_AUTH=1 leaves real-auth mode on.
 	./test-auth.sh
 
+test-model-scope: ## Verify per-model API-key scoping and x-llm-client-id anti-spoofing through the gateway (deploys two mock models, asserts the 200/403/401 matrix). KEEP=1 leaves the models.
+	./test-model-scope.sh
+
 logs-operator: ## Tail operator logs
 	$(KUBECTL) -n llm-operator-system logs -f deployment/llm-operator
 

dev/manifests/operator.yaml

@@ -56,6 +56,14 @@ rules:
   - apiGroups: ["inference.networking.k8s.io"]
     resources: ["inferencepools"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # The operator creates a per-model EPP Role granting these
+  # inference.networking.x-k8s.io permissions (see reconcilers/inferencepool.go).
+  # RBAC privilege-escalation prevention requires the operator to already hold
+  # any permission it grants, so it must carry them here too (mirrors the Helm
+  # chart ClusterRole; #121).
+  - apiGroups: ["inference.networking.x-k8s.io"]
+    resources: ["inferenceobjectives", "inferencemodelrewrites", "inferencepools", "inferencepoolimports"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["aigateway.envoyproxy.io"]
     resources: ["aigatewayroutes", "aiservicebackends", "backendsecuritypolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

dev/manifests/passthrough-test-model.yaml

@@ -5,7 +5,9 @@
 #
 # Once reconciled (kubectl -n llm-operator-system get passthroughmodel), reach
 # it through the gateway. The external endpoint uses API-key auth, so inject a
-# client key into the api-keys Secret to skip the key-manager:
+# client key into the api-keys Secret to skip the key-manager. A curl right
+# after the patch can 403 until the operator re-renders the SecurityPolicy
+# allow-list from the Secret (a few seconds) - retry on 403:
 #   kubectl -n llm-operator-system patch secret openrouter-api-keys --type merge \
 #     -p '{"stringData":{"localtester":"sk-localtest-abc123"}}'
 #   kubectl -n envoy-gateway-system port-forward svc/envoy-...-nebari-gateway 8443:443 &

dev/manifests/test-model.yaml

@@ -23,6 +23,10 @@ spec:
       memory: "256Mi"
   serving:
     image: mock-vllm:dev
+    # The operator defaults the container command to the real vLLM entrypoint
+    # (required by the llm-d-cuda serving image); the mock image runs a plain
+    # Python HTTP server instead, so override it here.
+    command: ["python", "/server.py"]
     replicas: 1
     monitoring:
       enabled: false

dev/mock-vllm/server.py

@@ -1,4 +1,4 @@
-"""Mock vLLM server that responds to health check and model list endpoints."""
+"""Mock vLLM server that responds to health check, model list, and chat endpoints."""
 import json
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
@@ -19,6 +19,32 @@ def do_GET(self):
             self.send_response(404)
             self.end_headers()
 
+    def do_POST(self):
+        # Minimal OpenAI-compatible chat-completion response so the dev stack can
+        # exercise the full request path (gateway auth/routing -> backend -> 200)
+        # without a real model.
+        if self.path in ("/v1/chat/completions", "/v1/completions"):
+            length = int(self.headers.get("Content-Length", 0) or 0)
+            if length:
+                self.rfile.read(length)
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            body = json.dumps({
+                "id": "chatcmpl-mock",
+                "object": "chat.completion",
+                "model": "test/tiny-model",
+                "choices": [{
+                    "index": 0,
+                    "message": {"role": "assistant", "content": "ok"},
+                    "finish_reason": "stop",
+                }],
+            })
+            self.wfile.write(body.encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
     def log_message(self, format, *args):
         pass  # suppress logs
 

dev/test-model-scope.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+# Prove per-model API-key scoping AND that a client cannot spoof the
+# x-llm-client-id authorization header (issues #116 / #122; PR #117 security
+# assessment finding R-04) end-to-end through the real gateway data path.
+#
+# Envoy Gateway authenticates once on the shared external listener (every
+# model's api-keys Secret is pooled into credentialRefs), then each model's
+# HTTPRoute carries a deny-by-default SecurityPolicy whose single Allow rule
+# lists only that model's client IDs, matched on the x-llm-client-id header.
+# The api_key_auth filter populates that header with setReferenceKey (a
+# replace-all "set"), so any client-supplied x-llm-client-id - duplicate,
+# mixed-case, or comma-joined - is discarded before the authorization filter
+# runs. This script locks that behavior in against a gateway/Envoy version bump.
+#
+# It deploys two served mock models (scope-a, scope-b), injects a known API key
+# into each model's api-keys Secret (skipping the key-manager, exactly as
+# local-development.md does), then drives the external gateway with a request
+# matrix. The security-critical assertions are the negative ones (403/401): the
+# gateway returns those from its authorization filter before proxying to any
+# backend, so they hold even if a mock backend is not ready.
+#
+#   key for A            -> model A                    : 200  (positive control)
+#   key for A            -> model B                    : 403  (cross-model denied)
+#   key for A + spoof x-llm-client-id=<B id> -> B      : 403  (R-04: header ignored)
+#   key for A + duplicate spoof header       -> B      : 403  (R-04)
+#   key for A + mixed-case spoof header      -> B      : 403  (R-04)
+#   key for A + comma-joined spoof value     -> B      : 403  (R-04)
+#   key for B            -> model B                    : 200  (positive control)
+#   invalid key          -> model B                    : 401
+#
+# Cleanup deletes the two test models on exit. Set KEEP=1 to leave them.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+CLUSTER_NAME="${CLUSTER_NAME:-llm-serving-test}"
+NS=llm-operator-system
+GW_NS=envoy-gateway-system
+GW_NAME=nebari-gateway
+GW_PORT="${GW_PORT:-8443}"
+HOST="llm.${LLM_BASE_DOMAIN:-local}"
+
+# Pin kubectl to the kind cluster's context so this can never act on whatever
+# the current context happens to be (e.g. a remote/production cluster).
+KUBECTL="kubectl --context kind-${CLUSTER_NAME}"
+
+# Two served mock models. The x-ai-eg-model header the AI Gateway derives from
+# the request body's "model" field is spec.model.name, so that is what we send.
+MODEL_A=scope-a
+MODEL_B=scope-b
+SERVE_A="test/scope-a"
+SERVE_B="test/scope-b"
+
+# Client IDs and keys we inject directly into each model's api-keys Secret. We
+# choose the client IDs so we know B's id to attempt to spoof it from A.
+CID_A=clienta
+CID_B=clientb
+KEY_A="sk-scope-a-$(printf 'a%.0s' {1..24})"
+KEY_B="sk-scope-b-$(printf 'b%.0s' {1..24})"
+
+# --- preflight -------------------------------------------------------------
+if ! $KUBECTL -n "$NS" get deploy/llm-operator >/dev/null 2>&1; then
+  echo "ERROR: operator is not deployed. Run 'make deploy' (or './run-dev.sh') first." >&2
+  exit 1
+fi
+if ! $KUBECTL -n "$GW_NS" get gateway "$GW_NAME" >/dev/null 2>&1; then
+  echo "ERROR: gateway '$GW_NAME' not found. Run 'make setup' first." >&2
+  exit 1
+fi
+
+# --- deploy two served mock models -----------------------------------------
+apply_model() { # $1 = metadata.name  $2 = spec.model.name
+  $KUBECTL apply -f - >/dev/null <<YAML
+apiVersion: llm.nebari.dev/v1alpha1
+kind: LLMModel
+metadata:
+  name: $1
+  namespace: $NS
+spec:
+  model:
+    name: "$2"
+    source: huggingface
+    storage:
+      type: emptyDir
+      size: 1Gi
+    preload: false
+  resources:
+    gpu:
+      count: 0
+      type: nvidia
+    requests: { cpu: "100m", memory: "128Mi" }
+    limits: { cpu: "200m", memory: "256Mi" }
+  serving:
+    image: mock-vllm:dev
+    command: ["python", "/server.py"]
+    replicas: 1
+    monitoring: { enabled: false }
+  access:
+    public: true
+  endpoints:
+    external: { enabled: true, subdomain: $1 }
+    internal: { enabled: true }
+YAML
+}
+
+echo "==> deploying mock models '$MODEL_A' and '$MODEL_B'..."
+# The validating webhook may not be serving the instant the operator rollout
+# returns (it waits on the cert mount); retry so a momentary refusal does not
+# trip set -e.
+for attempt in $(seq 1 30); do
+  if apply_model "$MODEL_A" "$SERVE_A" && apply_model "$MODEL_B" "$SERVE_B"; then break; fi
+  [[ $attempt -eq 30 ]] && { echo "ERROR: operator webhook never became ready" >&2; exit 1; }
+  sleep 2
+done
+
+# --- cleanup ---------------------------------------------------------------
+PF_PID=""; PF_LOG=""
+cleanup() {
+  [[ -n "$PF_PID" ]] && kill "$PF_PID" 2>/dev/null || true
+  [[ -n "$PF_LOG" ]] && rm -f "$PF_LOG" 2>/dev/null || true
+  if [[ "${KEEP:-0}" != "1" ]]; then
+    echo "==> deleting mock models..."
+    $KUBECTL -n "$NS" delete llmmodel "$MODEL_A" "$MODEL_B" --ignore-not-found >/dev/null 2>&1 || true
+  else
+    echo "==> KEEP=1: leaving mock models in place."
+  fi
+}
+trap cleanup EXIT INT TERM
+
+echo "==> waiting for models to report Ready..."
+$KUBECTL -n "$NS" wait llmmodel/"$MODEL_A" --for=jsonpath='{.status.phase}'=Ready --timeout=180s
+$KUBECTL -n "$NS" wait llmmodel/"$MODEL_B" --for=jsonpath='{.status.phase}'=Ready --timeout=180s
+
+# --- inject known API keys into each model's api-keys Secret ----------------
+# The operator creates <model>-api-keys as an empty Opaque Secret; we add a
+# data key (the client ID) whose value is the API key, exactly as the
+# key-manager would. Patching the Secret re-triggers the operator to re-render
+# both models' SecurityPolicies (pooled credentialRefs + per-model allow-list).
+echo "==> injecting API keys (client IDs '$CID_A', '$CID_B')..."
+$KUBECTL -n "$NS" patch secret "${MODEL_A}-api-keys" --type merge \
+  -p "{\"stringData\":{\"${CID_A}\":\"${KEY_A}\"}}" >/dev/null
+$KUBECTL -n "$NS" patch secret "${MODEL_B}-api-keys" --type merge \
+  -p "{\"stringData\":{\"${CID_B}\":\"${KEY_B}\"}}" >/dev/null
+
+# --- port-forward the external gateway --------------------------------------
+SVC="$($KUBECTL -n "$GW_NS" get svc \
+  -l gateway.envoyproxy.io/owning-gateway-name="$GW_NAME" -o name | head -1)"
+if [[ -z "$SVC" ]]; then
+  echo "ERROR: could not find the Envoy service for gateway '$GW_NAME'." >&2
+  exit 1
+fi
+PF_LOG="$(mktemp)"
+echo "==> port-forwarding $SVC :$GW_PORT -> 443..."
+$KUBECTL -n "$GW_NS" port-forward "$SVC" "${GW_PORT}:443" >"$PF_LOG" 2>&1 &
+PF_PID=$!
+for _ in $(seq 1 40); do
+  grep -q "Forwarding from" "$PF_LOG" 2>/dev/null && break
+  sleep 0.5
+done
+
+# --- request helpers --------------------------------------------------------
+GW="https://${HOST}:${GW_PORT}/v1/chat/completions"
+RESOLVE="${HOST}:${GW_PORT}:127.0.0.1"
+
+# req MODEL BEARER [extra curl args...] -> echoes HTTP status code
+req() {
+  local model="$1" bearer="$2"; shift 2
+  curl -sk -o /dev/null -w '%{http_code}' --max-time 10 \
+    --resolve "$RESOLVE" \
+    -H "Authorization: Bearer ${bearer}" \
+    -H "Content-Type: application/json" \
+    "$@" \
+    -d "{\"model\":\"${model}\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}" \
+    "$GW"
+}
+
+FAILED=0
+check() { # name expected actual
+  if [[ "$2" == "$3" ]]; then
+    echo "  PASS: $1 (HTTP $3)"
+  else
+    echo "  FAIL: $1 - expected $2, got $3"
+    FAILED=1
+  fi
+}
+
+# --- wait for the gateway + allow-list to be ready --------------------------
+# The positive control returns 200 only once THREE things converge: the HTTPS
+# listener is Programmed (on a cold cluster this alone can take ~100s), Envoy
+# Gateway has synced the api-keys Secret for authentication, and the operator
+# has re-rendered the model's authorization allow-list. Poll it until 200 (or
+# give up after ~4m and let the assertions report the failure).
+echo "==> waiting for the gateway and SecurityPolicy allow-list to converge..."
+for _ in $(seq 1 120); do
+  [[ "$(req "$SERVE_A" "$KEY_A")" == "200" ]] && break
+  sleep 2
+done
+
+# --- assertions -------------------------------------------------------------
+echo "==> running per-model scope + header-spoofing matrix against $GW"
+check "A's key -> model A (control)"                200 "$(req "$SERVE_A" "$KEY_A")"
+check "B's key -> model B (control)"                200 "$(req "$SERVE_B" "$KEY_B")"
+check "A's key -> model B (cross-model denied)"     403 "$(req "$SERVE_B" "$KEY_A")"
+check "invalid key -> model B"                      401 "$(req "$SERVE_B" "not-a-real-key")"
+
+# R-04: a client-supplied x-llm-client-id must never influence authorization.
+check "spoof: A's key + x-llm-client-id=B -> model B" \
+  403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_B}")"
+check "spoof: A's key + duplicate x-llm-client-id=B -> model B" \
+  403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_B}" -H "x-llm-client-id: ${CID_B}")"
+check "spoof: A's key + mixed-case X-LLM-CLIENT-ID=B -> model B" \
+  403 "$(req "$SERVE_B" "$KEY_A" -H "X-LLM-CLIENT-ID: ${CID_B}")"
+check "spoof: A's key + comma-joined x-llm-client-id=A,B -> model B" \
+  403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_A},${CID_B}")"
+
+echo
+if [[ "$FAILED" -eq 0 ]]; then
+  echo "==> PASS: per-model API-key scoping holds and x-llm-client-id cannot be spoofed."
+else
+  echo "==> FAIL: see assertions above." >&2
+fi
+exit "$FAILED"