|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Prove per-model API-key scoping AND that a client cannot spoof the |
| 3 | +# x-llm-client-id authorization header (issues #116 / #122; PR #117 security |
| 4 | +# assessment finding R-04) end-to-end through the real gateway data path. |
| 5 | +# |
| 6 | +# Envoy Gateway authenticates once on the shared external listener (every |
| 7 | +# model's api-keys Secret is pooled into credentialRefs), then each model's |
| 8 | +# HTTPRoute carries a deny-by-default SecurityPolicy whose single Allow rule |
| 9 | +# lists only that model's client IDs, matched on the x-llm-client-id header. |
| 10 | +# The api_key_auth filter populates that header with setReferenceKey (a |
| 11 | +# replace-all "set"), so any client-supplied x-llm-client-id - duplicate, |
| 12 | +# mixed-case, or comma-joined - is discarded before the authorization filter |
| 13 | +# runs. This script locks that behavior in against a gateway/Envoy version bump. |
| 14 | +# |
| 15 | +# It deploys two served mock models (scope-a, scope-b), injects a known API key |
| 16 | +# into each model's api-keys Secret (skipping the key-manager, exactly as |
| 17 | +# local-development.md does), then drives the external gateway with a request |
| 18 | +# matrix. The security-critical assertions are the negative ones (403/401): the |
| 19 | +# gateway returns those from its authorization filter before proxying to any |
| 20 | +# backend, so they hold even if a mock backend is not ready. |
| 21 | +# |
| 22 | +# key for A -> model A : 200 (positive control) |
| 23 | +# key for A -> model B : 403 (cross-model denied) |
| 24 | +# key for A + spoof x-llm-client-id=<B id> -> B : 403 (R-04: header ignored) |
| 25 | +# key for A + duplicate spoof header -> B : 403 (R-04) |
| 26 | +# key for A + mixed-case spoof header -> B : 403 (R-04) |
| 27 | +# key for A + comma-joined spoof value -> B : 403 (R-04) |
| 28 | +# key for B -> model B : 200 (positive control) |
| 29 | +# invalid key -> model B : 401 |
| 30 | +# |
| 31 | +# Cleanup deletes the two test models on exit. Set KEEP=1 to leave them. |
| 32 | +set -euo pipefail |
| 33 | + |
| 34 | +cd "$(dirname "$0")" |
| 35 | + |
| 36 | +CLUSTER_NAME="${CLUSTER_NAME:-llm-serving-test}" |
| 37 | +NS=llm-operator-system |
| 38 | +GW_NS=envoy-gateway-system |
| 39 | +GW_NAME=nebari-gateway |
| 40 | +GW_PORT="${GW_PORT:-8443}" |
| 41 | +HOST="llm.${LLM_BASE_DOMAIN:-local}" |
| 42 | + |
| 43 | +# Pin kubectl to the kind cluster's context so this can never act on whatever |
| 44 | +# the current context happens to be (e.g. a remote/production cluster). |
| 45 | +KUBECTL="kubectl --context kind-${CLUSTER_NAME}" |
| 46 | + |
| 47 | +# Two served mock models. The x-ai-eg-model header the AI Gateway derives from |
| 48 | +# the request body's "model" field is spec.model.name, so that is what we send. |
| 49 | +MODEL_A=scope-a |
| 50 | +MODEL_B=scope-b |
| 51 | +SERVE_A="test/scope-a" |
| 52 | +SERVE_B="test/scope-b" |
| 53 | + |
| 54 | +# Client IDs and keys we inject directly into each model's api-keys Secret. We |
| 55 | +# choose the client IDs so we know B's id to attempt to spoof it from A. |
| 56 | +CID_A=clienta |
| 57 | +CID_B=clientb |
| 58 | +KEY_A="sk-scope-a-$(printf 'a%.0s' {1..24})" |
| 59 | +KEY_B="sk-scope-b-$(printf 'b%.0s' {1..24})" |
| 60 | + |
| 61 | +# --- preflight ------------------------------------------------------------- |
| 62 | +if ! $KUBECTL -n "$NS" get deploy/llm-operator >/dev/null 2>&1; then |
| 63 | + echo "ERROR: operator is not deployed. Run 'make deploy' (or './run-dev.sh') first." >&2 |
| 64 | + exit 1 |
| 65 | +fi |
| 66 | +if ! $KUBECTL -n "$GW_NS" get gateway "$GW_NAME" >/dev/null 2>&1; then |
| 67 | + echo "ERROR: gateway '$GW_NAME' not found. Run 'make setup' first." >&2 |
| 68 | + exit 1 |
| 69 | +fi |
| 70 | + |
| 71 | +# --- deploy two served mock models ----------------------------------------- |
| 72 | +apply_model() { # $1 = metadata.name $2 = spec.model.name |
| 73 | + $KUBECTL apply -f - >/dev/null <<YAML |
| 74 | +apiVersion: llm.nebari.dev/v1alpha1 |
| 75 | +kind: LLMModel |
| 76 | +metadata: |
| 77 | + name: $1 |
| 78 | + namespace: $NS |
| 79 | +spec: |
| 80 | + model: |
| 81 | + name: "$2" |
| 82 | + source: huggingface |
| 83 | + storage: |
| 84 | + type: emptyDir |
| 85 | + size: 1Gi |
| 86 | + preload: false |
| 87 | + resources: |
| 88 | + gpu: |
| 89 | + count: 0 |
| 90 | + type: nvidia |
| 91 | + requests: { cpu: "100m", memory: "128Mi" } |
| 92 | + limits: { cpu: "200m", memory: "256Mi" } |
| 93 | + serving: |
| 94 | + image: mock-vllm:dev |
| 95 | + command: ["python", "/server.py"] |
| 96 | + replicas: 1 |
| 97 | + monitoring: { enabled: false } |
| 98 | + access: |
| 99 | + public: true |
| 100 | + endpoints: |
| 101 | + external: { enabled: true, subdomain: $1 } |
| 102 | + internal: { enabled: true } |
| 103 | +YAML |
| 104 | +} |
| 105 | + |
| 106 | +echo "==> deploying mock models '$MODEL_A' and '$MODEL_B'..." |
| 107 | +# The validating webhook may not be serving the instant the operator rollout |
| 108 | +# returns (it waits on the cert mount); retry so a momentary refusal does not |
| 109 | +# trip set -e. |
| 110 | +for attempt in $(seq 1 30); do |
| 111 | + if apply_model "$MODEL_A" "$SERVE_A" && apply_model "$MODEL_B" "$SERVE_B"; then break; fi |
| 112 | + [[ $attempt -eq 30 ]] && { echo "ERROR: operator webhook never became ready" >&2; exit 1; } |
| 113 | + sleep 2 |
| 114 | +done |
| 115 | + |
| 116 | +# --- cleanup --------------------------------------------------------------- |
| 117 | +PF_PID=""; PF_LOG="" |
| 118 | +cleanup() { |
| 119 | + [[ -n "$PF_PID" ]] && kill "$PF_PID" 2>/dev/null || true |
| 120 | + [[ -n "$PF_LOG" ]] && rm -f "$PF_LOG" 2>/dev/null || true |
| 121 | + if [[ "${KEEP:-0}" != "1" ]]; then |
| 122 | + echo "==> deleting mock models..." |
| 123 | + $KUBECTL -n "$NS" delete llmmodel "$MODEL_A" "$MODEL_B" --ignore-not-found >/dev/null 2>&1 || true |
| 124 | + else |
| 125 | + echo "==> KEEP=1: leaving mock models in place." |
| 126 | + fi |
| 127 | +} |
| 128 | +trap cleanup EXIT INT TERM |
| 129 | + |
| 130 | +echo "==> waiting for models to report Ready..." |
| 131 | +$KUBECTL -n "$NS" wait llmmodel/"$MODEL_A" --for=jsonpath='{.status.phase}'=Ready --timeout=180s |
| 132 | +$KUBECTL -n "$NS" wait llmmodel/"$MODEL_B" --for=jsonpath='{.status.phase}'=Ready --timeout=180s |
| 133 | + |
| 134 | +# --- inject known API keys into each model's api-keys Secret ---------------- |
| 135 | +# The operator creates <model>-api-keys as an empty Opaque Secret; we add a |
| 136 | +# data key (the client ID) whose value is the API key, exactly as the |
| 137 | +# key-manager would. Patching the Secret re-triggers the operator to re-render |
| 138 | +# both models' SecurityPolicies (pooled credentialRefs + per-model allow-list). |
| 139 | +echo "==> injecting API keys (client IDs '$CID_A', '$CID_B')..." |
| 140 | +$KUBECTL -n "$NS" patch secret "${MODEL_A}-api-keys" --type merge \ |
| 141 | + -p "{\"stringData\":{\"${CID_A}\":\"${KEY_A}\"}}" >/dev/null |
| 142 | +$KUBECTL -n "$NS" patch secret "${MODEL_B}-api-keys" --type merge \ |
| 143 | + -p "{\"stringData\":{\"${CID_B}\":\"${KEY_B}\"}}" >/dev/null |
| 144 | + |
| 145 | +# --- port-forward the external gateway -------------------------------------- |
| 146 | +SVC="$($KUBECTL -n "$GW_NS" get svc \ |
| 147 | + -l gateway.envoyproxy.io/owning-gateway-name="$GW_NAME" -o name | head -1)" |
| 148 | +if [[ -z "$SVC" ]]; then |
| 149 | + echo "ERROR: could not find the Envoy service for gateway '$GW_NAME'." >&2 |
| 150 | + exit 1 |
| 151 | +fi |
| 152 | +PF_LOG="$(mktemp)" |
| 153 | +echo "==> port-forwarding $SVC :$GW_PORT -> 443..." |
| 154 | +$KUBECTL -n "$GW_NS" port-forward "$SVC" "${GW_PORT}:443" >"$PF_LOG" 2>&1 & |
| 155 | +PF_PID=$! |
| 156 | +for _ in $(seq 1 40); do |
| 157 | + grep -q "Forwarding from" "$PF_LOG" 2>/dev/null && break |
| 158 | + sleep 0.5 |
| 159 | +done |
| 160 | + |
| 161 | +# --- request helpers -------------------------------------------------------- |
| 162 | +GW="https://${HOST}:${GW_PORT}/v1/chat/completions" |
| 163 | +RESOLVE="${HOST}:${GW_PORT}:127.0.0.1" |
| 164 | + |
| 165 | +# req MODEL BEARER [extra curl args...] -> echoes HTTP status code |
| 166 | +req() { |
| 167 | + local model="$1" bearer="$2"; shift 2 |
| 168 | + curl -sk -o /dev/null -w '%{http_code}' --max-time 10 \ |
| 169 | + --resolve "$RESOLVE" \ |
| 170 | + -H "Authorization: Bearer ${bearer}" \ |
| 171 | + -H "Content-Type: application/json" \ |
| 172 | + "$@" \ |
| 173 | + -d "{\"model\":\"${model}\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}" \ |
| 174 | + "$GW" |
| 175 | +} |
| 176 | + |
| 177 | +FAILED=0 |
| 178 | +check() { # name expected actual |
| 179 | + if [[ "$2" == "$3" ]]; then |
| 180 | + echo " PASS: $1 (HTTP $3)" |
| 181 | + else |
| 182 | + echo " FAIL: $1 - expected $2, got $3" |
| 183 | + FAILED=1 |
| 184 | + fi |
| 185 | +} |
| 186 | + |
| 187 | +# --- wait for the gateway + allow-list to be ready -------------------------- |
| 188 | +# The positive control returns 200 only once THREE things converge: the HTTPS |
| 189 | +# listener is Programmed (on a cold cluster this alone can take ~100s), Envoy |
| 190 | +# Gateway has synced the api-keys Secret for authentication, and the operator |
| 191 | +# has re-rendered the model's authorization allow-list. Poll it until 200 (or |
| 192 | +# give up after ~4m and let the assertions report the failure). |
| 193 | +echo "==> waiting for the gateway and SecurityPolicy allow-list to converge..." |
| 194 | +for _ in $(seq 1 120); do |
| 195 | + [[ "$(req "$SERVE_A" "$KEY_A")" == "200" ]] && break |
| 196 | + sleep 2 |
| 197 | +done |
| 198 | + |
| 199 | +# --- assertions ------------------------------------------------------------- |
| 200 | +echo "==> running per-model scope + header-spoofing matrix against $GW" |
| 201 | +check "A's key -> model A (control)" 200 "$(req "$SERVE_A" "$KEY_A")" |
| 202 | +check "B's key -> model B (control)" 200 "$(req "$SERVE_B" "$KEY_B")" |
| 203 | +check "A's key -> model B (cross-model denied)" 403 "$(req "$SERVE_B" "$KEY_A")" |
| 204 | +check "invalid key -> model B" 401 "$(req "$SERVE_B" "not-a-real-key")" |
| 205 | + |
| 206 | +# R-04: a client-supplied x-llm-client-id must never influence authorization. |
| 207 | +check "spoof: A's key + x-llm-client-id=B -> model B" \ |
| 208 | + 403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_B}")" |
| 209 | +check "spoof: A's key + duplicate x-llm-client-id=B -> model B" \ |
| 210 | + 403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_B}" -H "x-llm-client-id: ${CID_B}")" |
| 211 | +check "spoof: A's key + mixed-case X-LLM-CLIENT-ID=B -> model B" \ |
| 212 | + 403 "$(req "$SERVE_B" "$KEY_A" -H "X-LLM-CLIENT-ID: ${CID_B}")" |
| 213 | +check "spoof: A's key + comma-joined x-llm-client-id=A,B -> model B" \ |
| 214 | + 403 "$(req "$SERVE_B" "$KEY_A" -H "x-llm-client-id: ${CID_A},${CID_B}")" |
| 215 | + |
| 216 | +echo |
| 217 | +if [[ "$FAILED" -eq 0 ]]; then |
| 218 | + echo "==> PASS: per-model API-key scoping holds and x-llm-client-id cannot be spoofed." |
| 219 | +else |
| 220 | + echo "==> FAIL: see assertions above." >&2 |
| 221 | +fi |
| 222 | +exit "$FAILED" |
0 commit comments