fix(dev): harden local dev stack for Helm 4 and the webhook startup race

Addresses review feedback on the local dev path:
- Gateway API CRDs: apply with --server-side and pass --force-conflicts to
  the eg chart install, gated on Helm major version (Helm 4 needs it; Helm 3
  has no server-side apply, skips present crds/, and rejects the flag). Fixes
  the CRD ownership conflict that blocked `make setup` on Helm 4.
- make setup: guard `kind create cluster` so a partial setup is re-runnable.
- operator.yaml: add a readinessProbe on the webhook port (9443) so the
  `make deploy` rollout waits until the webhook is serving.
- run-dev.sh: bounded retry around the webhook-gated PassthroughModel apply
  (covers the residual endpoint-propagation window), and a per-run mktemp
  file for the port-forward log.

Author: dcmcand

dev/Makefile

@@ -10,6 +10,14 @@ GIE_VERSION ?= v1.4.0
 ENVOY_GATEWAY_VERSION ?= v1.6.7
 AI_GATEWAY_VERSION ?= v0.5.0
 
+# Helm 4 applies the eg chart's bundled Gateway API CRDs (crds/) server-side,
+# which conflicts with the standalone install in `setup` (owned by a different
+# field manager). Force ownership so the chart wins. Helm 3 has no server-side
+# apply and skips already-present crds/, so it neither needs nor accepts this
+# flag - keep it empty there. Empty when helm is missing (setup fails later).
+HELM_MAJOR := $(shell helm version --template '{{.Version}}' 2>/dev/null | sed -E 's/^v?([0-9]+).*/\1/')
+HELM_FORCE_CONFLICTS := $(if $(filter-out 1 2 3,$(HELM_MAJOR)),--force-conflicts,)
+
 .PHONY: setup teardown build-images load-images deploy deploy-operator deploy-key-manager apply-test-model apply-passthrough-model create-openrouter-secret run-dev ui logs-operator logs-key-manager clean help
 
 help: ## Show this help
@@ -19,12 +27,16 @@ run-dev: ## One-command UI dev environment: cluster + models + port-forward + ho
 	./run-dev.sh
 
 setup: ## Create kind cluster and install dependencies
-	kind create cluster --name $(CLUSTER_NAME)
+	# Idempotent: skip creation if the cluster already exists so a setup that
+	# died midway can be re-run without `make teardown` first.
+	@kind get clusters | grep -qx $(CLUSTER_NAME) || kind create cluster --name $(CLUSTER_NAME)
 	# cert-manager
 	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
 	kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
-	# Gateway API CRDs
-	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEWAY_API_VERSION)/standard-install.yaml
+	# Gateway API CRDs. Server-side apply so the field ownership matches the
+	# eg chart's own SSA on Helm 4, letting it cleanly take co-ownership with
+	# --force-conflicts (see HELM_FORCE_CONFLICTS above) instead of erroring.
+	kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEWAY_API_VERSION)/standard-install.yaml
 	# GIE CRDs (includes the graduated inference.networking.k8s.io group)
 	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/$(GIE_VERSION)/manifests.yaml
 	# Envoy AI Gateway first: envoy-gateway's extensionManager below points at
@@ -36,7 +48,7 @@ setup: ## Create kind cluster and install dependencies
 	# Envoy Gateway, wired with the AI Gateway ext_proc extension (enableBackend,
 	# extensionManager, backendResources). Without this the per-model routing
 	# layer 404s and passthrough upstreams never get a TLS transport socket.
-	helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm --version $(ENVOY_GATEWAY_VERSION) -n envoy-gateway-system --create-namespace -f eg-extension-values.yaml
+	helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm --version $(ENVOY_GATEWAY_VERSION) -n envoy-gateway-system --create-namespace $(HELM_FORCE_CONFLICTS) -f eg-extension-values.yaml
 	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
 	# CRDs (served LLMModels and external-provider PassthroughModels)
 	kubectl apply -f ../charts/nebari-llm-serving/crds/llmmodel-crd.yaml

dev/manifests/operator.yaml

@@ -147,6 +147,16 @@ spec:
             - containerPort: 9443
               name: webhook-server
               protocol: TCP
+          # The validating webhook binds 9443 only after controller-runtime
+          # loads the mounted serving cert. Gate readiness on that port so
+          # `kubectl rollout status` in `make deploy` does not return before the
+          # webhook can accept PassthroughModel creates (the manager's /readyz
+          # is only a ping and goes green before the webhook is serving).
+          readinessProbe:
+            tcpSocket:
+              port: 9443
+            initialDelaySeconds: 2
+            periodSeconds: 2
           volumeMounts:
             - name: cert
               mountPath: /tmp/k8s-webhook-server/serving-certs

dev/run-dev.sh

@@ -45,7 +45,20 @@ echo "==> applying OpenRouter credential and dev models..."
 kubectl -n "$NS" create secret generic openrouter-api-key \
   --from-literal=apiKey="$OPENROUTER_API_KEY" \
   --dry-run=client -o yaml | kubectl apply -f - >/dev/null
-kubectl apply -f manifests/dev-models.yaml >/dev/null
+# The operator's validating webhook gates PassthroughModel creates and isn't
+# guaranteed to be serving the instant `make deploy`'s rollout returns (it waits
+# on the cert mount). Retry so a momentary "connection refused" doesn't trip
+# `set -e` and abort the whole run.
+for attempt in $(seq 1 30); do
+  kubectl apply -f manifests/dev-models.yaml >/dev/null 2>&1 && break
+  if [[ $attempt -eq 30 ]]; then
+    echo "ERROR: operator webhook never became ready" >&2
+    kubectl apply -f manifests/dev-models.yaml >&2 || true
+    exit 1
+  fi
+  echo "==> operator webhook not ready yet, retrying ($attempt)..."
+  sleep 2
+done
 for m in claude-sonnet-45 gemini-25-flash llama-33-70b; do
   kubectl -n "$NS" wait passthroughmodel/$m --for=jsonpath='{.status.phase}'=Ready --timeout=90s
 done
@@ -55,14 +68,18 @@ cleanup() {
   echo
   echo "==> shutting down..."
   [[ -n "${PF_PID:-}" ]] && kill "$PF_PID" 2>/dev/null || true
+  [[ -n "${PF_LOG:-}" ]] && rm -f "$PF_LOG"
 }
 trap cleanup EXIT INT TERM
 
+# Fresh temp file per run: a fixed path could carry "Forwarding from" from a
+# crashed prior run and make the readiness check below pass instantly.
+PF_LOG="$(mktemp)"
 echo "==> port-forwarding key-manager to localhost:${KM_PORT}..."
-kubectl -n "$NS" port-forward svc/llm-key-manager "${KM_PORT}:8080" >/tmp/km-portforward.log 2>&1 &
+kubectl -n "$NS" port-forward svc/llm-key-manager "${KM_PORT}:8080" >"$PF_LOG" 2>&1 &
 PF_PID=$!
 for _ in $(seq 1 20); do
-  if grep -q "Forwarding from" /tmp/km-portforward.log 2>/dev/null; then break; fi
+  if grep -q "Forwarding from" "$PF_LOG" 2>/dev/null; then break; fi
   sleep 0.5
 done