Merge remote-tracking branch 'upstream/main'

Author: moulalis

docs/guides/deploy-k8s.md

@@ -6,7 +6,7 @@ One-click deployment script for batch-gateway on Kubernetes. Supports `install`,
 
 **Prerequisites**:
 
-- **Tools**: `kubectl`, `helm`, `helmfile`, `git`, `curl`, `jq`, `yq`, and the [`helm-diff`](https://github.com/databus23/helm-diff) plugin.
+- **Tools**: `kubectl`, `helm`, `git`, `curl`, `jq`, `yq`.
 - **Cluster access**: You must be logged in to the target cluster. Use `kubectl config current-context` (or `oc whoami` on OpenShift) to verify.
 
 ## Usage
@@ -23,7 +23,7 @@ bash examples/deploy-demo/deploy-k8s.sh install
 |-----------|---------|
 | cert-manager | TLS certificate management |
 | Istio | Service mesh + ingress gateway (HTTPS:443) |
-| llm-d stack | GAIE InferencePool + vllm-sim (single model, default: random) |
+| llm-d stack | llm-d Router (gateway mode) + vllm-sim (single model, default: random) |
 | Kuadrant | Auth + rate limiting (installed via Helm) |
 | Redis | Exchange backend (Bitnami Helm chart, configurable via `BATCH_EXCHANGE_CLIENT_TYPE`) |
 | PostgreSQL | Batch metadata store (Bitnami Helm chart) |
@@ -140,13 +140,13 @@ Use that only on **ephemeral or dedicated** demo clusters. See [issue #309](http
 | `GATEWAY_LOCAL_PORT` | `8080` | Port-forward local port |
 | `BATCH_INTERNAL_GATEWAY_NAME` | `batch-internal-gateway` | Internal Gateway resource name |
 | `BATCH_INTERNAL_GATEWAY_NAMESPACE` | `${GATEWAY_NAMESPACE}` | Internal Gateway namespace |
-| `LLMD_VERSION` | `v0.7.0` | llm-d git ref to install |
+| `LLMD_VERSION` | `v0.8.1` | llm-d git ref to install |
 | `LLMD_RELEASE_POSTFIX` | `llmd` | Helm release postfix |
 | `MODEL_NAME` | `random` | Model name for routing |
-| `CERT_MANAGER_VERSION` | `v1.15.3` | cert-manager Helm chart version |
+| `CERT_MANAGER_VERSION` | `v1.20.3` | cert-manager Helm chart version |
 | `KUADRANT_VERSION` | `1.3.1` | Kuadrant Helm chart version |
-| `GAIE_CHART_VERSION` | `v1.5.0` | InferencePool (GAIE) Helm chart version |
-| `MODELSERVICE_CHART_VERSION` | `v0.4.12` | ModelService (vllm-sim) Helm chart version |
+| `ROUTER_CHART_VERSION` | (from llm-d env.sh) | llm-d Router chart/CRD version (auto-detected from LLMD_VERSION) |
+| `ISTIO_VERSION` | `1.29.2` | Istio Helm chart version |
 | `ENABLE_FLOW_CONTROL` | `true` | Enable GIE priority-based flow control |
 | `BATCH_FLOW_CONTROL_OBJECTIVE` | `batch-sheddable` | InferenceObjective name for batch requests (priority -1) |
 | `UNINSTALL_ALL` | `0` | Set to `1` to remove Kuadrant, Istio, cert-manager, CRDs (ephemeral clusters only) |

examples/deploy-demo/deploy-k8s.md

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llmd-sim-decode
+  labels:
+    llm-d.ai/role: decode
+    llm-d.ai/inference-serving: "true"
+    llm-d.ai/guide: "simulated-accelerators"
+    llm-d.ai/accelerator-variant: "cpu"
+    llm-d.ai/model: "random"
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      llm-d.ai/role: decode
+      llm-d.ai/inference-serving: "true"
+      llm-d.ai/guide: "simulated-accelerators"
+      llm-d.ai/accelerator-variant: "cpu"
+      llm-d.ai/model: "random"
+  template:
+    metadata:
+      labels:
+        llm-d.ai/role: decode
+        llm-d.ai/inference-serving: "true"
+        llm-d.ai/guide: "simulated-accelerators"
+        llm-d.ai/accelerator-variant: "cpu"
+        llm-d.ai/model: "random"
+    spec:
+      serviceAccountName: llmd-sim-sa
+      containers:
+        - name: modelserver
+          image: ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
+          args:
+            - "--model"
+            - "random"
+            - "--served-model-name"
+            - "random"
+          ports:
+            - name: modelserver
+              containerPort: 8000
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: modelserver
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /v1/models
+              port: modelserver
+            periodSeconds: 5
+            timeoutSeconds: 2
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /v1/models
+              port: modelserver
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 30

examples/deploy-demo/deploy-k8s.sh

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - decode-deployment.yaml
+  - sa.yaml

examples/deploy-demo/llmd-sim/gaie-sim-values.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: llmd-sim-sa
+automountServiceAccountToken: false

examples/deploy-demo/llmd-sim/modelserver/decode-deployment.yaml

@@ -0,0 +1,44 @@
+# Flow control overlay for llm-d Router chart.
+# Use with: helm install ... -f sim-values.yaml -f overlays/flow-control.yaml
+#
+# Enables flow control with two priority bands:
+#   - Interactive (priority 100): low-latency, round-robin fairness, FCFS ordering
+#   - Batch (priority -1): sheddable, throughput-optimized, SLO-deadline ordering
+#
+# See docs/guides/flow-control-setup.md for details.
+
+router:
+  epp:
+    pluginsConfigFile: "flow-control-plugins.yaml"
+    pluginsCustomConfig:
+      flow-control-plugins.yaml: |
+        apiVersion: llm-d.ai/v1alpha1
+        kind: EndpointPickerConfig
+        featureGates:
+          - flowControl
+        plugins:
+          - type: round-robin-fairness-policy
+          - type: global-strict-fairness-policy
+          - type: slo-deadline-ordering-policy
+          - type: utilization-detector
+            parameters:
+              queueDepthThreshold: 5
+              kvCacheUtilThreshold: 0.8
+        flowControl:
+          maxBytes: 4294967296
+          defaultRequestTTL: 30s
+          priorityBands:
+            - priority: 100
+              maxBytes: 1073741824
+              fairnessPolicyRef: round-robin-fairness-policy
+              orderingPolicyRef: fcfs-ordering-policy
+            - priority: -1
+              maxBytes: 3221225472
+              fairnessPolicyRef: global-strict-fairness-policy
+              orderingPolicyRef: slo-deadline-ordering-policy
+          defaultPriorityBand:
+            maxBytes: 536870912
+            fairnessPolicyRef: global-strict-fairness-policy
+            orderingPolicyRef: fcfs-ordering-policy
+        saturationDetector:
+          pluginRef: utilization-detector