feat: add Istio backend for Gateway

.github/workflows/test.yaml

@@ -123,11 +123,8 @@ jobs:
 
       - name: Install prerequisites
         if: steps.list-changed.outputs.changed == 'true'
-        run: |
-          pushd chart-dependencies
-          bash ci-deps.sh
-          popd
-
+        shell: bash
+        run: .chart-dependencies/ci-deps.sh
 
       - name: Run chart-testing (install)
         if: steps.list-changed.outputs.changed == 'true'

.gitignore

@@ -3,8 +3,6 @@ charts/llm-d/tmp.yaml
 charts/llm-d/temp/
 ct-previous-revision*
 
-chart-dependencies/00-base-crds/crd/
-chart-dependencies/00-base-crds/crds-kgateway
 chart-dependencies/quickstart/gateway-api-inference-extension/
 
 # misc

chart-dependencies/ci-deps.sh

@@ -4,46 +4,29 @@
 # This is a dependency for the CI job .github/workflows/test.yaml
 # Prep installation of dependencies for GAIE
 
-set -x
+set +x
 set -e
 set -o pipefail
 
-if [ -z "$(command -v git)" ] || [ -z "$(command -v kubectl)" ] || [ -z "$(command -v helm)" ]; then
-    echo "This script depends on \`git\`, \`kubectl\` and \`helm\`. Please install them."
+if [ -z "$(command -v kubectl)" ] || [ -z "$(command -v helm)" ]; then
+    echo "This script depends on \`kubectl\` and \`helm\`. Please install them."
     exit 1
 fi
 
-if [ "chart-dependencies" != "${PWD/*\//}" ]; then
-    echo "Script must be invoked within this directory"
-    exit 1
-fi
+CWD=$( dirname -- "$( readlink -f -- "$0"; )"; )
 
 ## Populate manifests
 
-#### GAIE manifests
-
-if [ -d "gateway-api-inference-extension" ]; then # idempotency
-    rm -rf ./gateway-api-inference-extension
-fi
-
-git clone --filter=blob:none --no-checkout https://github.com/neuralmagic/gateway-api-inference-extension.git
-pushd gateway-api-inference-extension
-
-git sparse-checkout init --cone
-git checkout dev
-
-git sparse-checkout set config/crd
-
-cp -r config/crd ../00-base-crds
-
-git sparse-checkout set deploy/components
-
-cp -r deploy/components/crds-kgateway ../00-base-crds
+### Base CRDs
+echo -e "\e[32m📜 Applying base CRDs\e[0m"
+kubectl apply -k https://github.com/neuralmagic/gateway-api-inference-extension/deploy/components/crds-gateway-api?ref=dev
 
-popd
-rm -rf gateway-api-inference-extension
+### GAIE CRDs
+echo -e "\e[32m🚪 Applying GAIE CRDs\e[0m"
+kubectl apply -k https://github.com/neuralmagic/gateway-api-inference-extension/config/crd?ref=dev
 
-#### Apply manifests
+### Install Gateway provider
+backend=$(helm show values $CWD/../charts/llm-d --jsonpath '{.gateway.gatewayClassName}')
 
-kubectl kustomize 00-base-crds --enable-helm=true  | kubectl apply -f -
-01-kgateway-control-plane/helm-install.sh
+echo -e "\e[32m🎒 Installing Gateway provider:\e[0m '\e[34m$backend\e[0m'"
+$CWD/$backend/install.sh

chart-dependencies/istio/install.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+TAG=1.26-alpha.9befed2f1439d883120f8de70fd70d84ca0ebc3d
+HUB=gcr.io/istio-testing
+
+helm upgrade -i istio-base oci://$HUB/charts/base --version $TAG -n istio-system --create-namespace
+helm upgrade -i istiod oci://$HUB/charts/istiod --version $TAG -n istio-system --set tag=$TAG --set hub=$HUB --wait

chart-dependencies/01-kgateway-control-plane/helm-install.sh → chart-dependencies/kgateway/install.sh

@@ -1,3 +1,11 @@
+#!/bin/bash
+
+helm upgrade -i \
+  --namespace kgateway-system \
+  --create-namespace \
+  --version v2.0.0 \
+  kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
+
 helm upgrade -i \
   --namespace kgateway-system \
   --create-namespace \

chart-dependencies/validating-deployment-and-testing.md

@@ -16,6 +16,16 @@ There should be no error logs in the `kgateway` pod. If there is a non-correct `
 you will see logs there. This, in combination with there not being a pod from the deployment of the gateway in the namespace in which
 `llm-d` is deployed. If this part is not situated you will not get a valid pod from the deployment as a child resource of the `gateway`.
 
+### Istio
+
+```bash
+$ oc get deployments -n istio-system
+NAME     READY   UP-TO-DATE   AVAILABLE
+istiod   1/1     1            1
+```
+
+Similar to the kGateway setup, Istio can be verified by looking into the `istio-system` namespace. There should be no error logs in the `istiod` pod.
+
 ### LLM-D namespace
 
 #### Deployments

charts/llm-d/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: llm-d
 type: application
-version: 0.1.0
+version: 0.2.0
 appVersion: "0.0.1"
 icon: data:null
 description: A Helm chart for llm-d

charts/llm-d/README.md

@@ -1,7 +1,7 @@
 
 # llm-d Helm Chart for OpenShift
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square)
+![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square)
 ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 A Helm chart for llm-d
@@ -146,11 +146,11 @@ Kubernetes: `>= 1.25.0-0`
 | commonLabels | Labels to add to all deployed objects | object | `{}` |
 | extraDeploy | Array of extra objects to deploy with the release | list | `[]` |
 | fullnameOverride | String to fully override common.names.fullname | string | `""` |
-| gateway | kGateway configuration | object | `{"annotations":{},"enabled":true,"fullnameOverride":"","gatewayClassName":"kgateway","listeners":[{"name":"default","path":"/","port":80,"protocol":"HTTP"}],"nameOverride":"","parameters":{"proxyUID":false},"serviceType":"NodePort"}` |
+| gateway | Gateway configuration | object | `{"annotations":{},"enabled":true,"fullnameOverride":"","gatewayClassName":"kgateway","kGatewayParameters":{"proxyUID":false},"listeners":[{"name":"default","path":"/","port":80,"protocol":"HTTP"}],"nameOverride":"","serviceType":"NodePort"}` |
 | gateway.annotations | Additional annotations provided to the Gateway resource | object | `{}` |
-| gateway.enabled | Deploy resources related to kGateway | bool | `true` |
+| gateway.enabled | Deploy resources related to Gateway | bool | `true` |
 | gateway.fullnameOverride | String to fully override gateway.fullname | string | `""` |
-| gateway.gatewayClassName | Gateway class that determines the backend used | string | `"kgateway"` |
+| gateway.gatewayClassName | Gateway class that determines the backend used Currently supported values: "kgateway" or "istio" | string | `"kgateway"` |
 | gateway.nameOverride | String to partially override gateway.fullname | string | `""` |
 | gateway.serviceType | Gateway's service type. Ingress is only available if the service type is set to NodePort. Accepted values: ["LoadBalancer", "NodePort"] | string | `"NodePort"` |
 | global | Global parameters Global Docker image parameters Please, note that this will override the image parameters, including dependencies, configured to use the global value Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass | object | See below |

charts/llm-d/templates/inference-gateway/gateway.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.gateway.enabled }}
+{{ $isIstio := (eq .Values.gateway.gatewayClassName "istio") }}
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -10,13 +11,19 @@ metadata:
     {{- if .Values.commonLabels }}
     {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" . ) | nindent 4 }}
     {{- end }}
+    {{- if $isIstio }}
+    istio.io/enable-inference-extproc: "true"
+    {{- end }}
   annotations:
     {{- if .Values.commonAnnotations }}
     {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" . ) | nindent 4 }}
     {{- end }}
     {{- if .Values.gateway.annotations }}
     {{- include "common.tplvalues.render" ( dict "value" .Values.gateway.annotations "context" .) | nindent 4 }}
     {{- end }}
+    {{- if $isIstio }}
+    networking.istio.io/service-type: ClusterIP
+    {{- end }}
 spec:
   gatewayClassName: {{ .Values.gateway.gatewayClassName | quote }}
   listeners:
@@ -25,7 +32,7 @@ spec:
       port: {{ .port }}
       protocol: {{ .protocol }}
   {{- end }}
-  {{- if .Values.gateway.parameters.proxyUID }}
+  {{- if and .Values.gateway.kGatewayParameters.proxyUID  (eq .Values.gateway.gatewayClassName "kgateway") }}
   infrastructure:
     parametersRef:
       name: {{ include "gateway.fullname" . }}

charts/llm-d/templates/inference-gateway/gatewayparameters.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.gateway.enabled }}
+{{ if and .Values.gateway.enabled (eq .Values.gateway.gatewayClassName "kgateway") .Values.gateway.kGatewayParameters.proxyUID }}
 apiVersion: gateway.kgateway.dev/v1alpha1
 kind: GatewayParameters
 metadata:
@@ -23,8 +23,8 @@ spec:
         allowPrivilegeEscalation: false
         readOnlyRootFilesystem: true
         runAsNonRoot: true
-        {{- if .Values.gateway.parameters.proxyUID }}
-        runAsUser: {{ .Values.gateway.parameters.proxyUID }}
+        {{- if .Values.gateway.kGatewayParameters.proxyUID }}
+        runAsUser: {{ .Values.gateway.kGatewayParameters.proxyUID }}
         {{- end}}
         seccompProfile:
           type: RuntimeDefault
@@ -39,8 +39,8 @@ spec:
       securityContext:
         allowPrivilegeEscalation: false
         readOnlyRootFilesystem: true
-        {{- if .Values.gateway.parameters.proxyUID }}
-        runAsUser: {{ .Values.gateway.parameters.proxyUID }}
+        {{- if .Values.gateway.kGatewayParameters.proxyUID }}
+        runAsUser: {{ .Values.gateway.kGatewayParameters.proxyUID }}
         {{- end}}
         seccompProfile:
           type: RuntimeDefault

charts/llm-d/templates/sample-application/destinationrule.yaml

@@ -0,0 +1,12 @@
+{{- if and .Values.gateway.enabled .Values.sampleApplication.enabled (eq .Values.gateway.gatewayClassName "istio") }}
+apiVersion: networking.istio.io/v1
+kind: DestinationRule
+metadata:
+  name: {{ include "sampleApplication.sanitizedModelName" . }}-insecure-tls
+spec:
+  host: {{ include "sampleApplication.sanitizedModelName" . }}-epp-service
+  trafficPolicy:
+    tls:
+      mode: SIMPLE
+      insecureSkipVerify: true
+{{- end }}

charts/llm-d/values.yaml

@@ -75,10 +75,10 @@ sampleApplication:
   # -- InferencePool port configuration
   inferencePoolPort: 8000
 
-# -- kGateway configuration
+# -- Gateway configuration
 gateway:
 
-  # -- Deploy resources related to kGateway
+  # -- Deploy resources related to Gateway
   enabled: true
 
   # --  String to fully override gateway.fullname
@@ -88,13 +88,14 @@ gateway:
   nameOverride: ""
 
   # -- Gateway class that determines the backend used
+  # Currently supported values: "kgateway" or "istio"
   gatewayClassName: kgateway
 
   # -- Additional annotations provided to the Gateway resource
   annotations: {}
 
-  # Special parameters applied via GatewayParameters resource
-  parameters:
+  # Special parameters applied to kGateway via GatewayParameters resource
+  kGatewayParameters:
 
     proxyUID: false