diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 497c864..b145cb7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -123,11 +123,8 @@ jobs: - name: Install prerequisites if: steps.list-changed.outputs.changed == 'true' - run: | - pushd chart-dependencies - bash ci-deps.sh - popd - + shell: bash + run: .chart-dependencies/ci-deps.sh - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' diff --git a/.gitignore b/.gitignore index 4292d36..1e52217 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ charts/llm-d/tmp.yaml charts/llm-d/temp/ ct-previous-revision* -chart-dependencies/00-base-crds/crd/ -chart-dependencies/00-base-crds/crds-kgateway chart-dependencies/quickstart/gateway-api-inference-extension/ # misc diff --git a/chart-dependencies/00-base-crds/kustomization.yaml b/chart-dependencies/00-base-crds/kustomization.yaml deleted file mode 100644 index adb7b4e..0000000 --- a/chart-dependencies/00-base-crds/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- https://github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.2.1 -- crd -- crds-kgateway diff --git a/chart-dependencies/ci-deps.sh b/chart-dependencies/ci-deps.sh old mode 100644 new mode 100755 index 747500b..54d2e54 --- a/chart-dependencies/ci-deps.sh +++ b/chart-dependencies/ci-deps.sh @@ -4,46 +4,29 @@ # This is a dependency for the CI job .github/workflows/test.yaml # Prep installation of dependencies for GAIE -set -x +set +x set -e set -o pipefail -if [ -z "$(command -v git)" ] || [ -z "$(command -v kubectl)" ] || [ -z "$(command -v helm)" ]; then - echo "This script depends on \`git\`, \`kubectl\` and \`helm\`. Please install them." +if [ -z "$(command -v kubectl)" ] || [ -z "$(command -v helm)" ]; then + echo "This script depends on \`kubectl\` and \`helm\`. Please install them." exit 1 fi -if [ "chart-dependencies" != "${PWD/*\//}" ]; then - echo "Script must be invoked within this directory" - exit 1 -fi +CWD=$( dirname -- "$( readlink -f -- "$0"; )"; ) ## Populate manifests -#### GAIE manifests - -if [ -d "gateway-api-inference-extension" ]; then # idempotency - rm -rf ./gateway-api-inference-extension -fi - -git clone --filter=blob:none --no-checkout https://github.com/neuralmagic/gateway-api-inference-extension.git -pushd gateway-api-inference-extension - -git sparse-checkout init --cone -git checkout dev - -git sparse-checkout set config/crd - -cp -r config/crd ../00-base-crds - -git sparse-checkout set deploy/components - -cp -r deploy/components/crds-kgateway ../00-base-crds +### Base CRDs +echo -e "\e[32m📜 Applying base CRDs\e[0m" +kubectl apply -k https://github.com/neuralmagic/gateway-api-inference-extension/deploy/components/crds-gateway-api?ref=dev -popd -rm -rf gateway-api-inference-extension +### GAIE CRDs +echo -e "\e[32m🚪 Applying GAIE CRDs\e[0m" +kubectl apply -k https://github.com/neuralmagic/gateway-api-inference-extension/config/crd?ref=dev -#### Apply manifests +### Install Gateway provider +backend=$(helm show values $CWD/../charts/llm-d --jsonpath '{.gateway.gatewayClassName}') -kubectl kustomize 00-base-crds --enable-helm=true | kubectl apply -f - -01-kgateway-control-plane/helm-install.sh +echo -e "\e[32m🎒 Installing Gateway provider:\e[0m '\e[34m$backend\e[0m'" +$CWD/$backend/install.sh diff --git a/chart-dependencies/istio/install.sh b/chart-dependencies/istio/install.sh new file mode 100755 index 0000000..b5760f1 --- /dev/null +++ b/chart-dependencies/istio/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +TAG=1.26-alpha.9befed2f1439d883120f8de70fd70d84ca0ebc3d +HUB=gcr.io/istio-testing + +helm upgrade -i istio-base oci://$HUB/charts/base --version $TAG -n istio-system --create-namespace +helm upgrade -i istiod oci://$HUB/charts/istiod --version $TAG -n istio-system --set tag=$TAG --set hub=$HUB --wait diff --git a/chart-dependencies/01-kgateway-control-plane/helm-install.sh b/chart-dependencies/kgateway/install.sh similarity index 69% rename from chart-dependencies/01-kgateway-control-plane/helm-install.sh rename to chart-dependencies/kgateway/install.sh index e5bc5b3..4552fdd 100755 --- a/chart-dependencies/01-kgateway-control-plane/helm-install.sh +++ b/chart-dependencies/kgateway/install.sh @@ -1,3 +1,11 @@ +#!/bin/bash + +helm upgrade -i \ + --namespace kgateway-system \ + --create-namespace \ + --version v2.0.0 \ + kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds + helm upgrade -i \ --namespace kgateway-system \ --create-namespace \ diff --git a/chart-dependencies/validating-deployment-and-testing.md b/chart-dependencies/validating-deployment-and-testing.md index d85473b..3bfb8eb 100644 --- a/chart-dependencies/validating-deployment-and-testing.md +++ b/chart-dependencies/validating-deployment-and-testing.md @@ -16,6 +16,16 @@ There should be no error logs in the `kgateway` pod. If there is a non-correct ` you will see logs there. This, in combination with there not being a pod from the deployment of the gateway in the namespace in which `llm-d` is deployed. If this part is not situated you will not get a valid pod from the deployment as a child resource of the `gateway`. +### Istio + +```bash +$ oc get deployments -n istio-system +NAME READY UP-TO-DATE AVAILABLE +istiod 1/1 1 1 +``` + +Similar to the kGateway setup, Istio can be verified by looking into the `istio-system` namespace. There should be no error logs in the `istiod` pod. + ### LLM-D namespace #### Deployments diff --git a/charts/llm-d/Chart.yaml b/charts/llm-d/Chart.yaml index 7cbbfaa..dcfb02b 100644 --- a/charts/llm-d/Chart.yaml +++ b/charts/llm-d/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: llm-d type: application -version: 0.1.0 +version: 0.2.0 appVersion: "0.0.1" icon: data:null description: A Helm chart for llm-d diff --git a/charts/llm-d/README.md b/charts/llm-d/README.md index cb4002b..e907ca2 100644 --- a/charts/llm-d/README.md +++ b/charts/llm-d/README.md @@ -1,7 +1,7 @@ # llm-d Helm Chart for OpenShift -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) +![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) A Helm chart for llm-d @@ -146,11 +146,11 @@ Kubernetes: `>= 1.25.0-0` | commonLabels | Labels to add to all deployed objects | object | `{}` | | extraDeploy | Array of extra objects to deploy with the release | list | `[]` | | fullnameOverride | String to fully override common.names.fullname | string | `""` | -| gateway | kGateway configuration | object | `{"annotations":{},"enabled":true,"fullnameOverride":"","gatewayClassName":"kgateway","listeners":[{"name":"default","path":"/","port":80,"protocol":"HTTP"}],"nameOverride":"","parameters":{"proxyUID":false},"serviceType":"NodePort"}` | +| gateway | Gateway configuration | object | `{"annotations":{},"enabled":true,"fullnameOverride":"","gatewayClassName":"kgateway","kGatewayParameters":{"proxyUID":false},"listeners":[{"name":"default","path":"/","port":80,"protocol":"HTTP"}],"nameOverride":"","serviceType":"NodePort"}` | | gateway.annotations | Additional annotations provided to the Gateway resource | object | `{}` | -| gateway.enabled | Deploy resources related to kGateway | bool | `true` | +| gateway.enabled | Deploy resources related to Gateway | bool | `true` | | gateway.fullnameOverride | String to fully override gateway.fullname | string | `""` | -| gateway.gatewayClassName | Gateway class that determines the backend used | string | `"kgateway"` | +| gateway.gatewayClassName | Gateway class that determines the backend used Currently supported values: "kgateway" or "istio" | string | `"kgateway"` | | gateway.nameOverride | String to partially override gateway.fullname | string | `""` | | gateway.serviceType | Gateway's service type. Ingress is only available if the service type is set to NodePort. Accepted values: ["LoadBalancer", "NodePort"] | string | `"NodePort"` | | global | Global parameters Global Docker image parameters Please, note that this will override the image parameters, including dependencies, configured to use the global value Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass | object | See below | diff --git a/charts/llm-d/templates/inference-gateway/gateway.yaml b/charts/llm-d/templates/inference-gateway/gateway.yaml index 7ba616e..a640b62 100644 --- a/charts/llm-d/templates/inference-gateway/gateway.yaml +++ b/charts/llm-d/templates/inference-gateway/gateway.yaml @@ -1,4 +1,5 @@ {{- if .Values.gateway.enabled }} +{{ $isIstio := (eq .Values.gateway.gatewayClassName "istio") }} apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -10,6 +11,9 @@ metadata: {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" . ) | nindent 4 }} {{- end }} + {{- if $isIstio }} + istio.io/enable-inference-extproc: "true" + {{- end }} annotations: {{- if .Values.commonAnnotations }} {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" . ) | nindent 4 }} @@ -17,6 +21,9 @@ metadata: {{- if .Values.gateway.annotations }} {{- include "common.tplvalues.render" ( dict "value" .Values.gateway.annotations "context" .) | nindent 4 }} {{- end }} + {{- if $isIstio }} + networking.istio.io/service-type: ClusterIP + {{- end }} spec: gatewayClassName: {{ .Values.gateway.gatewayClassName | quote }} listeners: @@ -25,7 +32,7 @@ spec: port: {{ .port }} protocol: {{ .protocol }} {{- end }} - {{- if .Values.gateway.parameters.proxyUID }} + {{- if and .Values.gateway.kGatewayParameters.proxyUID (eq .Values.gateway.gatewayClassName "kgateway") }} infrastructure: parametersRef: name: {{ include "gateway.fullname" . }} diff --git a/charts/llm-d/templates/inference-gateway/gatewayparameters.yaml b/charts/llm-d/templates/inference-gateway/gatewayparameters.yaml index 9127d3a..b160c8b 100644 --- a/charts/llm-d/templates/inference-gateway/gatewayparameters.yaml +++ b/charts/llm-d/templates/inference-gateway/gatewayparameters.yaml @@ -1,4 +1,4 @@ -{{ if .Values.gateway.enabled }} +{{ if and .Values.gateway.enabled (eq .Values.gateway.gatewayClassName "kgateway") .Values.gateway.kGatewayParameters.proxyUID }} apiVersion: gateway.kgateway.dev/v1alpha1 kind: GatewayParameters metadata: @@ -23,8 +23,8 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true - {{- if .Values.gateway.parameters.proxyUID }} - runAsUser: {{ .Values.gateway.parameters.proxyUID }} + {{- if .Values.gateway.kGatewayParameters.proxyUID }} + runAsUser: {{ .Values.gateway.kGatewayParameters.proxyUID }} {{- end}} seccompProfile: type: RuntimeDefault @@ -39,8 +39,8 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - {{- if .Values.gateway.parameters.proxyUID }} - runAsUser: {{ .Values.gateway.parameters.proxyUID }} + {{- if .Values.gateway.kGatewayParameters.proxyUID }} + runAsUser: {{ .Values.gateway.kGatewayParameters.proxyUID }} {{- end}} seccompProfile: type: RuntimeDefault diff --git a/charts/llm-d/templates/sample-application/destinationrule.yaml b/charts/llm-d/templates/sample-application/destinationrule.yaml new file mode 100644 index 0000000..161d40f --- /dev/null +++ b/charts/llm-d/templates/sample-application/destinationrule.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.gateway.enabled .Values.sampleApplication.enabled (eq .Values.gateway.gatewayClassName "istio") }} +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + name: {{ include "sampleApplication.sanitizedModelName" . }}-insecure-tls +spec: + host: {{ include "sampleApplication.sanitizedModelName" . }}-epp-service + trafficPolicy: + tls: + mode: SIMPLE + insecureSkipVerify: true +{{- end }} diff --git a/charts/llm-d/values.yaml b/charts/llm-d/values.yaml index 5a0c06e..79ea564 100644 --- a/charts/llm-d/values.yaml +++ b/charts/llm-d/values.yaml @@ -75,10 +75,10 @@ sampleApplication: # -- InferencePool port configuration inferencePoolPort: 8000 -# -- kGateway configuration +# -- Gateway configuration gateway: - # -- Deploy resources related to kGateway + # -- Deploy resources related to Gateway enabled: true # -- String to fully override gateway.fullname @@ -88,13 +88,14 @@ gateway: nameOverride: "" # -- Gateway class that determines the backend used + # Currently supported values: "kgateway" or "istio" gatewayClassName: kgateway # -- Additional annotations provided to the Gateway resource annotations: {} - # Special parameters applied via GatewayParameters resource - parameters: + # Special parameters applied to kGateway via GatewayParameters resource + kGatewayParameters: proxyUID: false