Skip to content

Commit 65aba6d

Browse files
moulalismoulalis
committed
Merge remote-tracking branch 'upstream/main'
2 parents acbbd0b + 9650bb6 commit 65aba6d

4 files changed

Lines changed: 171 additions & 170 deletions

File tree

‎examples/deploy-demo/common-rhoai.sh‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env bash
2+
# common-rhoai.sh — RHOAI/ODH-specific helpers (DSC-based deployment).
3+
# Sourced by deploy-rhoai.sh and deploy-maas.sh after common.sh.
4+
5+
# ── DSC-based Batch Gateway Deployment ──────────────────────────────────
6+
#
7+
# do_deploy_batch_gateway_dsc isvc_name [pass_through_headers]
8+
#
9+
# Deploys batch-gateway via the LLMBatchGateway CR (operator-managed).
10+
# Installs dependencies (Redis, PostgreSQL, MinIO/PVC), creates the CR,
11+
# waits for it to be ready, and sets up HTTPRoute + DestinationRule.
12+
#
13+
# Args:
14+
# isvc_name Model / InferenceService name (used in the model URL path)
15+
# pass_through_headers Comma-separated list of HTTP headers to forward (optional, empty = none)
16+
17+
do_deploy_batch_gateway_dsc() {
18+
local isvc_name="$1"
19+
local pass_through_headers="${2:-}"
20+
21+
local operator_type="${OPERATOR_TYPE:-rhoai}"
22+
local apps_namespace
23+
case "${operator_type}" in
24+
rhoai) apps_namespace="redhat-ods-applications" ;;
25+
odh) apps_namespace="opendatahub" ;;
26+
*) die "Unknown OPERATOR_TYPE: ${operator_type}" ;;
27+
esac
28+
29+
step "Enabling aigateway + batchGateway in DataScienceCluster..."
30+
kubectl patch datasciencecluster default-dsc --type=merge -p '{"spec":{"components":{"aigateway":{"managementState":"Managed","batchGateway":{"managementState":"Managed"}}}}}'
31+
32+
wait_for_deployment "ai-gateway-operator" "${apps_namespace}" 300s
33+
wait_for_deployment "llm-d-batch-gateway-operator" "${apps_namespace}" 300s
34+
wait_for_crd "llmbatchgateways.batch.llm-d.ai"
35+
36+
kubectl get namespace "${BATCH_NAMESPACE}" &>/dev/null || kubectl create namespace "${BATCH_NAMESPACE}"
37+
kubectl label namespace "${BATCH_NAMESPACE}" llm-d.ai/gateway-route=true --overwrite
38+
39+
install_batch_exchange
40+
install_batch_postgresql
41+
if [ "${BATCH_STORAGE_TYPE}" = "s3" ]; then
42+
install_batch_minio
43+
else
44+
create_batch_pvc
45+
fi
46+
create_batch_secret
47+
48+
local internal_gw_svc
49+
internal_gw_svc=$(kubectl get svc -n "${BATCH_INTERNAL_GATEWAY_NAMESPACE}" \
50+
-l "gateway.networking.k8s.io/gateway-name=${BATCH_INTERNAL_GATEWAY_NAME}" \
51+
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
52+
[ -z "${internal_gw_svc}" ] && die "No service found for Internal Gateway '${BATCH_INTERNAL_GATEWAY_NAME}'."
53+
local model_url="http://${internal_gw_svc}.${BATCH_INTERNAL_GATEWAY_NAMESPACE}.svc.cluster.local/${LLM_NAMESPACE}/${isvc_name}"
54+
log "Model URL (via Internal Gateway): ${model_url}"
55+
56+
local processor_inference_objective_yaml=""
57+
if [ "${ENABLE_FLOW_CONTROL}" = "true" ]; then
58+
processor_inference_objective_yaml=" inferenceObjective: ${BATCH_FLOW_CONTROL_OBJECTIVE}"
59+
log "Flow control: processor will send x-gateway-inference-objective: ${BATCH_FLOW_CONTROL_OBJECTIVE}"
60+
fi
61+
62+
local file_storage_yaml
63+
if [ "${BATCH_STORAGE_TYPE}" = "s3" ]; then
64+
local minio_endpoint="http://${BATCH_MINIO_RELEASE}.${BATCH_NAMESPACE}.svc.cluster.local:9000"
65+
file_storage_yaml=" s3:
66+
region: ${MINIO_REGION}
67+
bucket: ${MINIO_BUCKET}
68+
endpoint: ${minio_endpoint}
69+
accessKeyId: ${MINIO_ROOT_USER}
70+
prefix: ${MINIO_BUCKET}
71+
usePathStyle: true
72+
autoCreateBucket: true"
73+
else
74+
file_storage_yaml=" fs:
75+
basePath: /tmp/batch-gateway
76+
claimName: ${BATCH_FILES_PVC_NAME}"
77+
fi
78+
79+
local apiserver_batch_api_yaml=""
80+
if [ -n "${pass_through_headers}" ]; then
81+
local header_items=""
82+
IFS=',' read -ra _headers <<< "${pass_through_headers}"
83+
for h in "${_headers[@]}"; do
84+
header_items="${header_items}
85+
- ${h}"
86+
done
87+
apiserver_batch_api_yaml=" config:
88+
batchAPI:
89+
passThroughHeaders:${header_items}"
90+
fi
91+
92+
step "Creating LLMBatchGateway CR..."
93+
kubectl apply -f - <<EOF
94+
apiVersion: batch.llm-d.ai/v1alpha1
95+
kind: LLMBatchGateway
96+
metadata:
97+
name: ${BATCH_INSTANCE_NAME}
98+
namespace: ${BATCH_NAMESPACE}
99+
spec:
100+
secretRef:
101+
name: ${BATCH_APP_SECRET_NAME}
102+
dbBackend: ${BATCH_DB_TYPE}
103+
fileStorage:
104+
${file_storage_yaml}
105+
apiServer:
106+
replicas: 1
107+
${apiserver_batch_api_yaml}
108+
processor:
109+
replicas: 1
110+
resources:
111+
requests:
112+
cpu: 100m
113+
memory: 256Mi
114+
limits:
115+
cpu: "1"
116+
memory: 1Gi
117+
globalInferenceGateway:
118+
url: ${model_url}
119+
requestTimeout: ${GW_REQUEST_TIMEOUT}
120+
maxRetries: ${GW_MAX_RETRIES}
121+
initialBackoff: ${GW_INITIAL_BACKOFF}
122+
maxBackoff: ${GW_MAX_BACKOFF}
123+
${processor_inference_objective_yaml}
124+
gc:
125+
interval: 30m
126+
tls:
127+
enabled: true
128+
certManager:
129+
issuerName: ${TLS_ISSUER_NAME}
130+
issuerKind: ClusterIssuer
131+
dnsNames:
132+
- ${BATCH_INSTANCE_NAME}-apiserver
133+
- ${BATCH_INSTANCE_NAME}-apiserver.${BATCH_NAMESPACE}.svc.cluster.local
134+
- localhost
135+
EOF
136+
137+
step "Waiting for LLMBatchGateway to be ready..."
138+
kubectl wait llmbatchgateway/${BATCH_INSTANCE_NAME} -n "${BATCH_NAMESPACE}" \
139+
--for=condition=Ready --timeout=300s
140+
log "LLMBatchGateway is ready."
141+
142+
step "Waiting for AIGateway to be ready..."
143+
kubectl wait aigateway/default-aigateway --for=condition=Ready --timeout=300s
144+
log "AIGateway is ready."
145+
146+
# TODO: Change to die once https://github.com/opendatahub-io/ai-gateway-operator/issues/47 is fixed.
147+
step "Waiting for DataScienceCluster to be ready..."
148+
kubectl wait dsc/default-dsc --for=condition=Ready --timeout=60s \
149+
|| warn "DataScienceCluster is not Ready (known issue: github.com/opendatahub-io/ai-gateway-operator/issues/47)"
150+
151+
create_batch_httproute
152+
create_batch_destinationrule
153+
}

‎examples/deploy-demo/common.sh‎

Lines changed: 0 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -716,157 +716,6 @@ do_deploy_batch_gateway_helm() {
716716
create_batch_destinationrule
717717
}
718718

719-
# ── DSC-based Batch Gateway Deployment ──────────────────────────────────
720-
#
721-
# do_deploy_batch_gateway_dsc isvc_name [pass_through_headers]
722-
#
723-
# Deploys batch-gateway via the LLMBatchGateway CR (operator-managed).
724-
# Installs dependencies (Redis, PostgreSQL, MinIO/PVC), creates the CR,
725-
# waits for it to be ready, and sets up HTTPRoute + DestinationRule.
726-
#
727-
# Args:
728-
# isvc_name Model / InferenceService name (used in the model URL path)
729-
# pass_through_headers Comma-separated list of HTTP headers to forward (optional, empty = none)
730-
731-
do_deploy_batch_gateway_dsc() {
732-
local isvc_name="$1"
733-
local pass_through_headers="${2:-}"
734-
735-
local operator_type="${OPERATOR_TYPE:-rhoai}"
736-
local apps_namespace
737-
case "${operator_type}" in
738-
rhoai) apps_namespace="redhat-ods-applications" ;;
739-
odh) apps_namespace="opendatahub" ;;
740-
*) die "Unknown OPERATOR_TYPE: ${operator_type}" ;;
741-
esac
742-
743-
step "Enabling aigateway + batchGateway in DataScienceCluster..."
744-
kubectl patch datasciencecluster default-dsc --type=merge -p '{"spec":{"components":{"aigateway":{"managementState":"Managed","batchGateway":{"managementState":"Managed"}}}}}'
745-
746-
wait_for_deployment "ai-gateway-operator" "${apps_namespace}" 300s
747-
wait_for_deployment "llm-d-batch-gateway-operator" "${apps_namespace}" 300s
748-
wait_for_crd "llmbatchgateways.batch.llm-d.ai"
749-
750-
kubectl get namespace "${BATCH_NAMESPACE}" &>/dev/null || kubectl create namespace "${BATCH_NAMESPACE}"
751-
kubectl label namespace "${BATCH_NAMESPACE}" llm-d.ai/gateway-route=true --overwrite
752-
753-
install_batch_exchange
754-
install_batch_postgresql
755-
if [ "${BATCH_STORAGE_TYPE}" = "s3" ]; then
756-
install_batch_minio
757-
else
758-
create_batch_pvc
759-
fi
760-
create_batch_secret
761-
762-
local internal_gw_svc
763-
internal_gw_svc=$(kubectl get svc -n "${BATCH_INTERNAL_GATEWAY_NAMESPACE}" \
764-
-l "gateway.networking.k8s.io/gateway-name=${BATCH_INTERNAL_GATEWAY_NAME}" \
765-
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
766-
[ -z "${internal_gw_svc}" ] && die "No service found for Internal Gateway '${BATCH_INTERNAL_GATEWAY_NAME}'."
767-
local model_url="http://${internal_gw_svc}.${BATCH_INTERNAL_GATEWAY_NAMESPACE}.svc.cluster.local/${LLM_NAMESPACE}/${isvc_name}"
768-
log "Model URL (via Internal Gateway): ${model_url}"
769-
770-
local processor_config_yaml=""
771-
if [ "${ENABLE_FLOW_CONTROL}" = "true" ]; then
772-
processor_config_yaml=" config:
773-
inferenceObjective: ${BATCH_FLOW_CONTROL_OBJECTIVE}"
774-
log "Flow control: processor will send x-gateway-inference-objective: ${BATCH_FLOW_CONTROL_OBJECTIVE}"
775-
fi
776-
777-
local file_storage_yaml
778-
if [ "${BATCH_STORAGE_TYPE}" = "s3" ]; then
779-
local minio_endpoint="http://${BATCH_MINIO_RELEASE}.${BATCH_NAMESPACE}.svc.cluster.local:9000"
780-
file_storage_yaml=" s3:
781-
region: ${MINIO_REGION}
782-
bucket: ${MINIO_BUCKET}
783-
endpoint: ${minio_endpoint}
784-
accessKeyId: ${MINIO_ROOT_USER}
785-
prefix: ${MINIO_BUCKET}
786-
usePathStyle: true
787-
autoCreateBucket: true"
788-
else
789-
file_storage_yaml=" fs:
790-
basePath: /tmp/batch-gateway
791-
claimName: ${BATCH_FILES_PVC_NAME}"
792-
fi
793-
794-
local apiserver_batch_api_yaml=""
795-
if [ -n "${pass_through_headers}" ]; then
796-
local header_items=""
797-
IFS=',' read -ra _headers <<< "${pass_through_headers}"
798-
for h in "${_headers[@]}"; do
799-
header_items="${header_items}
800-
- ${h}"
801-
done
802-
apiserver_batch_api_yaml=" config:
803-
batchAPI:
804-
passThroughHeaders:${header_items}"
805-
fi
806-
807-
step "Creating LLMBatchGateway CR..."
808-
kubectl apply -f - <<EOF
809-
apiVersion: batch.llm-d.ai/v1alpha1
810-
kind: LLMBatchGateway
811-
metadata:
812-
name: ${BATCH_INSTANCE_NAME}
813-
namespace: ${BATCH_NAMESPACE}
814-
spec:
815-
secretRef:
816-
name: ${BATCH_APP_SECRET_NAME}
817-
dbBackend: ${BATCH_DB_TYPE}
818-
fileStorage:
819-
${file_storage_yaml}
820-
apiServer:
821-
replicas: 1
822-
${apiserver_batch_api_yaml}
823-
processor:
824-
replicas: 1
825-
resources:
826-
requests:
827-
cpu: 100m
828-
memory: 256Mi
829-
limits:
830-
cpu: "1"
831-
memory: 1Gi
832-
globalInferenceGateway:
833-
url: ${model_url}
834-
requestTimeout: ${GW_REQUEST_TIMEOUT}
835-
maxRetries: ${GW_MAX_RETRIES}
836-
initialBackoff: ${GW_INITIAL_BACKOFF}
837-
maxBackoff: ${GW_MAX_BACKOFF}
838-
${processor_config_yaml}
839-
gc:
840-
interval: 30m
841-
tls:
842-
enabled: true
843-
certManager:
844-
issuerName: ${TLS_ISSUER_NAME}
845-
issuerKind: ClusterIssuer
846-
dnsNames:
847-
- ${BATCH_INSTANCE_NAME}-apiserver
848-
- ${BATCH_INSTANCE_NAME}-apiserver.${BATCH_NAMESPACE}.svc.cluster.local
849-
- localhost
850-
EOF
851-
852-
step "Waiting for LLMBatchGateway to be ready..."
853-
kubectl wait llmbatchgateway/${BATCH_INSTANCE_NAME} -n "${BATCH_NAMESPACE}" \
854-
--for=condition=Ready --timeout=300s
855-
log "LLMBatchGateway is ready."
856-
857-
step "Waiting for AIGateway to be ready..."
858-
kubectl wait aigateway/default-aigateway --for=condition=Ready --timeout=300s
859-
log "AIGateway is ready."
860-
861-
# TODO: Change to die once https://github.com/opendatahub-io/ai-gateway-operator/issues/47 is fixed.
862-
step "Waiting for DataScienceCluster to be ready..."
863-
kubectl wait dsc/default-dsc --for=condition=Ready --timeout=60s \
864-
|| warn "DataScienceCluster is not Ready (known issue: github.com/opendatahub-io/ai-gateway-operator/issues/47)"
865-
866-
create_batch_httproute
867-
create_batch_destinationrule
868-
}
869-
870719
# ── Internal Gateway ──────────────────────────────────────────────────────────
871720
#
872721
# Creates a ClusterIP-only Gateway for the batch processor to access LLM

‎examples/deploy-demo/deploy-maas.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ set -euo pipefail
1717
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1818

1919
source "${SCRIPT_DIR}/common.sh"
20+
source "${SCRIPT_DIR}/common-rhoai.sh"
2021

2122
# ── Configuration (set before sourcing common.sh to override its defaults) ──
2223
GATEWAY_NAME="${GATEWAY_NAME:-maas-default-gateway}"

0 commit comments

Comments
 (0)