44# Environment variables:
55# RELEASE_NAME - Helm release name (default: rhai-on-xks)
66# NAMESPACE - Helm release namespace (default: rhai-on-xks)
7- # CLOUD_PROVIDER - Cloud provider: azure or coreweave (default: azure)
7+ # CLOUD_PROVIDER - Cloud provider: azure, coreweave, or aws (default: azure)
88# TIMEOUT - Max wait time in seconds per check (default: 300)
99
1010set -euo pipefail
@@ -19,12 +19,51 @@ if ! [[ "$TIMEOUT" =~ ^[0-9]+$ ]]; then
1919fi
2020INTERVAL=10
2121
22+ declare -A PROVIDER_CRDS=(
23+ [azure]=" azurekubernetesengines.infrastructure.opendatahub.io"
24+ [coreweave]=" coreweavekubernetesengines.infrastructure.opendatahub.io"
25+ [aws]=" awskubernetesengines.infrastructure.opendatahub.io"
26+ )
27+
28+ declare -A PROVIDER_CR_DISPLAY=(
29+ [azure]=" AzureKubernetesEngine"
30+ [coreweave]=" CoreWeaveKubernetesEngine"
31+ [aws]=" AWSKubernetesEngine"
32+ )
33+
34+ if [[ -z " ${PROVIDER_CRDS[$CLOUD_PROVIDER]+_} " ; then
35+ echo " ERROR: unsupported CLOUD_PROVIDER '${CLOUD_PROVIDER} '. Valid values: ${! PROVIDER_CRDS[*]} " >&2
36+ exit 1
37+ fi
38+
2239ERRORS=0
2340ERROR_MESSAGES=()
2441
2542log_ok () { echo " OK: $1 " ; }
2643log_fail () { echo " FAIL: $1 " ; ERRORS=$(( ERRORS + 1 )) ; ERROR_MESSAGES+=(" $1 " ; }
2744
45+ debug_namespace () {
46+ local ns=" $1 "
47+ local filter=" ${2:- } "
48+ local filter_args=()
49+ if [ -n " ${filter} " ; then
50+ filter_args=(-l " ${filter} "
51+ fi
52+ echo " DEBUG: pods in '${ns} '${filter: + (filter: ${filter} )} :"
53+ kubectl get pods -n " ${ns} " " ${filter_args[@]} " 2> /dev/null || true
54+ for pod in $( kubectl get pods -n " ${ns} " " ${filter_args[@]} " ' {.items[*].metadata.name}' 2> /dev/null) ; do
55+ local phase
56+ phase=$( kubectl get pod " ${pod} " " ${ns} " ' {.status.phase}' 2> /dev/null)
57+ echo " --- pod: ${pod} (${phase} ) ---"
58+ if [ " ${phase} " != " Running" && [ " ${phase} " != " Succeeded" ; then
59+ echo " DEBUG: describe pod ${pod} :"
60+ kubectl describe pod " ${pod} " " ${ns} " 2> /dev/null || true
61+ else
62+ kubectl logs " ${pod} " " ${ns} " 2> /dev/null || true
63+ fi
64+ done
65+ }
66+
2867wait_for () {
2968 local description=" $1 "
3069 shift
129168
130169# --- Namespaces ---
131170echo " --- Namespaces ---"
132- EXPECTED_NAMESPACES=(" redhat-ods-operator" " redhat-ods-applications" " ${NAMESPACE} "
133- if [ " ${CLOUD_PROVIDER} " = " azure" || [ " ${CLOUD_PROVIDER} " = " coreweave" ; then
134- EXPECTED_NAMESPACES+=(" rhai-cloudmanager-system"
135- fi
171+ EXPECTED_NAMESPACES=(" redhat-ods-operator" " redhat-ods-applications" " ${NAMESPACE} " " rhai-cloudmanager-system"
136172
137173for ns in " ${EXPECTED_NAMESPACES[@]} " ; do
138174 if kubectl get namespace " ${ns} " > /dev/null 2>&1 ; then
@@ -150,55 +186,44 @@ else
150186 log_fail " CRD 'kserves.components.platform.opendatahub.io' not found"
151187fi
152188
153- if [ " ${CLOUD_PROVIDER} " = " azure" ; then
154- if kubectl get crd azurekubernetesengines.infrastructure.opendatahub.io > /dev/null 2>&1 ; then
155- log_ok " CRD 'azurekubernetesengines.infrastructure.opendatahub.io' exists"
156- else
157- log_fail " CRD 'azurekubernetesengines.infrastructure.opendatahub.io' not found"
158- fi
159- if kubectl get crd coreweavekubernetesengines.infrastructure.opendatahub.io > /dev/null 2>&1 ; then
160- log_fail " CRD 'coreweavekubernetesengines.infrastructure.opendatahub.io' should NOT exist for azure provider"
161- else
162- log_ok " CRD 'coreweavekubernetesengines.infrastructure.opendatahub.io' correctly absent"
163- fi
164- elif [ " ${CLOUD_PROVIDER} " = " coreweave" ; then
165- if kubectl get crd coreweavekubernetesengines.infrastructure.opendatahub.io > /dev/null 2>&1 ; then
166- log_ok " CRD 'coreweavekubernetesengines.infrastructure.opendatahub.io' exists"
167- else
168- log_fail " CRD 'coreweavekubernetesengines.infrastructure.opendatahub.io' not found"
169- fi
170- if kubectl get crd azurekubernetesengines.infrastructure.opendatahub.io > /dev/null 2>&1 ; then
171- log_fail " CRD 'azurekubernetesengines.infrastructure.opendatahub.io' should NOT exist for coreweave provider"
189+ for provider in " ${! PROVIDER_CRDS[@]} " ; do
190+ crd=" ${PROVIDER_CRDS[$provider]} "
191+ if [ " ${CLOUD_PROVIDER} " = " ${provider} " ; then
192+ if kubectl get crd " ${crd} " > /dev/null 2>&1 ; then
193+ log_ok " CRD '${crd} ' exists"
194+ else
195+ log_fail " CRD '${crd} ' not found"
196+ fi
172197 else
173- log_ok " CRD 'azurekubernetesengines.infrastructure.opendatahub.io' correctly absent"
198+ if kubectl get crd " ${crd} " > /dev/null 2>&1 ; then
199+ log_fail " CRD '${crd} ' should NOT exist for ${CLOUD_PROVIDER} provider"
200+ else
201+ log_ok " CRD '${crd} ' correctly absent"
202+ fi
174203 fi
175- fi
204+ done
176205
177206# --- Step 1: Cloud manager deployment ---
178207echo " --- Cloud Manager ---"
179208CM_NS=" rhai-cloudmanager-system"
180- if [ " ${CLOUD_PROVIDER} " = " azure" ; then
181- wait_for_deployment " azure-cloud-manager-operator" " ${CM_NS} "
182- if kubectl get deployment coreweave-cloud-manager-operator -n " ${CM_NS} " > /dev/null 2>&1 ; then
183- log_fail " Deployment 'coreweave-cloud-manager-operator' should NOT exist for azure provider"
184- else
185- log_ok " Deployment 'coreweave-cloud-manager-operator' correctly absent"
186- fi
187- elif [ " ${CLOUD_PROVIDER} " = " coreweave" ; then
188- wait_for_deployment " coreweave-cloud-manager-operator" " ${CM_NS} "
189- if kubectl get deployment azure-cloud-manager-operator -n " ${CM_NS} " > /dev/null 2>&1 ; then
190- log_fail " Deployment 'azure-cloud-manager-operator' should NOT exist for coreweave provider"
209+ for provider in " ${! PROVIDER_CRDS[@]} " ; do
210+ deploy=" ${provider} -cloud-manager-operator"
211+ if [ " ${CLOUD_PROVIDER} " = " ${provider} " ; then
212+ wait_for_deployment " ${deploy} " " ${CM_NS} "
191213 else
192- log_ok " Deployment 'azure-cloud-manager-operator' correctly absent"
214+ if kubectl get deployment " ${deploy} " " ${CM_NS} " > /dev/null 2>&1 ; then
215+ log_fail " Deployment '${deploy} ' should NOT exist for ${CLOUD_PROVIDER} provider"
216+ else
217+ log_ok " Deployment '${deploy} ' correctly absent"
218+ fi
193219 fi
194- fi
220+ done
195221
196222# --- Step 2: Cloud provider CR status ---
197223echo " --- Cloud Provider CR ---"
198- if [ " ${CLOUD_PROVIDER} " = " azure" ; then
199- wait_for_cr_ready " azurekubernetesengines.infrastructure.opendatahub.io" " default-azurekubernetesengine" " AzureKubernetesEngine 'default-azurekubernetesengine'"
200- elif [ " ${CLOUD_PROVIDER} " = " coreweave" ; then
201- wait_for_cr_ready " coreweavekubernetesengines.infrastructure.opendatahub.io" " default-coreweavekubernetesengine" " CoreWeaveKubernetesEngine 'default-coreweavekubernetesengine'"
224+ cr_name=" default-${CLOUD_PROVIDER} kubernetesengine"
225+ if ! wait_for_cr_ready " ${PROVIDER_CRDS[$CLOUD_PROVIDER]} " " ${cr_name} " " ${PROVIDER_CR_DISPLAY[$CLOUD_PROVIDER]} '${cr_name} '" ; then
226+ debug_namespace " ${CM_NS} "
202227fi
203228
204229# --- Step 3: cert-manager deployments ---
@@ -211,7 +236,10 @@ wait_for_deployment "rhai-operator" "redhat-ods-operator" 3
211236
212237# --- Step 5: KServe component CR status ---
213238echo " --- KServe Component ---"
214- wait_for_cr_ready " kserves.components.platform.opendatahub.io" " default-kserve" " Kserve 'default-kserve'"
239+ if ! wait_for_cr_ready " kserves.components.platform.opendatahub.io" " default-kserve" " Kserve 'default-kserve'" ; then
240+ debug_namespace " redhat-ods-operator"
241+ debug_namespace " redhat-ods-applications" " app.kubernetes.io/part-of=kserve"
242+ fi
215243
216244# --- Step 6: Inference Gateway Istio ---
217245echo " --- Inference Gateway Istio ---"
0 commit comments