Skip to content

Commit 068c4f1

Browse files
adamanciniadamancini
authored
ci(mlflow): speed up k3s CI jobs (#140)
* ci(mlflow): speed up k3s CI jobs - Use 1-node k3s clusters instead of 3-node; GKE keeps 3 nodes - Remove invalid skip-preflights and debug inputs from kots-install action - Replace 5-minute service poll loop with kubectl wait --for=condition=Available * ci(mlflow): apply Copilot review suggestions - Poll for deployment existence before kubectl wait (avoids immediate failure if KOTS hasn't created the Deployment yet) - Add || true to kubectl diagnostics in error handler (prevents bash -e from swallowing the original error when describe returns non-zero) - Fail port-forward step explicitly if MLflow never becomes reachable, with pod logs and port-forward log printed before exiting non-zero * ci(mlflow): fix KOTS port-forward race and reduce postgres instances kubectl wait --for=condition=Available returns immediately for a Deployment with maxUnavailable=1 and replicas=1 because 0 ready pods satisfies the condition (0 >= 1-1). The pod was still Init:0/2 when port-forward ran. Replace with kubectl rollout status which correctly waits until all desired replicas are Running and Ready, with a 10m timeout to cover the CNPG bootstrap + pip init containers. Also set postgres.embedded.instances=1 in the CI test values; the default 3-instance cluster is unnecessary for smoke testing and adds significant startup latency on single-node k3s clusters. * ci(mlflow): increase port-forward readiness poll to cover DB migrations MLflow runs alembic DB migrations on first start before binding to port 5000. With no readiness probe, kubectl rollout status completes as soon as the container process starts, not when the server is actually listening. The 30s curl loop (10x3s) was too short to cover the full migration sequence on GKE. Increase to 10s initial wait + 30 attempts x 5s = up to 160s total, which comfortably covers the migration runtime.
1 parent 3c16b31 commit 068c4f1

2 files changed

Lines changed: 59 additions & 51 deletions

File tree

‎.github/workflows/mlflow-ci.yml‎

Lines changed: 54 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,10 @@ jobs:
166166
cluster:
167167
- distribution: k3s
168168
version: 1.32
169+
nodes: 1
169170
- distribution: gke
170171
version: 1.32
172+
nodes: 3
171173
config:
172174
- name: nodeport-ingress-disabled
173175
values_file: tests/helm/nodeport-ingress-disabled.yaml
@@ -229,7 +231,7 @@ jobs:
229231
kubernetes-version: ${{ matrix.cluster.version }}
230232
cluster-name: mlflow-ci-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-${{ matrix.config.name }}
231233
disk: 100
232-
nodes: 3
234+
nodes: ${{ matrix.cluster.nodes }}
233235
ttl: 1h
234236
export-kubeconfig: true
235237

@@ -303,8 +305,10 @@ jobs:
303305
cluster:
304306
- distribution: k3s
305307
version: 1.32
308+
nodes: 1
306309
- distribution: gke
307310
version: 1.32
311+
nodes: 3
308312
steps:
309313
- name: Checkout
310314
uses: actions/checkout@v4
@@ -340,7 +344,7 @@ jobs:
340344
kubernetes-version: ${{ matrix.cluster.version }}
341345
cluster-name: mlflow-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
342346
disk: 100
343-
nodes: 3
347+
nodes: ${{ matrix.cluster.nodes }}
344348
ttl: 1h
345349
export-kubeconfig: true
346350

@@ -407,71 +411,70 @@ jobs:
407411
namespace: default
408412
wait-duration: 10m
409413
shared-password: 'replicatedmlflow'
410-
skip-preflights: true
411-
debug: true
412414

413415
# Set up port forwarding after KOTS installation is complete
414416
- name: Set up port forwarding
415417
id: port-forward
416418
run: |
417419
KUBECONFIG_FILE="/tmp/kubeconfig-kots-test-${{ github.run_id }}"
418420
echo "$KUBECONFIG" > "$KUBECONFIG_FILE"
419-
echo "Saved kubeconfig to $KUBECONFIG_FILE"
420421
PORT="5000"
421-
echo "Using port: $PORT for testing"
422-
echo "Waiting for MLflow service to be created..."
423-
MAX_RETRIES=30
424-
RETRY_INTERVAL=10
425-
RETRY_COUNT=0
426-
SERVICE_FOUND=false
427-
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
428-
echo "Check $((RETRY_COUNT+1))/$MAX_RETRIES: Looking for MLflow service..."
429-
if KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc mlflow -n default --no-headers 2>/dev/null; then
430-
echo "MLflow service found!"
431-
SERVICE_FOUND=true
422+
423+
echo "Waiting for MLflow deployment to be created..."
424+
for i in $(seq 1 30); do
425+
if KUBECONFIG="$KUBECONFIG_FILE" kubectl get deployment -l app.kubernetes.io/name=mlflow -n default -o name 2>/dev/null | grep -q .; then
426+
echo "MLflow deployment found."
432427
break
433-
else
434-
echo "MLflow service not found yet. Waiting $RETRY_INTERVAL seconds..."
435-
RETRY_COUNT=$((RETRY_COUNT+1))
436-
sleep $RETRY_INTERVAL
437428
fi
429+
echo "MLflow deployment not found yet (attempt $i/30), waiting..."
430+
sleep 10
438431
done
439-
if [ "$SERVICE_FOUND" != "true" ]; then
440-
echo "ERROR: MLflow service not found after $((MAX_RETRIES * RETRY_INTERVAL)) seconds."
441-
echo "Showing all available services in the namespace:"
442-
KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n default
443-
echo "Showing KOTS application status:"
444-
KUBECONFIG="$KUBECONFIG_FILE" kubectl get app -n default || true
445-
echo "Showing all pods in the namespace:"
446-
KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n default
432+
433+
if ! KUBECONFIG="$KUBECONFIG_FILE" kubectl get deployment -l app.kubernetes.io/name=mlflow -n default -o name 2>/dev/null | grep -q .; then
434+
echo "ERROR: MLflow deployment was not created within the expected time"
435+
KUBECONFIG="$KUBECONFIG_FILE" kubectl get deployments,pods -n default || true
447436
exit 1
448437
fi
449-
KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n default
450-
echo "Checking pod status..."
451-
KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n default
452-
echo "Waiting for MLflow pods to be running..."
453-
KUBECONFIG="$KUBECONFIG_FILE" kubectl wait --for=condition=Ready pods --selector=app.kubernetes.io/name=mlflow -n default --timeout=2m || {
454-
echo "WARNING: Timed out waiting for pods to be ready, will try port-forwarding anyway"
455-
KUBECONFIG="$KUBECONFIG_FILE" kubectl describe pods -n default
438+
439+
echo "Waiting for MLflow rollout to complete..."
440+
KUBECONFIG="$KUBECONFIG_FILE" kubectl rollout status deployment/mlflow \
441+
--timeout=10m \
442+
-n default || {
443+
echo "ERROR: MLflow deployment did not complete rollout within 10m"
444+
KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n default || true
445+
KUBECONFIG="$KUBECONFIG_FILE" kubectl describe deployment/mlflow -n default || true
446+
exit 1
456447
}
457-
echo "Setting up port forwarding to run in the background"
448+
449+
echo "Setting up port forwarding..."
458450
nohup bash -c "KUBECONFIG='$KUBECONFIG_FILE' kubectl port-forward -n default svc/mlflow $PORT:5000 &>/tmp/port-forward-kots-${{ github.run_id }}.log" &
459-
PORT_FORWARD_PID=$!
460-
echo "port_forward_pid=$PORT_FORWARD_PID" >> $GITHUB_OUTPUT
461-
echo "Set up port forwarding with PID: $PORT_FORWARD_PID"
451+
echo "port_forward_pid=$!" >> $GITHUB_OUTPUT
462452
echo "hostname=localhost:$PORT" >> $GITHUB_OUTPUT
463-
echo "Test endpoint will be: localhost:$PORT"
464-
echo "Waiting for port-forward to establish..."
465-
sleep 15
466-
echo "Checking connectivity to MLflow..."
467-
if curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/; then
468-
echo "Successfully connected to MLflow service!"
469-
else
470-
echo "Warning: Initial connection attempt failed, service may still be starting"
471-
echo "Port-forward log:"
472-
cat /tmp/port-forward-kots-${{ github.run_id }}.log || true
473-
echo "Pod logs:"
474-
KUBECONFIG="$KUBECONFIG_FILE" kubectl logs -n default -l app.kubernetes.io/name=mlflow --tail=20 || true
453+
454+
echo "Waiting for port-forward to become ready..."
455+
sleep 10
456+
success=false
457+
for i in $(seq 1 30); do
458+
if curl -sf http://localhost:$PORT/health >/dev/null 2>&1; then
459+
echo "MLflow is reachable"
460+
success=true
461+
break
462+
fi
463+
sleep 5
464+
done
465+
466+
if [ "$success" != "true" ]; then
467+
echo "ERROR: MLflow never became reachable via port-forward after 30s"
468+
echo "==== Port-forward log ===="
469+
cat "/tmp/port-forward-kots-${{ github.run_id }}.log" || true
470+
echo "==== MLflow pods ===="
471+
KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -l app.kubernetes.io/name=mlflow -n default || true
472+
pod_name="$(KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -l app.kubernetes.io/name=mlflow -n default -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")"
473+
if [ -n "$pod_name" ]; then
474+
echo "==== Logs for pod $pod_name ===="
475+
KUBECONFIG="$KUBECONFIG_FILE" kubectl logs "$pod_name" -n default || true
476+
fi
477+
exit 1
475478
fi
476479
env:
477480
KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}

‎applications/mlflow/tests/helm/nodeport-ingress-disabled.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,8 @@ mlflow:
1818
# Environment configuration
1919
# env:
2020
# container: []
21+
22+
postgres:
23+
embedded:
24+
# Single instance is sufficient for CI; reduces startup time on 1-node clusters
25+
instances: 1

0 commit comments

Comments
 (0)