|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Real-cluster chart e2e: k3d cluster + RustFS container, operator installed |
| 3 | +# from the Helm charts (CRDs chart first), resources declared via the |
| 4 | +# rustfs-resources chart, convergence and finalizer cleanup asserted. |
| 5 | +# |
| 6 | +# Required: IMAGE=<operator image present in the local docker daemon> |
| 7 | +# Uses: docker, k3d, helm, kubectl |
| 8 | +set -euo pipefail |
| 9 | +cd "$(dirname "$0")/.." |
| 10 | + |
| 11 | +IMAGE="${IMAGE:?set IMAGE to the operator image to test (must exist locally)}" |
| 12 | +K3S_IMAGE="${K3S_IMAGE:-ghcr.io/openprojectx/dockerhub/rancher/k3s:v1.34.9-k3s1}" |
| 13 | +RUSTFS_IMAGE="${RUSTFS_IMAGE:-ghcr.io/openprojectx/dockerhub/rustfs/rustfs:1.0.0-beta.8}" |
| 14 | +# k3s system images (exact tags k3s v1.34.9 wants). Imported into the |
| 15 | +# cluster nodes so nothing is pulled from inside the cluster at runtime — |
| 16 | +# in-cluster docker.io pulls don't work on proxied/firewalled hosts. |
| 17 | +K3S_SYSTEM_IMAGES=( |
| 18 | + docker.io/rancher/mirrored-pause:3.6 |
| 19 | + docker.io/rancher/mirrored-coredns-coredns:1.14.4 |
| 20 | + docker.io/rancher/local-path-provisioner:v0.0.36 |
| 21 | +) |
| 22 | +CLUSTER=rfo-chart-e2e |
| 23 | +RUSTFS_NAME=${CLUSTER}-rustfs |
| 24 | + |
| 25 | +WORK=$(mktemp -d) |
| 26 | +export KUBECONFIG=$WORK/kubeconfig |
| 27 | + |
| 28 | +cleanup() { |
| 29 | + if [ "${KEEP:-}" = "1" ]; then |
| 30 | + echo "KEEP=1: leaving cluster '$CLUSTER' and container '$RUSTFS_NAME' running (kubeconfig: $KUBECONFIG)" |
| 31 | + return |
| 32 | + fi |
| 33 | + k3d cluster delete $CLUSTER >/dev/null 2>&1 || true |
| 34 | + docker rm -f $RUSTFS_NAME >/dev/null 2>&1 || true |
| 35 | + rm -rf "$WORK" |
| 36 | +} |
| 37 | +trap cleanup EXIT |
| 38 | + |
| 39 | +# Relaxed kubelet eviction thresholds so the cluster survives nearly-full |
| 40 | +# host disks (k3s ignores --kubelet-arg for these; a config drop-in works). |
| 41 | +cat > "$WORK/99-eviction.conf" <<EOF |
| 42 | +apiVersion: kubelet.config.k8s.io/v1beta1 |
| 43 | +kind: KubeletConfiguration |
| 44 | +evictionHard: |
| 45 | + nodefs.available: 100Mi |
| 46 | + imagefs.available: 100Mi |
| 47 | + nodefs.inodesFree: 1% |
| 48 | +EOF |
| 49 | + |
| 50 | +echo "== create k3d cluster" |
| 51 | +k3d cluster create $CLUSTER \ |
| 52 | + --image "$K3S_IMAGE" \ |
| 53 | + --no-lb \ |
| 54 | + --k3s-arg '--disable=traefik@server:0' \ |
| 55 | + --k3s-arg '--disable=metrics-server@server:0' \ |
| 56 | + --volume "$WORK/99-eviction.conf:/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/99-eviction.conf@server:0" \ |
| 57 | + --kubeconfig-update-default=false \ |
| 58 | + --kubeconfig-switch-context=false \ |
| 59 | + --wait --timeout 180s |
| 60 | +k3d kubeconfig get $CLUSTER > "$KUBECONFIG" |
| 61 | + |
| 62 | +echo "== start RustFS on the cluster network" |
| 63 | +docker run -d --rm --name $RUSTFS_NAME --network k3d-$CLUSTER \ |
| 64 | + -e RUSTFS_ACCESS_KEY=rustfsadmin -e RUSTFS_SECRET_KEY=rustfsadmin \ |
| 65 | + "$RUSTFS_IMAGE" >/dev/null |
| 66 | + |
| 67 | +echo "== import operator + system images (no in-cluster registry pulls)" |
| 68 | +# `k3d image import` (and plain `docker save`) trips over multi-arch |
| 69 | +# attestation manifests when docker uses the containerd image store, so |
| 70 | +# save each image single-platform and stream it into the node's containerd. |
| 71 | +# The --platform flag only exists (and is only needed) on the containerd |
| 72 | +# store; the classic overlay2 store (GitHub runners) saves single-platform. |
| 73 | +save_args=() |
| 74 | +if [ "$(docker info -f '{{.Driver}}')" = "overlayfs" ]; then |
| 75 | + save_args=(--platform "linux/$(docker version --format '{{.Server.Arch}}')") |
| 76 | +fi |
| 77 | +for img in "$IMAGE" "${K3S_SYSTEM_IMAGES[@]}"; do |
| 78 | + docker image inspect "$img" >/dev/null 2>&1 || docker pull -q "$img" |
| 79 | + docker save "${save_args[@]}" "$img" \ |
| 80 | + | docker exec -i k3d-$CLUSTER-server-0 ctr --namespace k8s.io images import - >/dev/null |
| 81 | +done |
| 82 | + |
| 83 | +echo "== install CRDs chart" |
| 84 | +helm install rustfs-crds charts/rustfs-operator-crds --wait |
| 85 | + |
| 86 | +echo "== install operator chart (skips existing CRDs) with bootstrapped ClusterConnection" |
| 87 | +repo=${IMAGE%:*}; tag=${IMAGE##*:} |
| 88 | +helm install rustfs-operator charts/rustfs-operator \ |
| 89 | + -n rustfs-operator --create-namespace \ |
| 90 | + --set image.repository="$repo" --set image.tag="$tag" --set image.pullPolicy=Never \ |
| 91 | + --set-json "clusterConnections=[{\"name\":\"prod\",\"endpoint\":\"http://$RUSTFS_NAME:9000\",\"credentials\":{\"accessKey\":\"rustfsadmin\",\"secretKey\":\"rustfsadmin\"}}]" \ |
| 92 | + --wait --timeout 300s |
| 93 | + |
| 94 | +echo "== install resources chart" |
| 95 | +kubectl create namespace team-a |
| 96 | +helm install app-storage charts/rustfs-resources -n team-a \ |
| 97 | + -f scripts/testdata/resources-values.yaml |
| 98 | + |
| 99 | +wait_ready() { # <kind> <name> |
| 100 | + for _ in $(seq 1 60); do |
| 101 | + [ "$(kubectl -n team-a get "$1" "$2" -o jsonpath='{.status.ready}' 2>/dev/null)" = "true" ] && { |
| 102 | + echo " $1/$2 ready"; return 0; } |
| 103 | + sleep 3 |
| 104 | + done |
| 105 | + echo "FAIL: timeout waiting for $1/$2:" >&2 |
| 106 | + kubectl -n team-a get "$1" "$2" -o yaml 2>/dev/null | tail -15 >&2 |
| 107 | + return 1 |
| 108 | +} |
| 109 | +wait_ready bucket app-data |
| 110 | +wait_ready policy app-data-rw |
| 111 | +wait_ready user ci-user |
| 112 | + |
| 113 | +echo "== uninstall resources release; finalizers must clean up remote state" |
| 114 | +helm uninstall app-storage -n team-a --wait |
| 115 | +for _ in $(seq 1 60); do |
| 116 | + [ -z "$(kubectl -n team-a get buckets,users,policies -o name 2>/dev/null)" ] && break |
| 117 | + sleep 3 |
| 118 | +done |
| 119 | +remaining=$(kubectl -n team-a get buckets,users,policies -o name 2>/dev/null) |
| 120 | +[ -z "$remaining" ] || { echo "FAIL: CRs not cleaned up: $remaining" >&2; exit 1; } |
| 121 | + |
| 122 | +echo "chart-e2e OK" |
0 commit comments