[MISC] Switch to Canonical k8s (16/edge)

.github/workflows/integration_test.yaml

@@ -101,11 +101,38 @@ jobs:
     needs:
       - collect-integration-tests
     runs-on: ${{ matrix.job.runner }}
-    timeout-minutes: 216  # Sum of steps `timeout-minutes` + 5
+    timeout-minutes: 222  # Sum of steps `timeout-minutes` + 5
     environment:
       name: integration-test
       deployment: false
     steps:
+      - name: Purge Docker
+        # GitHub-hosted Azure runners ship Docker pre-installed and active.
+        # Canonical K8s requires "A system with no previous installations
+        # of containerd/docker as this may cause conflicts"
+        # (see https://documentation.ubuntu.com/canonical-kubernetes
+        # /release-1.32/snap/tutorial/getting-started/).
+        # Completely purging Docker requires also
+        # deleting the Docker-specific `iptables-nft` rulesets
+        # created on installation.
+        timeout-minutes: 5
+        run: |
+          sudo apt-get remove --purge -y docker-ce docker-ce-cli containerd.io docker.io containerd 2>/dev/null || true
+          sudo nft flush ruleset
+          sudo ip link delete docker0 2>/dev/null || true
+      - name: Remove podman CNI bridge config
+        # Ubuntu 24.04 runners ship podman with a CNI bridge config
+        # (/etc/cni/net.d/87-podman-bridge.conflist) that sorts
+        # lexicographically before Cilium's config (05-cilium.conflist).
+        # When that file is present, containerd picks the podman bridge
+        # as the active CNI before Cilium is ready, bypassing the
+        # node.kubernetes.io/network-unavailable taint and scheduling
+        # pods onto the 10.88.0.0/16 bridge instead of the Cilium pod
+        # CIDR. Removing the file ensures Cilium is the only CNI plugin
+        # from the start.
+        timeout-minutes: 1
+        run: |
+          sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist
       - name: Checkout
         timeout-minutes: 3
         uses: actions/checkout@v7

concierge.yaml

@@ -2,13 +2,11 @@ juju:
   model-defaults:
     logging-config: <root>=INFO; unit=DEBUG
 providers:
-  microk8s:
+  k8s:
     enable: true
     bootstrap: true
-    addons:
-      - dns
-      - hostpath-storage
-      - rbac
+    bootstrap-constraints:
+      root-disk: 5G
 host:
   snaps:
     jhack:

spread.yaml

@@ -60,7 +60,7 @@ backends:
       # For example, with microk8s, using `concierge restore` takes twice as long as this (e.g. 6
       # min instead of 3 min between every spread job)
       juju destroy-model --force --no-wait --destroy-storage --no-prompt testing
-      juju kill-controller --no-prompt concierge-microk8s
+      juju kill-controller --no-prompt concierge-k8s
     restore: |
       rm -rf "$SPREAD_PATH"
 
@@ -124,6 +124,17 @@ prepare-each: |
   # `concierge prepare` needs to be run for each spread job in case Juju version changed
   concierge prepare --trace
 
+  # k8s status --wait-ready returns as soon as 1 node is ready
+  # (see https://github.com/canonical/k8s-snap/issues/1789)
+  # but Cilium, CoreDNS, and storage may still be initialising
+  for res in deployment/cilium-operator deployment/coredns deployment/metrics-server daemonset/cilium daemonset/ck-storage-rawfile-csi-node statefulset/ck-storage-rawfile-csi-controller; do
+      if k8s kubectl -n kube-system get "$res" >/dev/null 2>&1; then
+          k8s kubectl -n kube-system rollout status "$res" --timeout=5m
+      else
+          echo "Skipping rollout status for missing $res"
+      fi
+  done
+
   # Unable to set constraint on all models because of Juju bug:
   # https://bugs.launchpad.net/juju/+bug/2065050
   juju set-model-constraints arch="$(dpkg --print-architecture)"

tests/integration/ha_tests/helpers.py

@@ -1082,8 +1082,9 @@ def get_pvcs(ops_test: OpsTest, unit_name: str):
     pvcs = {}
     client = Client(namespace=ops_test.model.name)
     pvc_list = client.list(PersistentVolumeClaim, namespace=ops_test.model.name)
+    unit_label = unit_name.replace("/", "-")
     for pvc in pvc_list:
-        if unit_name.replace("/", "-") in pvc.metadata.name:
+        if unit_label in pvc.metadata.name:
             pvc_storage_name = pvc.metadata.name.replace(unit_name.split("/")[0], "").split("-")[1]
             logger.info(f"got pvc for {pvc_storage_name} storage: {pvc.metadata.name}")
             pvcs[pvc_storage_name] = pvc
@@ -1095,8 +1096,9 @@ def get_pvs(ops_test: OpsTest, unit_name: str):
     pvs = {}
     client = Client(namespace=ops_test.model.name)
     pv_list = client.list(PersistentVolume, namespace=ops_test.model.name)
+    unit_label = unit_name.replace("/", "-")
     for pv in pv_list:
-        if unit_name.replace("/", "-") in str(pv.spec.hostPath.path):
+        if pv.spec.claimRef.name.endswith(unit_label):
             pvc_storage_name = pv.spec.claimRef.name.replace(unit_name.split("/")[0], "").split(
                 "-"
             )[1]

tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh

@@ -12,10 +12,10 @@ fi
 
 deploy_chaos_mesh() {
 	echo "adding chaos-mesh helm repo"
-	microk8s.helm3 repo add chaos-mesh https://charts.chaos-mesh.org
+	sudo k8s helm repo add chaos-mesh https://charts.chaos-mesh.org
 
 	echo "installing chaos-mesh"
-        microk8s.helm3 install chaos-mesh chaos-mesh/chaos-mesh \
+        sudo k8s helm install chaos-mesh chaos-mesh/chaos-mesh \
           --namespace="${chaos_mesh_ns}" \
           --set chaosDaemon.runtime=containerd \
           --set chaosDaemon.socketPath=/var/snap/microk8s/common/run/containerd.sock \

tests/integration/ha_tests/scripts/destroy_chaos_mesh.sh

@@ -11,43 +11,43 @@ fi
 
 destroy_chaos_mesh() {
 	echo "deleting api-resources"
-	for i in $(kubectl api-resources | awk '/chaos-mesh/ {print $1}'); do
-	    timeout 30 kubectl delete "${i}" --all --all-namespaces || true
+	for i in $(sudo k8s kubectl api-resources | awk '/chaos-mesh/ {print $1}'); do
+	    timeout 30 sudo k8s kubectl delete "${i}" --all --all-namespaces || true
 	done
 
-	if kubectl -n "${chaos_mesh_ns}" get mutatingwebhookconfiguration | grep -q 'choas-mesh-mutation'; then
-		timeout 30 kubectl -n "${chaos_mesh_ns}" delete mutatingwebhookconfiguration chaos-mesh-mutation || true
+	if sudo k8s kubectl get mutatingwebhookconfiguration | grep -q 'chaos-mesh-mutation'; then
+		timeout 30 sudo k8s kubectl delete mutatingwebhookconfiguration chaos-mesh-mutation || true
 	fi
 
-	if kubectl -n "${chaos_mesh_ns}" get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation'; then
-		timeout 30 kubectl -n "${chaos_mesh_ns}" delete validatingwebhookconfiguration chaos-mesh-validation || true
+	if sudo k8s kubectl get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation-auth'; then
+		timeout 30 sudo k8s kubectl delete validatingwebhookconfiguration chaos-mesh-validation-auth || true
 	fi
 
-	if kubectl -n "${chaos_mesh_ns}" get validatingwebhookconfiguration | grep -q 'chaos-mesh-validate-auth'; then
-		timeout 30 kubectl -n "${chaos_mesh_ns}" delete validatingwebhookconfiguration chaos-mesh-validate-auth || true
+	if sudo k8s kubectl get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation'; then
+		timeout 30 sudo k8s kubectl delete validatingwebhookconfiguration chaos-mesh-validation || true
 	fi
 
-	if kubectl get clusterrolebinding | grep -q 'chaos-mesh'; then
+	if sudo k8s kubectl get clusterrolebinding | grep -q 'chaos-mesh'; then
 		echo "deleting clusterrolebindings"
-		readarray -t args < <(kubectl get clusterrolebinding | awk '/chaos-mesh/ {print $1}')
-		timeout 30 kubectl delete clusterrolebinding "${args[@]}" || true
+		readarray -t args < <(sudo k8s kubectl get clusterrolebinding | awk '/chaos-mesh/ {print $1}')
+		timeout 30 sudo k8s kubectl delete clusterrolebinding "${args[@]}" || true
 	fi
 
-	if kubectl get clusterrole | grep -q 'chaos-mesh'; then
+	if sudo k8s kubectl get clusterrole | grep -q 'chaos-mesh'; then
 		echo "deleting clusterroles"
-		readarray -t args < <(kubectl get clusterrole | awk '/chaos-mesh/ {print $1}')
-		timeout 30 kubectl delete clusterrole "${args[@]}" || true
+		readarray -t args < <(sudo k8s kubectl get clusterrole | awk '/chaos-mesh/ {print $1}')
+		timeout 30 sudo k8s kubectl delete clusterrole "${args[@]}" || true
 	fi
 
-	if kubectl get crd | grep -q 'chaos-mesh.org'; then
+	if sudo k8s kubectl get crd | grep -q 'chaos-mesh.org'; then
 		echo "deleting crds"
-		readarray -t args < <(kubectl get crd | awk '/chaos-mesh.org/ {print $1}')
-		timeout 30 kubectl delete crd "${args[@]}" || true
+		readarray -t args < <(sudo k8s kubectl get crd | awk '/chaos-mesh.org/ {print $1}')
+		timeout 30 sudo k8s kubectl delete crd "${args[@]}" || true
 	fi
 
-	if [ -n "${chaos_mesh_ns}" ] && microk8s.helm3 repo list --namespace="${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then
+	if [ -n "${chaos_mesh_ns}" ] && sudo k8s helm repo list --namespace="${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then
 		echo "uninstalling chaos-mesh helm repo"
-		microk8s.helm3 uninstall chaos-mesh --namespace="${chaos_mesh_ns}" || true
+		sudo k8s helm uninstall chaos-mesh --namespace="${chaos_mesh_ns}" || true
 	fi
 }
 

tests/integration/helpers.py

@@ -62,7 +62,7 @@ class SecretNotFoundError(Exception):
     check_call(["kubectl", "version", "--client=true"])
     KUBECTL = "kubectl"
 except FileNotFoundError:
-    KUBECTL = "microk8s kubectl"
+    KUBECTL = "sudo k8s kubectl"
 
 logger = logging.getLogger(__name__)
 
@@ -775,14 +775,13 @@ async def scale_application(
     await model.applications[application_name].scale(scale)
     if scale == 0:
         await model.block_until(
-            lambda: len(model.applications[application_name].units) == scale,
-            timeout=1000,
+            lambda: len(model.applications[application_name].units) == scale, timeout=1200
         )
     else:
         await model.wait_for_idle(
             apps=[application_name],
             status="active",
-            timeout=1000,
+            timeout=1200,
             wait_for_exact_units=scale,
         )