From ff52007680694a14bd6efe6ed1cd2a5fc2860535 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 19 Jun 2026 15:36:49 +0300 Subject: [PATCH 1/6] Switch Canonical k8s --- .github/workflows/integration_test.yaml | 29 ++++++++++++++ concierge.yaml | 8 ++-- spread.yaml | 13 ++++++- .../ha_tests/scripts/deploy_chaos_mesh.sh | 4 +- .../ha_tests/scripts/destroy_chaos_mesh.sh | 38 +++++++++---------- tests/integration/helpers.py | 2 +- 6 files changed, 66 insertions(+), 28 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index dcabfdec74f..61fbf6777b0 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,6 +24,35 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + - name: Purge Docker + # GitHub-hosted Azure runners ship Docker pre-installed and active. + # Canonical K8s requires "A system with no previous installations + # of containerd/docker as this may cause conflicts" + # (see https://documentation.ubuntu.com/canonical-kubernetes + # /release-1.32/snap/tutorial/getting-started/). + # Completely purging Docker requires also + # deleting the Docker-specific `iptables-nft` rulesets + # created on installation. + if: ${{ inputs.path-to-charm-directory == 'kubernetes' }} + timeout-minutes: 5 + run: | + sudo apt-get remove --purge -y docker-ce docker-ce-cli containerd.io docker.io containerd 2>/dev/null || true + sudo nft flush ruleset + sudo ip link delete docker0 2>/dev/null || true + - name: Remove podman CNI bridge config + # Ubuntu 24.04 runners ship podman with a CNI bridge config + # (/etc/cni/net.d/87-podman-bridge.conflist) that sorts + # lexicographically before Cilium's config (05-cilium.conflist). + # When that file is present, containerd picks the podman bridge + # as the active CNI before Cilium is ready, bypassing the + # node.kubernetes.io/network-unavailable taint and scheduling + # pods onto the 10.88.0.0/16 bridge instead of the Cilium pod + # CIDR. Removing the file ensures Cilium is the only CNI plugin + # from the start. + if: ${{ inputs.path-to-charm-directory == 'kubernetes' }} + timeout-minutes: 1 + run: | + sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist - name: Checkout uses: actions/checkout@v6 with: diff --git a/concierge.yaml b/concierge.yaml index 0d2bbc8c7a1..6fa2f05709e 100644 --- a/concierge.yaml +++ b/concierge.yaml @@ -2,13 +2,11 @@ juju: model-defaults: logging-config: =INFO; unit=DEBUG providers: - microk8s: + k8s: enable: true bootstrap: true - addons: - - dns - - hostpath-storage - - rbac + bootstrap-constraints: + root-disk: 5G host: snaps: jhack: diff --git a/spread.yaml b/spread.yaml index be564d141dd..80752102bd4 100644 --- a/spread.yaml +++ b/spread.yaml @@ -60,7 +60,7 @@ backends: # For example, with microk8s, using `concierge restore` takes twice as long as this (e.g. 6 # min instead of 3 min between every spread job) juju destroy-model --force --no-wait --destroy-storage --no-prompt testing - juju kill-controller --no-prompt concierge-microk8s + juju kill-controller --no-prompt concierge-k8s restore: | rm -rf "$SPREAD_PATH" @@ -124,6 +124,17 @@ prepare-each: | # `concierge prepare` needs to be run for each spread job in case Juju version changed concierge prepare --trace + # k8s status --wait-ready returns as soon as 1 node is ready + # (see https://github.com/canonical/k8s-snap/issues/1789) + # but Cilium, CoreDNS, and storage may still be initialising + for res in deployment/cilium-operator deployment/coredns deployment/metrics-server daemonset/cilium daemonset/ck-storage-rawfile-csi-node statefulset/ck-storage-rawfile-csi-controller; do + if k8s kubectl -n kube-system get "$res" >/dev/null 2>&1; then + k8s kubectl -n kube-system rollout status "$res" --timeout=5m + else + echo "Skipping rollout status for missing $res" + fi + done + # Unable to set constraint on all models because of Juju bug: # https://bugs.launchpad.net/juju/+bug/2065050 juju set-model-constraints arch="$(dpkg --print-architecture)" diff --git a/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh b/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh index 9aaacc7019f..57ceb9d3ede 100755 --- a/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh +++ b/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh @@ -12,10 +12,10 @@ fi deploy_chaos_mesh() { echo "adding chaos-mesh helm repo" - microk8s.helm3 repo add chaos-mesh https://charts.chaos-mesh.org + sudo k8s helm repo add chaos-mesh https://charts.chaos-mesh.org echo "installing chaos-mesh" - microk8s.helm3 install chaos-mesh chaos-mesh/chaos-mesh \ + sudo k8s helm install chaos-mesh chaos-mesh/chaos-mesh \ --namespace="${chaos_mesh_ns}" \ --set chaosDaemon.runtime=containerd \ --set chaosDaemon.socketPath=/var/snap/microk8s/common/run/containerd.sock \ diff --git a/tests/integration/ha_tests/scripts/destroy_chaos_mesh.sh b/tests/integration/ha_tests/scripts/destroy_chaos_mesh.sh index 4e23b082760..e02cbb51909 100755 --- a/tests/integration/ha_tests/scripts/destroy_chaos_mesh.sh +++ b/tests/integration/ha_tests/scripts/destroy_chaos_mesh.sh @@ -11,43 +11,43 @@ fi destroy_chaos_mesh() { echo "deleting api-resources" - for i in $(kubectl api-resources | awk '/chaos-mesh/ {print $1}'); do - timeout 30 kubectl delete "${i}" --all --all-namespaces || true + for i in $(sudo k8s kubectl api-resources | awk '/chaos-mesh/ {print $1}'); do + timeout 30 sudo k8s kubectl delete "${i}" --all --all-namespaces || true done - if kubectl -n "${chaos_mesh_ns}" get mutatingwebhookconfiguration | grep -q 'choas-mesh-mutation'; then - timeout 30 kubectl -n "${chaos_mesh_ns}" delete mutatingwebhookconfiguration chaos-mesh-mutation || true + if sudo k8s kubectl get mutatingwebhookconfiguration | grep -q 'chaos-mesh-mutation'; then + timeout 30 sudo k8s kubectl delete mutatingwebhookconfiguration chaos-mesh-mutation || true fi - if kubectl -n "${chaos_mesh_ns}" get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation'; then - timeout 30 kubectl -n "${chaos_mesh_ns}" delete validatingwebhookconfiguration chaos-mesh-validation || true + if sudo k8s kubectl get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation-auth'; then + timeout 30 sudo k8s kubectl delete validatingwebhookconfiguration chaos-mesh-validation-auth || true fi - if kubectl -n "${chaos_mesh_ns}" get validatingwebhookconfiguration | grep -q 'chaos-mesh-validate-auth'; then - timeout 30 kubectl -n "${chaos_mesh_ns}" delete validatingwebhookconfiguration chaos-mesh-validate-auth || true + if sudo k8s kubectl get validatingwebhookconfiguration | grep -q 'chaos-mesh-validation'; then + timeout 30 sudo k8s kubectl delete validatingwebhookconfiguration chaos-mesh-validation || true fi - if kubectl get clusterrolebinding | grep -q 'chaos-mesh'; then + if sudo k8s kubectl get clusterrolebinding | grep -q 'chaos-mesh'; then echo "deleting clusterrolebindings" - readarray -t args < <(kubectl get clusterrolebinding | awk '/chaos-mesh/ {print $1}') - timeout 30 kubectl delete clusterrolebinding "${args[@]}" || true + readarray -t args < <(sudo k8s kubectl get clusterrolebinding | awk '/chaos-mesh/ {print $1}') + timeout 30 sudo k8s kubectl delete clusterrolebinding "${args[@]}" || true fi - if kubectl get clusterrole | grep -q 'chaos-mesh'; then + if sudo k8s kubectl get clusterrole | grep -q 'chaos-mesh'; then echo "deleting clusterroles" - readarray -t args < <(kubectl get clusterrole | awk '/chaos-mesh/ {print $1}') - timeout 30 kubectl delete clusterrole "${args[@]}" || true + readarray -t args < <(sudo k8s kubectl get clusterrole | awk '/chaos-mesh/ {print $1}') + timeout 30 sudo k8s kubectl delete clusterrole "${args[@]}" || true fi - if kubectl get crd | grep -q 'chaos-mesh.org'; then + if sudo k8s kubectl get crd | grep -q 'chaos-mesh.org'; then echo "deleting crds" - readarray -t args < <(kubectl get crd | awk '/chaos-mesh.org/ {print $1}') - timeout 30 kubectl delete crd "${args[@]}" || true + readarray -t args < <(sudo k8s kubectl get crd | awk '/chaos-mesh.org/ {print $1}') + timeout 30 sudo k8s kubectl delete crd "${args[@]}" || true fi - if [ -n "${chaos_mesh_ns}" ] && microk8s.helm3 repo list --namespace="${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then + if [ -n "${chaos_mesh_ns}" ] && sudo k8s helm repo list --namespace="${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then echo "uninstalling chaos-mesh helm repo" - microk8s.helm3 uninstall chaos-mesh --namespace="${chaos_mesh_ns}" || true + sudo k8s helm uninstall chaos-mesh --namespace="${chaos_mesh_ns}" || true fi } diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f56fcfc7c87..58bc85fca42 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -59,7 +59,7 @@ class SecretNotFoundError(Exception): check_call(["kubectl", "version", "--client=true"]) KUBECTL = "kubectl" except FileNotFoundError: - KUBECTL = "microk8s kubectl" + KUBECTL = "sudo k8s kubectl" logger = logging.getLogger(__name__) From 351710aecba59453659d62b744589fc271d47a7d Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 19 Jun 2026 16:07:48 +0300 Subject: [PATCH 2/6] Remove monorepo checks --- .github/workflows/integration_test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 61fbf6777b0..28fb08f036d 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -33,7 +33,6 @@ jobs: # Completely purging Docker requires also # deleting the Docker-specific `iptables-nft` rulesets # created on installation. - if: ${{ inputs.path-to-charm-directory == 'kubernetes' }} timeout-minutes: 5 run: | sudo apt-get remove --purge -y docker-ce docker-ce-cli containerd.io docker.io containerd 2>/dev/null || true @@ -49,7 +48,6 @@ jobs: # pods onto the 10.88.0.0/16 bridge instead of the Cilium pod # CIDR. Removing the file ensures Cilium is the only CNI plugin # from the start. - if: ${{ inputs.path-to-charm-directory == 'kubernetes' }} timeout-minutes: 1 run: | sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist From 4698ef3a6dc50efc4ca3c520e2d16df86a5a88d3 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 19 Jun 2026 16:23:53 +0300 Subject: [PATCH 3/6] Wrong place to cleanup --- .github/workflows/integration_test.yaml | 56 ++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 28fb08f036d..55d080fbefd 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,33 +24,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Purge Docker - # GitHub-hosted Azure runners ship Docker pre-installed and active. - # Canonical K8s requires "A system with no previous installations - # of containerd/docker as this may cause conflicts" - # (see https://documentation.ubuntu.com/canonical-kubernetes - # /release-1.32/snap/tutorial/getting-started/). - # Completely purging Docker requires also - # deleting the Docker-specific `iptables-nft` rulesets - # created on installation. - timeout-minutes: 5 - run: | - sudo apt-get remove --purge -y docker-ce docker-ce-cli containerd.io docker.io containerd 2>/dev/null || true - sudo nft flush ruleset - sudo ip link delete docker0 2>/dev/null || true - - name: Remove podman CNI bridge config - # Ubuntu 24.04 runners ship podman with a CNI bridge config - # (/etc/cni/net.d/87-podman-bridge.conflist) that sorts - # lexicographically before Cilium's config (05-cilium.conflist). - # When that file is present, containerd picks the podman bridge - # as the active CNI before Cilium is ready, bypassing the - # node.kubernetes.io/network-unavailable taint and scheduling - # pods onto the 10.88.0.0/16 bridge instead of the Cilium pod - # CIDR. Removing the file ensures Cilium is the only CNI plugin - # from the start. - timeout-minutes: 1 - run: | - sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist - name: Checkout uses: actions/checkout@v6 with: @@ -128,11 +101,38 @@ jobs: needs: - collect-integration-tests runs-on: ${{ matrix.job.runner }} - timeout-minutes: 216 # Sum of steps `timeout-minutes` + 5 + timeout-minutes: 222 # Sum of steps `timeout-minutes` + 5 environment: name: integration-test deployment: false steps: + - name: Purge Docker + # GitHub-hosted Azure runners ship Docker pre-installed and active. + # Canonical K8s requires "A system with no previous installations + # of containerd/docker as this may cause conflicts" + # (see https://documentation.ubuntu.com/canonical-kubernetes + # /release-1.32/snap/tutorial/getting-started/). + # Completely purging Docker requires also + # deleting the Docker-specific `iptables-nft` rulesets + # created on installation. + timeout-minutes: 5 + run: | + sudo apt-get remove --purge -y docker-ce docker-ce-cli containerd.io docker.io containerd 2>/dev/null || true + sudo nft flush ruleset + sudo ip link delete docker0 2>/dev/null || true + - name: Remove podman CNI bridge config + # Ubuntu 24.04 runners ship podman with a CNI bridge config + # (/etc/cni/net.d/87-podman-bridge.conflist) that sorts + # lexicographically before Cilium's config (05-cilium.conflist). + # When that file is present, containerd picks the podman bridge + # as the active CNI before Cilium is ready, bypassing the + # node.kubernetes.io/network-unavailable taint and scheduling + # pods onto the 10.88.0.0/16 bridge instead of the Cilium pod + # CIDR. Removing the file ensures Cilium is the only CNI plugin + # from the start. + timeout-minutes: 1 + run: | + sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist - name: Checkout timeout-minutes: 3 uses: actions/checkout@v6 From 7168660175e43dd38796de79a830bd8a311c1db8 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Sat, 20 Jun 2026 01:07:46 +0300 Subject: [PATCH 4/6] Don't check hostpath --- tests/integration/ha_tests/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/ha_tests/helpers.py b/tests/integration/ha_tests/helpers.py index b4a4c57302a..a35ff937a90 100644 --- a/tests/integration/ha_tests/helpers.py +++ b/tests/integration/ha_tests/helpers.py @@ -1082,8 +1082,9 @@ def get_pvcs(ops_test: OpsTest, unit_name: str): pvcs = {} client = Client(namespace=ops_test.model.name) pvc_list = client.list(PersistentVolumeClaim, namespace=ops_test.model.name) + unit_label = unit_name.replace("/", "-") for pvc in pvc_list: - if unit_name.replace("/", "-") in pvc.metadata.name: + if unit_label in pvc.metadata.name: pvc_storage_name = pvc.metadata.name.replace(unit_name.split("/")[0], "").split("-")[1] logger.info(f"got pvc for {pvc_storage_name} storage: {pvc.metadata.name}") pvcs[pvc_storage_name] = pvc @@ -1095,8 +1096,9 @@ def get_pvs(ops_test: OpsTest, unit_name: str): pvs = {} client = Client(namespace=ops_test.model.name) pv_list = client.list(PersistentVolume, namespace=ops_test.model.name) + unit_label = unit_name.replace("/", "-") for pv in pv_list: - if unit_name.replace("/", "-") in str(pv.spec.hostPath.path): + if pv.spec.claimRef.name.endswith(unit_label): pvc_storage_name = pv.spec.claimRef.name.replace(unit_name.split("/")[0], "").split( "-" )[1] From fde6da5d45d6fc8041718cd64b9396e68a51864f Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Sat, 4 Jul 2026 21:45:16 +0300 Subject: [PATCH 5/6] Increase timeout --- tests/integration/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index db88fbae636..6be0b0e2679 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -775,8 +775,7 @@ async def scale_application( await model.applications[application_name].scale(scale) if scale == 0: await model.block_until( - lambda: len(model.applications[application_name].units) == scale, - timeout=1000, + lambda: len(model.applications[application_name].units) == scale, timeout=1200 ) else: await model.wait_for_idle( From 7ff83cf74b8b81ec34591733e09bc4af78f74e1b Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Sun, 5 Jul 2026 00:04:56 +0300 Subject: [PATCH 6/6] Increase timeout --- tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 6be0b0e2679..f906728d85c 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -781,7 +781,7 @@ async def scale_application( await model.wait_for_idle( apps=[application_name], status="active", - timeout=1000, + timeout=1200, wait_for_exact_units=scale, )