diff --git a/.github/workflows/build-node-image.yaml b/.github/workflows/build-node-image.yaml index 03a6fcf..866f76c 100644 --- a/.github/workflows/build-node-image.yaml +++ b/.github/workflows/build-node-image.yaml @@ -23,6 +23,10 @@ env: jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + kube-minor: ['1.34', '1.35', '1.36'] permissions: contents: read packages: write @@ -42,13 +46,13 @@ jobs: - name: Build bootc image working-directory: node-images/fedora - run: make build-bootc-image + run: make build-bootc-image KUBE_MINOR=${{ matrix.kube-minor }} - name: Determine image tag id: meta working-directory: node-images/fedora run: | - TAG=$(make -s print-image-tag) + TAG=$(make -s print-image-tag KUBE_MINOR=${{ matrix.kube-minor }}) echo "tag=${TAG}" >> "$GITHUB_OUTPUT" echo "Image tag: ${TAG}" @@ -58,19 +62,17 @@ jobs: working-directory: node-images/fedora run: | TAG=${{ steps.meta.outputs.tag }} - BOOTC_SRC=$(make -s print-bootc-image) + BOOTC_SRC=$(make -s print-bootc-image KUBE_MINOR=${{ matrix.kube-minor }}) PUSH_DEST=${{ env.PUSH_REGISTRY }}/${{ env.PUSH_IMAGE }} podman tag ${BOOTC_SRC} ${PUSH_DEST}:${TAG} podman push --digestfile=/tmp/bootc-digest ${PUSH_DEST}:${TAG} - podman tag ${BOOTC_SRC} ${PUSH_DEST}:latest - podman push ${PUSH_DEST}:latest BOOTC_DIGEST=$(cat /tmp/bootc-digest) echo "digest=${BOOTC_DIGEST}" >> "$GITHUB_OUTPUT" echo "Bootc image pushed with digest: ${BOOTC_DIGEST}" echo "Clean up local images" - podman rmi -f ${BOOTC_SRC} ${PUSH_DEST}:${TAG} ${PUSH_DEST}:latest + podman rmi -f ${BOOTC_SRC} ${PUSH_DEST}:${TAG} echo "push_dest=${PUSH_DEST}:${TAG}" >> "$GITHUB_OUTPUT" - name: Build disk image @@ -80,9 +82,9 @@ jobs: PUSH_DEST="${{ steps.push-bootc.outputs.push_dest }}" if [ -n "${BOOTC_DIGEST}" ] && [ -n "${PUSH_DEST}" ]; then podman pull "${PUSH_DEST}" - make build-disk-image BOOTC_IMAGE="${PUSH_DEST}" BOOTC_DIGEST="${BOOTC_DIGEST}" + make build-disk-image KUBE_MINOR=${{ matrix.kube-minor }} BOOTC_IMAGE="${PUSH_DEST}" BOOTC_DIGEST="${BOOTC_DIGEST}" else - make build-disk-image + make build-disk-image KUBE_MINOR=${{ matrix.kube-minor }} fi - name: Push disk image @@ -90,13 +92,11 @@ jobs: working-directory: node-images/fedora run: | TAG=${{ steps.meta.outputs.tag }} - DISK_SRC=$(make -s print-node-image) + DISK_SRC=$(make -s print-node-image KUBE_MINOR=${{ matrix.kube-minor }}) PUSH_DEST=${{ env.PUSH_REGISTRY }}/${{ env.PUSH_IMAGE }} podman tag ${DISK_SRC} ${PUSH_DEST}:${TAG}-disk podman push ${PUSH_DEST}:${TAG}-disk - podman tag ${DISK_SRC} ${PUSH_DEST}:latest-disk - podman push ${PUSH_DEST}:latest-disk - name: Build composefs disk image working-directory: node-images/fedora @@ -104,9 +104,9 @@ jobs: BOOTC_DIGEST="${{ steps.push-bootc.outputs.digest }}" PUSH_DEST="${{ steps.push-bootc.outputs.push_dest }}" if [ -n "${BOOTC_DIGEST}" ] && [ -n "${PUSH_DEST}" ]; then - make build-disk-image-composefs BOOTC_IMAGE="${PUSH_DEST}" BOOTC_DIGEST="${BOOTC_DIGEST}" + make build-disk-image-composefs KUBE_MINOR=${{ matrix.kube-minor }} BOOTC_IMAGE="${PUSH_DEST}" BOOTC_DIGEST="${BOOTC_DIGEST}" else - make build-disk-image-composefs + make build-disk-image-composefs KUBE_MINOR=${{ matrix.kube-minor }} fi - name: Push composefs disk image @@ -114,10 +114,8 @@ jobs: working-directory: node-images/fedora run: | TAG=${{ steps.meta.outputs.tag }} - DISK_SRC=$(make -s print-node-image-composefs) + DISK_SRC=$(make -s print-node-image-composefs KUBE_MINOR=${{ matrix.kube-minor }}) PUSH_DEST=${{ env.PUSH_REGISTRY }}/${{ env.PUSH_IMAGE }} podman tag ${DISK_SRC} ${PUSH_DEST}:${TAG}-disk-composefs podman push ${PUSH_DEST}:${TAG}-disk-composefs - podman tag ${DISK_SRC} ${PUSH_DEST}:latest-disk-composefs - podman push ${PUSH_DEST}:latest-disk-composefs diff --git a/.github/workflows/integration-tests-k8s-versions.yml b/.github/workflows/integration-tests-k8s-versions.yml new file mode 100644 index 0000000..d47da8b --- /dev/null +++ b/.github/workflows/integration-tests-k8s-versions.yml @@ -0,0 +1,161 @@ +name: Integration Tests (K8s Versions) + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: integration-k8s-versions-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + EXTERNAL_IMAGES: >- + docker.io/library/registry:2 + docker.io/library/haproxy:lts-alpine + quay.io/libpod/busybox:latest + +jobs: + integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + kube-minor: ['1.34', '1.36'] + + env: + BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v${{ matrix.kube-minor }}-fedora-44-disk + BINK_IMAGES: >- + ghcr.io/bootc-dev/bink/cluster:latest + ghcr.io/bootc-dev/bink/node:v${{ matrix.kube-minor }}-fedora-44-disk + ghcr.io/bootc-dev/bink/dns:latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure kernel for nested containers + run: | + sudo aa-teardown 2>/dev/null || true + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + + - name: Enable KSM (Kernel Same-page Merging) + run: | + sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' + sudo sh -c 'echo 5000 > /sys/kernel/mm/ksm/pages_to_scan' + cat /sys/kernel/mm/ksm/run + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + podman \ + libgpgme-dev \ + libbtrfs-dev \ + libdevmapper-dev \ + pkg-config + + - name: Set up KVM + run: | + sudo chmod 666 /dev/kvm + ls -la /dev/kvm + + - name: Configure Podman + run: | + podman --version + sudo mkdir -p /etc/containers + echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json + printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf + grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid + grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid + sudo systemctl start podman.socket + sudo podman info --format '{{.Store.GraphRoot}}' + + - name: Build bink binary + run: sudo make build-bink + + - name: Verify prerequisites + run: | + test -f ./bink + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + df -h / + free -h + + - name: Get image digests + id: digests + run: | + ALL_DIGESTS="" + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + digest=$(skopeo inspect --no-creds "docker://${img}" --format '{{.Digest}}') + echo "${img}: ${digest}" + ALL_DIGESTS="${ALL_DIGESTS}${digest}" + done + echo "hash=$(echo -n "${ALL_DIGESTS}" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + + - name: Restore cached images + id: image-cache + uses: actions/cache/restore@v4 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-k8s-${{ matrix.kube-minor }}-${{ steps.digests.outputs.hash }} + + - name: Load cached images + if: steps.image-cache.outputs.cache-hit == 'true' + run: | + for f in /tmp/podman-image-cache/*.tar; do + sudo podman load -i "$f" + done + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + + - name: Pre-pull container images + if: steps.image-cache.outputs.cache-hit != 'true' + run: | + mkdir -p /tmp/podman-image-cache + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + sudo podman pull "$img" + name=$(echo "$img" | sed 's|[/:]|_|g') + sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" + done + sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache + + - name: Save image cache + if: steps.image-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-k8s-${{ matrix.kube-minor }}-${{ steps.digests.outputs.hash }} + + - name: Run integration tests (K8s ${{ matrix.kube-minor }}) + run: sudo make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" + timeout-minutes: 90 + env: + CONTAINER_HOST: unix:///run/podman/podman.sock + BINK_NODE_IMAGE: ${{ env.BINK_NODE_IMAGE }} + + - name: Collect logs + if: failure() + run: .github/collect-logs.sh + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: test-logs-k8s-${{ matrix.kube-minor }} + path: /tmp/bink-logs/ + + - name: Cleanup test clusters + if: always() + run: | + sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r sudo podman rm -f 2>/dev/null || true + sudo podman volume prune -f 2>/dev/null || true diff --git a/test/integration/cluster_test.go b/test/integration/cluster_test.go index 11cf63e..beaa448 100644 --- a/test/integration/cluster_test.go +++ b/test/integration/cluster_test.go @@ -41,7 +41,7 @@ var _ = Describe("Cluster Lifecycle", func() { targetImgRef := "registry.cluster.local:5000/node:latest" By("Creating cluster with --expose, custom node name, memory ballooning, and target-imgref") - cmd := helpers.BinkCmd("cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef) + cmd := helpers.BinkCmd("cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef, "--node-image", helpers.NodeImage()) session := helpers.RunCommand(cmd) By("Verifying cluster creation command succeeded") diff --git a/test/integration/helpers/cluster.go b/test/integration/helpers/cluster.go index 291d94e..7a7e206 100644 --- a/test/integration/helpers/cluster.go +++ b/test/integration/helpers/cluster.go @@ -5,15 +5,26 @@ package helpers import ( "fmt" + "os" "os/exec" "time" + "github.com/bootc-dev/bink/internal/config" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" ) +// NodeImage returns the node image to use for tests. +// It reads BINK_NODE_IMAGE env var, falling back to config.DefaultNodeImage. +func NodeImage() string { + if img := os.Getenv("BINK_NODE_IMAGE"); img != "" { + return img + } + return config.DefaultNodeImage +} + // GenerateTestClusterName creates a unique cluster name for testing func GenerateTestClusterName() string { return fmt.Sprintf("test-bink-%s", uuid.New().String()[:8]) @@ -45,7 +56,7 @@ func RunCommand(cmd *exec.Cmd, timeout ...time.Duration) *gexec.Session { // Uses auto-assigned ports (--api-port 0) to avoid port conflicts in tests func CreateCluster(name string) { GinkgoWriter.Printf("Creating cluster: %s (with auto-assigned API port)\n", name) - cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--api-port", "0", "--memory", "1900", "--max-memory", "4096") + cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-image", NodeImage()) session := RunCommand(cmd, 10*time.Minute) Expect(session.ExitCode()).To(Equal(0), "Failed to create cluster: %s", string(session.Err.Contents())) } @@ -53,7 +64,7 @@ func CreateCluster(name string) { // CreateClusterWithNodeName creates a cluster with a custom control-plane node name func CreateClusterWithNodeName(name, nodeName string) { GinkgoWriter.Printf("Creating cluster: %s with node name: %s (with auto-assigned API port)\n", name, nodeName) - cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--node-name", nodeName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096") + cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--node-name", nodeName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-image", NodeImage()) session := RunCommand(cmd, 10*time.Minute) Expect(session.ExitCode()).To(Equal(0), "Failed to create cluster: %s", string(session.Err.Contents())) }