diff --git a/.github/kind-kvm-multinode-config.yaml b/.github/kind-kvm-multinode-config.yaml new file mode 100644 index 0000000..ad338dc --- /dev/null +++ b/.github/kind-kvm-multinode-config.yaml @@ -0,0 +1,27 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - hostPath: /dev/kvm + containerPath: /dev/kvm + - hostPath: /tmp/smolvm-kind-control-plane-var-lib + containerPath: /var/lib/smolvm + - hostPath: /tmp/smolvm-kind-control-plane-var-run + containerPath: /var/run/smolvm +- role: worker + extraMounts: + - hostPath: /dev/kvm + containerPath: /dev/kvm + - hostPath: /tmp/smolvm-kind-worker-var-lib + containerPath: /var/lib/smolvm + - hostPath: /tmp/smolvm-kind-worker-var-run + containerPath: /var/run/smolvm +- role: worker + extraMounts: + - hostPath: /dev/kvm + containerPath: /dev/kvm + - hostPath: /tmp/smolvm-kind-worker2-var-lib + containerPath: /var/lib/smolvm + - hostPath: /tmp/smolvm-kind-worker2-var-run + containerPath: /var/run/smolvm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 436e3cd..5b82b7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,27 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + packages: write + jobs: + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Run linters + run: make lint + envtest: name: envtest controller runs-on: ubuntu-latest + needs: [lint] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -25,22 +42,77 @@ jobs: name: envtest-reports path: reports/ + release-artifact: + name: install artifact reproducibility + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Build reproducible install artifact + run: make rc-artifacts VERSION=ci IMG=ghcr.io/${{ github.repository }}/smolvm-operator:${{ github.sha }} RUNTIME_IMG=ghcr.io/${{ github.repository }}/smolvm-runtime:${{ github.sha }} + - name: Upload install artifact + uses: actions/upload-artifact@v4 + with: + name: install-artifact + path: dist/ + + container-images: + name: immutable container images + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish immutable images + run: | + docker build -t ghcr.io/${{ github.repository }}/smolvm-operator:${{ github.sha }} . + docker build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=0.8.1 -t ghcr.io/${{ github.repository }}/smolvm-runtime:${{ github.sha }} . + docker push ghcr.io/${{ github.repository }}/smolvm-operator:${{ github.sha }} + docker push ghcr.io/${{ github.repository }}/smolvm-runtime:${{ github.sha }} + kind-e2e: name: kind topology e2e runs-on: ubuntu-latest + needs: [envtest, release-artifact] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version-file: go.mod cache: true + - name: Prepare KVM-backed multi-node kind mounts + run: | + set -euxo pipefail + test -e /dev/kvm + sudo chmod 666 /dev/kvm + for node in control-plane worker worker2; do + sudo mkdir -p "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + sudo chmod 777 "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + done + - name: Build e2e images and manifest + run: | + docker build -t smolvm-operator:e2e . + docker build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=0.8.1 -t smolvm-runtime:e2e . + make build-installer IMG=smolvm-operator:e2e RUNTIME_IMG=smolvm-runtime:e2e - uses: helm/kind-action@v1 with: cluster_name: kind - - name: Run deployed-topology e2e test + config: .github/kind-kvm-multinode-config.yaml + - name: Load e2e images + run: | + kind load docker-image smolvm-operator:e2e --name kind + kind load docker-image smolvm-runtime:e2e --name kind + - name: Run deployed topology e2e env: RUN_E2E: "true" - KIND_CLUSTER: kind + E2E_INSTALL_MANIFEST: dist/install.yaml run: make test-e2e-report - name: Upload e2e reports if: always() @@ -52,6 +124,7 @@ jobs: smolvm-runtime-e2e: name: smolvm runtime lifecycle e2e runs-on: ubuntu-latest + needs: [envtest, release-artifact] timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -59,22 +132,34 @@ jobs: with: go-version-file: go.mod cache: true - - name: Prepare KVM-backed kind mounts + - name: Prepare KVM-backed multi-node kind mounts run: | set -euxo pipefail test -e /dev/kvm sudo chmod 666 /dev/kvm - sudo mkdir -p /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run - sudo chmod 777 /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run + for node in control-plane worker worker2; do + sudo mkdir -p "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + sudo chmod 777 "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + done + - name: Build runtime e2e images and manifest + run: | + docker build -t smolvm-operator:runtime-e2e . + docker build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=0.8.1 -t smolvm-runtime:runtime-e2e . + make build-installer IMG=smolvm-operator:runtime-e2e RUNTIME_IMG=smolvm-runtime:runtime-e2e - uses: helm/kind-action@v1 with: cluster_name: runtime - config: .github/kind-runtime-config.yaml + config: .github/kind-kvm-multinode-config.yaml + - name: Load runtime e2e images + run: | + kind load docker-image smolvm-operator:runtime-e2e --name runtime + kind load docker-image smolvm-runtime:runtime-e2e --name runtime - name: Run runtime lifecycle e2e env: RUN_E2E: "true" - KIND_CLUSTER: runtime + E2E_INSTALL_MANIFEST: dist/install.yaml RUNTIME_E2E: "true" + KIND_CLUSTER: runtime run: make test-runtime-e2e-report - name: Upload runtime logs if: always() @@ -82,3 +167,63 @@ jobs: with: name: runtime-e2e-logs path: reports/ + + release-hardening-e2e: + name: upgrade multi-node and failure gates + runs-on: ubuntu-latest + needs: [envtest, release-artifact] + if: ${{ vars.PREVIOUS_INSTALL_MANIFEST_URL != '' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Prepare KVM-backed multi-node kind mounts + run: | + set -euxo pipefail + test -e /dev/kvm + sudo chmod 666 /dev/kvm + for node in control-plane worker worker2; do + sudo mkdir -p "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + sudo chmod 777 "/tmp/smolvm-kind-${node}-var-lib" "/tmp/smolvm-kind-${node}-var-run" + done + - name: Build hardening e2e images and manifests + env: + PREVIOUS_INSTALL_MANIFEST_URL: ${{ vars.PREVIOUS_INSTALL_MANIFEST_URL }} + run: | + set -euxo pipefail + if [ -z "$PREVIOUS_INSTALL_MANIFEST_URL" ]; then + echo "PREVIOUS_INSTALL_MANIFEST_URL repository variable is required for upgrade e2e" >&2 + exit 1 + fi + mkdir -p dist + curl -fsSL "$PREVIOUS_INSTALL_MANIFEST_URL" -o dist/previous-install.yaml + docker build -t smolvm-operator:rc . + docker build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=0.8.1 -t smolvm-runtime:rc . + make build-installer IMG=smolvm-operator:rc RUNTIME_IMG=smolvm-runtime:rc + cp dist/install.yaml dist/rc-install.yaml + - uses: helm/kind-action@v1 + with: + cluster_name: hardening + config: .github/kind-kvm-multinode-config.yaml + - name: Load hardening images + run: | + kind load docker-image smolvm-operator:rc --name hardening + kind load docker-image smolvm-runtime:rc --name hardening + - name: Run upgrade, multi-node, and failure checks + env: + RUN_E2E: "true" + E2E_INSTALL_MANIFEST: dist/rc-install.yaml + PREVIOUS_INSTALL_MANIFEST: dist/previous-install.yaml + RC_INSTALL_MANIFEST: dist/rc-install.yaml + run: | + make test-upgrade-e2e-report + make test-multinode-e2e-report + make test-failure-e2e-report + - name: Upload release hardening reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-hardening-reports + path: reports/ diff --git a/.golangci.yml b/.golangci.yml index aed8644..3c5c188 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,38 +3,21 @@ run: allow-parallel-runners: true issues: - # don't skip warning about doc comments - # don't exclude the default set of lint exclude-use-default: false - # restore some of the defaults - # (fill in the rest as needed) - exclude-rules: - - path: "api/*" - linters: - - lll - - path: "internal/*" - linters: - - dupl - - lll + linters: disable-all: true enable: - dupl - - errcheck - - exportloopref - - goconst - gocyclo - gofmt - goimports - gosimple - govet - ineffassign - - lll - misspell - nakedret - prealloc - staticcheck - typecheck - unconvert - - unparam - - unused diff --git a/Makefile b/Makefile index 0967555..4c70344 100644 --- a/Makefile +++ b/Makefile @@ -77,15 +77,30 @@ test-e2e: go test ./test/e2e/ -v -ginkgo.v .PHONY: test-e2e-report -test-e2e-report: gotestsum ## Run e2e tests and write a JUnit report under reports/. +test-e2e-report: gotestsum ## Run deployed topology e2e tests and write a JUnit report under reports/. mkdir -p reports - $(GOTESTSUM) --format testname --junitfile reports/e2e.xml -- ./test/e2e/ -v -ginkgo.v + $(GOTESTSUM) --format testname --junitfile reports/e2e.xml -- ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter='multinode || failure' .PHONY: test-runtime-e2e-report test-runtime-e2e-report: ## Run runtime lifecycle e2e tests and write a Ginkgo JUnit report under reports/. mkdir -p reports go test ./test/e2e -v -ginkgo.v -ginkgo.label-filter=runtime -ginkgo.junit-report=reports/runtime.xml +.PHONY: test-upgrade-e2e-report +test-upgrade-e2e-report: ## Run deployed upgrade e2e tests and write a Ginkgo JUnit report under reports/. + mkdir -p reports + go test ./test/e2e -v -ginkgo.v -ginkgo.label-filter=upgrade -ginkgo.junit-report=reports/upgrade.xml + +.PHONY: test-multinode-e2e-report +test-multinode-e2e-report: ## Run deployed multi-node e2e tests and write a Ginkgo JUnit report under reports/. + mkdir -p reports + go test ./test/e2e -v -ginkgo.v -ginkgo.label-filter=multinode -ginkgo.junit-report=reports/multinode.xml + +.PHONY: test-failure-e2e-report +test-failure-e2e-report: ## Run deployed failure-recovery e2e tests and write a Ginkgo JUnit report under reports/. + mkdir -p reports + go test ./test/e2e -v -ginkgo.v -ginkgo.label-filter=failure -ginkgo.junit-report=reports/failure.xml + .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint $(GOLANGCI_LINT) run @@ -139,15 +154,34 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform rm Dockerfile.cross .PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. +build-installer: manifests generate kustomize ## Generate a consolidated install manifest without mutating config/. mkdir -p dist - @if [ -d "config/crd" ]; then \ - $(KUSTOMIZE) build config/crd > dist/install.yaml; \ - fi - echo "---" >> dist/install.yaml # Add a document separator before appending - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - cd config/runtime && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG} - $(KUSTOMIZE) build config/default >> dist/install.yaml + out="$$(pwd)/dist/install.yaml"; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$${tmpdir}"' EXIT; \ + cp -R config "$${tmpdir}/config"; \ + cd "$${tmpdir}/config/manager" && $(KUSTOMIZE) edit set image controller=${IMG}; \ + cd "$${tmpdir}/config/runtime" && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG}; \ + $(KUSTOMIZE) build "$${tmpdir}/config/default" > "$${out}" + +.PHONY: release-artifacts +release-artifacts: build-installer ## Generate versioned release artifacts under dist/. + @if [ -z "$(VERSION)" ]; then echo "VERSION is required" >&2; exit 1; fi + cp dist/install.yaml dist/smolvm-operator-$(VERSION).yaml + shasum -a 256 dist/smolvm-operator-$(VERSION).yaml > dist/smolvm-operator-$(VERSION).yaml.sha256 + +.PHONY: verify-release-artifacts +verify-release-artifacts: release-artifacts ## Verify release artifacts are reproducible for the current inputs. + tmp="$$(mktemp)"; \ + cp dist/smolvm-operator-$(VERSION).yaml "$${tmp}"; \ + $(MAKE) release-artifacts VERSION=$(VERSION) IMG=$(IMG) RUNTIME_IMG=$(RUNTIME_IMG); \ + cmp "$${tmp}" dist/smolvm-operator-$(VERSION).yaml + +.PHONY: rc-check +rc-check: lint test-report test-e2e-report test-runtime-e2e-report test-upgrade-e2e-report test-multinode-e2e-report test-failure-e2e-report ## Run the release-candidate validation suite. + +.PHONY: rc-artifacts +rc-artifacts: verify-release-artifacts ## Generate and verify RC install artifacts. Requires VERSION, IMG, and RUNTIME_IMG. ##@ Deployment @@ -192,7 +226,7 @@ GOTESTSUM ?= $(LOCALBIN)/gotestsum-$(GOTESTSUM_VERSION) KUSTOMIZE_VERSION ?= v5.3.0 CONTROLLER_TOOLS_VERSION ?= v0.19.0 ENVTEST_VERSION ?= latest -GOLANGCI_LINT_VERSION ?= v1.54.2 +GOLANGCI_LINT_VERSION ?= v1.64.8 GOTESTSUM_VERSION ?= v1.13.0 .PHONY: kustomize diff --git a/cmd/runtime-agent/main.go b/cmd/runtime-agent/main.go index a08464d..f26bdd4 100644 --- a/cmd/runtime-agent/main.go +++ b/cmd/runtime-agent/main.go @@ -473,7 +473,8 @@ func storageGiB(path string) int64 { if err := syscall.Statfs(path, &stat); err != nil { return 0 } - return int64(stat.Bavail) * int64(stat.Bsize) / 1024 / 1024 / 1024 + bytesAvailable := uint64(stat.Bavail) * uint64(stat.Bsize) //nolint:unconvert + return int64(bytesAvailable / 1024 / 1024 / 1024) } func getenv(key, fallback string) string { diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..e7529a6 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,17 @@ +# Compatibility matrix + +| Component | Supported for RC | Notes | +| --- | --- | --- | +| Kubernetes | 1.29 | Envtest and kind validation use Kubernetes 1.29 assets/configuration. | +| smolvm | 0.8.1 | Runtime image downloads the pinned upstream smolvm release. | +| Runtime CPU architecture | linux/amd64 | smolvm 0.8.1 publishes `linux-x86_64`; runtime e2e must run on amd64. | +| Operator CPU architecture | linux/amd64, linux/arm64 | Go binaries are buildable for both; runtime image support is constrained by smolvm artifacts. | +| Container runtime | containerd via kind/Kubernetes CRI | Other CRI implementations require separate validation. | +| Host virtualization | KVM at `/dev/kvm` | Required on every schedulable runtime node. | +| Runtime state | `/var/lib/smolvm`, `/var/run/smolvm` host paths | Mounted by the runtime DaemonSet. | +| Runtime API | HTTPS on pod port 9443 | Bearer token from `smolvm-runtime-auth`; ingress restricted by NetworkPolicy. | + +## Known limitations + +- Bound VMs are node-local and do not migrate automatically after node or runtime loss. +- Force delete removes the Kubernetes finalizer while accepting possible orphaned node-local state. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..fac2795 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,64 @@ +# Operations + +## Install + +Create the runtime token first: + +```sh +kubectl create namespace operator-system --dry-run=client -o yaml | kubectl apply -f - +kubectl -n operator-system create secret generic smolvm-runtime-auth \ + --from-literal=token="$(openssl rand -base64 48)" +``` + +Build and apply a versioned artifact: + +```sh +make rc-artifacts VERSION= IMG= RUNTIME_IMG= +kubectl apply -f dist/smolvm-operator-.yaml +``` + +## Runtime requirements + +Each runtime node needs `/dev/kvm`, `/var/lib/smolvm`, `/var/run/smolvm`, and network connectivity from the controller to runtime pods on TCP 9443. + +## Upgrade + +1. Apply the generated RC artifact. +2. Wait for controller Deployment rollout. +3. Wait for runtime DaemonSet rollout. +4. Confirm existing VMs retain `status.nodeName` and `status.machineName`. +5. Confirm no duplicate runtime machines are created. +6. Exercise stop, start, and delete on an upgraded VM. + +Rollback uses the previous install artifact. Do not roll back CRDs after creating objects with fields unsupported by the older controller. + +## Deletion and force delete + +Normal deletion keeps the finalizer until the owning runtime deletes local state or reports the machine missing. + +If the owning runtime is unavailable, deletion is blocked with `DeletionBlocked=True`. To remove only the Kubernetes object while accepting possible orphaned local state: + +```yaml +metadata: + annotations: + vm.smolvm.dev/force-delete-local-state: "true" +``` + +Record `status.nodeName` before force delete and clean up local state manually on that node. + +## Observability + +Status conditions: + +- `SmolVM`: `Scheduled`, `Ready`, `RuntimeReady`, `GuestReady`, `Reconciled`, `DeletionBlocked`. +- `SmolVMNode`: `Ready`, `KVMAvailable`, `RuntimeReady`, `Schedulable`. + +Expected events/logs include create, start, stop, resize, delete, invalid spec, unsupported update, runtime unavailable, deletion blocked, and force delete paths. + +Recommended alert inputs: + +- stale `SmolVMNode.status.heartbeatTime`; +- `SmolVM` with `DeletionBlocked=True` beyond SLO; +- controller Deployment unavailable; +- runtime DaemonSet unavailable; +- repeated `RuntimeUnavailable` warning events. diff --git a/docs/release-candidate.md b/docs/release-candidate.md new file mode 100644 index 0000000..6d4a276 --- /dev/null +++ b/docs/release-candidate.md @@ -0,0 +1,27 @@ +# Release candidate readiness + +A build is RC-ready only when every required gate below is complete. + +| Area | Gate | Evidence | +| --- | --- | --- | +| Install artifact | Versioned install manifest is reproducible and has a checksum. | `make rc-artifacts VERSION= IMG= RUNTIME_IMG=` | +| Controller tests | Controller envtest passes with coverage and JUnit output. | `make test-report` | +| Topology e2e | KVM-backed kind deployment e2e passes. | `make test-e2e-report` in CI | +| Runtime lifecycle e2e | Create, run, stop, and delete cleanup pass against real smolvm runtime. | `make test-runtime-e2e-report` in CI | +| Upgrade e2e | Previous install artifact upgrades to the RC without losing VM identity. | `make test-upgrade-e2e-report` in CI with `PREVIOUS_INSTALL_MANIFEST_URL` | +| Failure recovery | Runtime unavailability, stale heartbeat, blocked deletion, and force-delete paths are tested. | `make test-failure-e2e-report` in CI | +| Multi-node behavior | Scheduling, pinning, selectors, port conflicts, and no silent migration are tested on at least two runtime nodes. | `make test-multinode-e2e-report` in CI | +| Security | RBAC, privileged runtime, token handling, NetworkPolicy, image provenance, and hostPath use are reviewed. | `docs/security-review.md` | +| Observability | Health checks, events, conditions, logs, and alert inputs are documented. | `docs/operations.md` | +| Compatibility | Supported Kubernetes, smolvm, CPU architecture, container runtime, and KVM requirements are published. | `docs/compatibility.md` | + +## CI requirements + +- E2E jobs run on linux/amd64 runners with `/dev/kvm`. +- `PREVIOUS_INSTALL_MANIFEST_URL` must point to an immutable previous release install manifest. +- Main-branch CI publishes immutable operator and runtime images tagged by commit SHA. + +## Known limitations + +- Bound VMs remain on their recorded node and require operator action after node-local data loss. +- Signed images, SBOMs, and manifest signatures are recommended follow-up release hardening items. diff --git a/docs/security-review.md b/docs/security-review.md new file mode 100644 index 0000000..a4099c2 --- /dev/null +++ b/docs/security-review.md @@ -0,0 +1,39 @@ +# Security review + +## RBAC + +The controller/runtime service account requires: + +| Resource | Verbs | Reason | +| --- | --- | --- | +| `smolvms` | `get`, `list`, `watch`, `create`, `update`, `patch`, `delete` | Reconcile VM lifecycle and finalizers. | +| `smolvms/status` | `get`, `update`, `patch` | Publish scheduling, runtime, and deletion state. | +| `smolvms/finalizers` | `update` | Hold deletion until node-local cleanup completes. | +| `smolvmnodes` | `get`, `list`, `watch`, `create`, `update`, `patch` | Runtime endpoint discovery and runtime node reporting. | +| `smolvmnodes/status` | `get`, `update`, `patch` | Runtime heartbeat, endpoint, and capability reporting. | +| `nodes` | `get`, `list`, `watch` | Scheduling, selectors, and node readiness checks. | +| `events` | `create`, `patch` | Lifecycle and failure event reporting. | + +## Runtime privileges + +The runtime DaemonSet is privileged and mounts `/dev/kvm`, `/var/lib/smolvm`, and `/var/run/smolvm`. Schedule it only on trusted KVM-capable worker nodes. + +## Runtime API token + +The manifests require a `smolvm-runtime-auth` Secret in the operator namespace; no reusable default token is shipped. + +```sh +kubectl create namespace operator-system --dry-run=client -o yaml | kubectl apply -f - +kubectl -n operator-system create secret generic smolvm-runtime-auth \ + --from-literal=token="$(openssl rand -base64 48)" +``` + +Rotate by replacing the Secret and restarting the controller Deployment and runtime DaemonSet. + +## NetworkPolicy + +`config/runtime/networkpolicy.yaml` restricts runtime API ingress on TCP 9443 to pods labeled `control-plane=controller-manager` in the operator namespace. + +## Image provenance + +RC artifacts should use immutable operator and runtime image tags or digests. Main-branch CI publishes SHA-tagged images to GHCR. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b81f4e..df78787 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/gomega" ) +const namespace = "operator-system" + // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { if os.Getenv("RUN_E2E") != "true" { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index c114f06..0000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package e2e - -import ( - "fmt" - "os/exec" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/manuschillerdev/smolvm-operator/test/utils" -) - -const namespace = "operator-system" - -var _ = Describe("controller", Ordered, func() { - const projectImage = "example.com/smolvm-operator:e2e" - - AfterAll(func() { - _, _ = utils.Run(exec.Command("kubectl", "patch", "smolvm", "smolvm-sample", "--type=merge", "-p", `{"metadata":{"finalizers":[]}}`)) - _, _ = utils.Run(exec.Command("kubectl", "delete", "smolvm", "smolvm-sample", "--ignore-not-found=true", "--wait=false")) - _, _ = utils.Run(exec.Command("kubectl", "delete", "-k", "config/default", "--ignore-not-found=true", "--wait=false")) - _, _ = utils.Run(exec.Command("kubectl", "delete", "-k", "config/crd", "--ignore-not-found=true", "--wait=false")) - }) - - It("deploys the controller/runtime topology and reports no eligible nodes without KVM", func() { - By("building the manager image") - _, err := utils.Run(exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))) - Expect(err).NotTo(HaveOccurred()) - - By("loading the manager image into kind") - Expect(utils.LoadImageToKindClusterWithName(projectImage)).To(Succeed()) - - By("installing the CRD") - _, err = utils.Run(exec.Command("make", "install")) - Expect(err).NotTo(HaveOccurred()) - - By("deploying the controller and runtime") - _, err = utils.Run(exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage), fmt.Sprintf("RUNTIME_IMG=%s", projectImage))) - Expect(err).NotTo(HaveOccurred()) - - By("waiting for the controller deployment") - _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "deployment/operator-controller-manager", "-n", namespace, "--timeout=2m")) - Expect(err).NotTo(HaveOccurred()) - - By("waiting for the runtime daemonset pod") - Eventually(func() string { - out, err := utils.Run(exec.Command("kubectl", "get", "daemonset/operator-smolvm-runtime", "-n", namespace, "-o", "jsonpath={.status.desiredNumberScheduled}")) - if err != nil { - return err.Error() - } - return strings.TrimSpace(string(out)) - }, 2*time.Minute, 2*time.Second).ShouldNot(Equal("0")) - - By("applying an unpinned SmolVM resource") - _, err = utils.Run(exec.Command("bash", "-c", ` -cat <=", 2)) + }) + + AfterAll(func() { + if os.Getenv("RUN_E2E") != "true" { + return + } + _, _ = utils.Run(exec.Command("kubectl", "delete", "smolvm", "--all", "--ignore-not-found=true", "--wait=false")) + }) + + It("schedules unpinned, pinned, selector-constrained, and port-conflicting VMs", func() { + nodes := smolVMNodeNames() + Expect(len(nodes)).To(BeNumerically(">=", 2)) + kubectl("label", "smolvmnode", nodes[1], "e2e-role=selected", "--overwrite") + + applySmolVM("hardening-unpinned", "", "", 0) + waitPath("smolvm/hardening-unpinned", "{.status.nodeName}", notEmpty) + waitPath("smolvm/hardening-unpinned", "{.status.machineName}", notEmpty) + + applySmolVM("hardening-pinned", "nodeName: "+nodes[1], "", 0) + waitPath("smolvm/hardening-pinned", "{.status.nodeName}", equals(nodes[1])) + + applySmolVM("hardening-selector", "", "e2e-role: selected", 0) + waitPath("smolvm/hardening-selector", "{.status.nodeName}", equals(nodes[1])) + + applySmolVM("hardening-port-owner", "nodeName: "+nodes[0], "", 18080) + waitPath("smolvm/hardening-port-owner", "{.status.nodeName}", equals(nodes[0])) + applySmolVM("hardening-port-conflict", "nodeName: "+nodes[0], "", 18080) + waitConditionReason("smolvm/hardening-port-conflict", "Scheduled", "NoEligibleNodes") + }) + + It("does not migrate an already-bound VM when its runtime is unavailable", func() { + node := strings.TrimSpace(kubectlOut("get", "smolvm/hardening-unpinned", "-o", "jsonpath={.status.nodeName}")) + Expect(node).NotTo(BeEmpty()) + markRuntimeUnavailable(node) + kubectl("annotate", "smolvm/hardening-unpinned", "e2e-reconcile=runtime-down", "--overwrite") + waitPath("smolvm/hardening-unpinned", "{.status.nodeName}", equals(node)) + waitConditionReason("smolvm/hardening-unpinned", "Reconciled", "RuntimeUnavailable") + }) + + It("blocks deletion while the owning runtime is unavailable and honors force delete", func() { + applySmolVM("hardening-delete-blocked", "", "", 0) + waitPath("smolvm/hardening-delete-blocked", "{.status.nodeName}", notEmpty) + node := strings.TrimSpace(kubectlOut("get", "smolvm/hardening-delete-blocked", "-o", "jsonpath={.status.nodeName}")) + markRuntimeUnavailable(node) + kubectl("delete", "smolvm/hardening-delete-blocked", "--wait=false") + waitConditionReason("smolvm/hardening-delete-blocked", "DeletionBlocked", "RuntimeUnavailable") + kubectl("annotate", "smolvm/hardening-delete-blocked", "vm.smolvm.dev/force-delete-local-state=true", "--overwrite") + Eventually(func() string { + return kubectlOutAllowFail("get", "smolvm/hardening-delete-blocked", "--ignore-not-found") + }, time.Minute, 3*time.Second).Should(BeEmpty()) + }) +}) + +var _ = Describe("upgrade hardening", Ordered, Label("upgrade"), func() { + It("upgrades a previous install artifact to the RC artifact without losing VM identity", func() { + if os.Getenv("RUN_E2E") != "true" { + Skip("set RUN_E2E=true to run deployed upgrade e2e tests") + } + previous := os.Getenv("PREVIOUS_INSTALL_MANIFEST") + rc := os.Getenv("RC_INSTALL_MANIFEST") + if previous == "" || rc == "" { + Fail("PREVIOUS_INSTALL_MANIFEST and RC_INSTALL_MANIFEST are required") + } + createRuntimeSecret() + kubectl("apply", "-f", previous) + rollout("deployment/operator-controller-manager") + rollout("daemonset/operator-smolvm-runtime") + applySmolVM("hardening-upgrade", "", "", 0) + node := waitPath("smolvm/hardening-upgrade", "{.status.nodeName}", notEmpty) + machine := waitPath("smolvm/hardening-upgrade", "{.status.machineName}", notEmpty) + + kubectl("apply", "-f", rc) + rollout("deployment/operator-controller-manager") + rollout("daemonset/operator-smolvm-runtime") + waitPath("smolvm/hardening-upgrade", "{.status.nodeName}", equals(node)) + waitPath("smolvm/hardening-upgrade", "{.status.machineName}", equals(machine)) + kubectl("patch", "smolvm/hardening-upgrade", "--type=merge", "-p", `{"spec":{"running":false}}`) + waitPath("smolvm/hardening-upgrade", "{.status.phase}", equals("Stopped")) + kubectl("patch", "smolvm/hardening-upgrade", "--type=merge", "-p", `{"spec":{"running":true}}`) + waitPath("smolvm/hardening-upgrade", "{.status.phase}", equals("Running")) + kubectl("delete", "smolvm/hardening-upgrade", "--timeout=90s") + }) +}) + +func createRuntimeSecret() { + bash("kubectl create namespace " + namespace + " --dry-run=client -o yaml | kubectl apply -f -") + bash("kubectl -n " + namespace + " create secret generic smolvm-runtime-auth " + + "--from-literal=token=test-token --dry-run=client -o yaml | kubectl apply -f -") +} + +func rollout(resource string) { + kubectl("-n", namespace, "rollout", "status", resource, "--timeout=180s") +} + +func applySmolVM(name, pin, selector string, hostPort int) { + manifest := smolVMManifest(name, pin, selector, hostPort) + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(manifest) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) +} + +func smolVMManifest(name, pin, selector string, hostPort int) string { + var b strings.Builder + b.WriteString("apiVersion: vm.smolvm.dev/v1alpha1\nkind: SmolVM\nmetadata:\n name: ") + b.WriteString(name) + b.WriteString("\nspec:\n") + if pin != "" { + b.WriteString(" ") + b.WriteString(pin) + b.WriteString("\n") + } + if selector != "" { + b.WriteString(" nodeSelector:\n ") + b.WriteString(selector) + b.WriteString("\n") + } + b.WriteString(" running: true\n image: alpine:latest\n resources:\n cpus: 1\n memoryMiB: 128\n") + if hostPort > 0 { + b.WriteString(fmt.Sprintf(" network:\n ports:\n - hostPort: %d\n guestPort: 80\n", hostPort)) + } + return b.String() +} + +func smolVMNodeNames() []string { + return nonEmptyLines(kubectlOut("get", "smolvmnodes", "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}")) +} + +func markRuntimeUnavailable(node string) { + patch := `{"status":{"heartbeatTime":"2000-01-01T00:00:00Z","endpoint":{"podIP":"127.0.0.1","port":1}}}` + kubectl("patch", "smolvmnode", node, "--subresource=status", "--type=merge", "-p", patch) +} + +func waitConditionReason(resource, conditionType, reason string) { + path := "{.status.conditions[?(@.type==\"" + conditionType + "\")].reason}" + waitPath(resource, path, equals(reason)) +} + +type stringPredicate func(string) bool + +func notEmpty(s string) bool { return strings.TrimSpace(s) != "" } +func equals(want string) stringPredicate { + return func(got string) bool { return strings.TrimSpace(got) == want } +} +func waitPath(resource, path string, predicate stringPredicate) string { + var value string + Eventually(func() bool { + value = kubectlOutAllowFail("get", resource, "-o", "jsonpath="+path) + return predicate(value) + }, 2*time.Minute, 3*time.Second).Should(BeTrue(), "last value: %q", value) + return strings.TrimSpace(value) +} + +func bash(script string) { + _, err := utils.Run(exec.Command("bash", "-c", script)) + Expect(err).NotTo(HaveOccurred()) +} + +func kubectl(args ...string) { + _, err := utils.Run(exec.Command("kubectl", args...)) + Expect(err).NotTo(HaveOccurred()) +} + +func kubectlOut(args ...string) string { + out, err := utils.Run(exec.Command("kubectl", args...)) + Expect(err).NotTo(HaveOccurred()) + return string(out) +} + +func kubectlOutAllowFail(args ...string) string { + out, _ := utils.Run(exec.Command("kubectl", args...)) + return string(out) +} + +func nonEmptyLines(s string) []string { + var out []string + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" { + out = append(out, strings.TrimSpace(line)) + } + } + return out +} diff --git a/test/e2e/runtime_lifecycle_test.go b/test/e2e/runtime_lifecycle_test.go index 9441a5d..af2cd4e 100644 --- a/test/e2e/runtime_lifecycle_test.go +++ b/test/e2e/runtime_lifecycle_test.go @@ -86,22 +86,15 @@ var _ = Describe("runtime lifecycle", Label("runtime"), Ordered, func() { }) It("waits for SmolVMNode Ready and RuntimeReady", func() { - nodeName := firstNodeName() - Expect(nodeName).NotTo(BeEmpty()) - + var nodeName string Eventually(func(g Gomega) { - out, err := utils.Run(exec.Command("kubectl", "get", "smolvmnode", nodeName, "-o", "yaml")) + out, err := utils.Run(exec.Command("kubectl", "get", "smolvmnodes", "-o", "yaml")) if err == nil { fmt.Fprintln(GinkgoWriter, string(out)) } - ready, err := kubectlOutputE("get", "smolvmnode", nodeName, "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}`) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(ready).To(Equal("True")) - - runtimeReady, err := kubectlOutputE("get", "smolvmnode", nodeName, "-o", `jsonpath={.status.conditions[?(@.type=="RuntimeReady")].status}`) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(runtimeReady).To(Equal("True")) + nodeName = readyRuntimeNodeName() + g.Expect(nodeName).NotTo(BeEmpty()) }, 3*time.Minute, 2*time.Second).Should(Succeed()) Expect(kubectlOutput("get", "smolvmnode", nodeName, "-o", "jsonpath={.status.endpoint.podNamespace}")).To(Equal(namespace)) @@ -164,8 +157,18 @@ spec: }) }) -func firstNodeName() string { - return kubectlOutput("get", "nodes", "-o", "jsonpath={.items[0].metadata.name}") +func readyRuntimeNodeName() string { + out, err := kubectlOutputE("get", "smolvmnodes", "-o", `jsonpath={range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\t"}{.status.conditions[?(@.type=="RuntimeReady")].status}{"\n"}{end}`) + if err != nil { + return "" + } + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) == 3 && fields[1] == "True" && fields[2] == "True" { + return fields[0] + } + } + return "" } func kubectlOutput(args ...string) string {